feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_note, # noqa: E402
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeNoteWithoutCommentsTest(unittest.TestCase):
|
||||
def test_no_comments_still_writes_metrics_report(self):
|
||||
info = {
|
||||
"title": "无评论测试笔记",
|
||||
"platform": "xiaohongshu",
|
||||
"source_url": "https://example.com/note",
|
||||
"scraped_at": "",
|
||||
"comment_count_total": 0,
|
||||
"like_count": 12,
|
||||
"favorite_count": 3,
|
||||
"comment_count_metric": 0,
|
||||
"share_count": 1,
|
||||
"view_count": 1000,
|
||||
"collect_count_yxyy": 0,
|
||||
"style_name": "测试款",
|
||||
"creator_name": "测试达人",
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output = Path(tmpdir) / "note.txt"
|
||||
argv = [
|
||||
"analyze_note.py",
|
||||
"--note-id",
|
||||
"1",
|
||||
"--no-send",
|
||||
"--brand",
|
||||
"光影行星",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
with patch.object(
|
||||
analyze_note, "load_note_from_db", return_value=(info, [])
|
||||
), patch.object(
|
||||
analyze_note, "resolve_layer_output", return_value=output
|
||||
), patch.object(sys, "argv", argv):
|
||||
result = analyze_note.main()
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertTrue(output.exists())
|
||||
report = output.read_text(encoding="utf-8-sig")
|
||||
self.assertIn("无有效评论可分析", report)
|
||||
self.assertIn("1,000", report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,11 @@
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
batch_rescrape_douyin as batch,
|
||||
)
|
||||
|
||||
|
||||
def test_result_logged_in_reads_nested_scraper_stats():
|
||||
result = {"stats": {"logged_in": True}}
|
||||
|
||||
assert batch.result_logged_in(result) is True
|
||||
assert batch.result_logged_in({"stats": {"logged_in": False}}) is False
|
||||
assert batch.result_logged_in({}) is False
|
||||
@@ -0,0 +1,253 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama
|
||||
|
||||
|
||||
def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values():
|
||||
for value in ("更新数据", " 更新中 ", "暂无数据", "待更新", "0", "0.0", 0):
|
||||
assert chanmama._classify_exposure_text(value) == "refresh"
|
||||
for value in (23000, "23,000", "10w+", "8.8万+", ""):
|
||||
assert chanmama._classify_exposure_text(value) == "ready"
|
||||
|
||||
|
||||
def test_refresh_then_export_waits_once_after_all_accounts(monkeypatch):
|
||||
events = []
|
||||
urls = ["account-1", "account-2"]
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda _driver, url: events.append(("navigate", url)))
|
||||
results = iter([
|
||||
{"clicked": 2, "pending": 0, "zero_without_button": 0, "pages": 2},
|
||||
{"clicked": 0, "pending": 1, "zero_without_button": 0, "pages": 1},
|
||||
])
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: next(results))
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda seconds: events.append(("sleep", seconds)))
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 123.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda _driver, mtime: events.append(("export", mtime)) or "x.xlsx")
|
||||
monkeypatch.setattr(chanmama, "parse_chanmama_excel", lambda _path: ([{"title": "video"}], None))
|
||||
|
||||
records, summaries = chanmama.refresh_then_export_accounts(object(), urls, wait_seconds=600)
|
||||
|
||||
assert len(records) == 2
|
||||
assert [item["clicked"] for item in summaries] == [2, 0]
|
||||
assert events == [
|
||||
("navigate", "account-1"), ("navigate", "account-2"), ("sleep", 600),
|
||||
("navigate", "account-1"), ("export", 123.0),
|
||||
("navigate", "account-2"), ("export", 123.0),
|
||||
]
|
||||
|
||||
|
||||
def test_refresh_then_export_skips_wait_when_nothing_needs_refresh(monkeypatch):
|
||||
sleeps = []
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
|
||||
"clicked": 0, "pending": 0, "zero_without_button": 0, "pages": 1,
|
||||
})
|
||||
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
|
||||
|
||||
records, _ = chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
|
||||
|
||||
assert records == []
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
class _RefreshElement:
|
||||
def __init__(self, text="", css_class="", element_id="node"):
|
||||
self.text = text
|
||||
self.css_class = css_class
|
||||
self.id = element_id
|
||||
|
||||
def is_displayed(self):
|
||||
return True
|
||||
|
||||
def get_attribute(self, name):
|
||||
return self.css_class if name == "class" else None
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
|
||||
class _RefreshDriver:
|
||||
def __init__(self, nodes=None):
|
||||
self.nodes = list(nodes or [])
|
||||
|
||||
def find_elements(self, _by, selector):
|
||||
return self.nodes if selector == "td .cursor-pointer" else []
|
||||
|
||||
|
||||
def test_next_page_button_keeps_the_source_xpath_fallback():
|
||||
expected = "//span[normalize-space()='下一页']/ancestor::button[1]"
|
||||
button = _RefreshElement("下一页", element_id="next")
|
||||
|
||||
class Driver:
|
||||
def find_elements(self, _by, selector):
|
||||
return [button] if selector == expected else []
|
||||
|
||||
assert chanmama._find_next_page_button(Driver()) is button
|
||||
|
||||
|
||||
def test_find_get_data_targets_selects_actual_cursor_pointer_only():
|
||||
outer = _RefreshElement("获取数据", "detail-text", "outer")
|
||||
clickable = _RefreshElement("获取数据", "cp cursor-pointer", "inner")
|
||||
other = _RefreshElement("查看详情", "cursor-pointer", "other")
|
||||
assert chanmama._find_get_data_targets(_RefreshDriver([outer, clickable, other])) == [clickable]
|
||||
|
||||
|
||||
def test_refresh_zero_exposure_videos_walks_all_pages(monkeypatch):
|
||||
class Button:
|
||||
def __init__(self, disabled):
|
||||
self.disabled = disabled
|
||||
|
||||
class Driver:
|
||||
page = 0
|
||||
|
||||
def execute_script(self, _script, _element):
|
||||
self.page += 1
|
||||
|
||||
driver = Driver()
|
||||
page_results = [
|
||||
{"clicked": 2, "pending": 1, "zero_without_button": 0},
|
||||
{"clicked": 1, "pending": 0, "zero_without_button": 1},
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
chanmama, "_page_signature", lambda current: f"page-{current.page}"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_refresh_current_page",
|
||||
lambda current: page_results[current.page],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_find_next_page_button",
|
||||
lambda current: Button(disabled=current.page == 1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"_pagination_element_disabled",
|
||||
lambda button: button.disabled,
|
||||
)
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert chanmama.refresh_zero_exposure_videos(driver) == {
|
||||
"clicked": 3,
|
||||
"pending": 1,
|
||||
"zero_without_button": 1,
|
||||
"unconfirmed": 0,
|
||||
"pages": 2,
|
||||
}
|
||||
|
||||
|
||||
def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch):
|
||||
sleeps = []
|
||||
monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None)
|
||||
monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: {
|
||||
"clicked": 0, "pending": 0, "zero_without_button": 1, "pages": 1,
|
||||
})
|
||||
monkeypatch.setattr(chanmama.time, "sleep", sleeps.append)
|
||||
monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0)
|
||||
monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None)
|
||||
chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600)
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch):
|
||||
events = []
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
events.append("quit")
|
||||
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True)
|
||||
monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: events.append("saved"))
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"load_cookies",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"login",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: ([], []),
|
||||
)
|
||||
|
||||
assert chanmama.main() == 0
|
||||
assert events == ["saved", "quit"]
|
||||
|
||||
|
||||
def test_acceptance_skips_when_profile_cookie_and_credentials_fail(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
|
||||
class Driver:
|
||||
def quit(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama, "HAS_SELENIUM", True)
|
||||
monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"])
|
||||
monkeypatch.setattr(chanmama, "ACCOUNT", "")
|
||||
monkeypatch.setattr(chanmama, "PASSWORD", "")
|
||||
monkeypatch.setattr(chanmama, "create_driver", lambda: Driver())
|
||||
monkeypatch.setattr(chanmama, "verify_login", lambda _driver: False)
|
||||
monkeypatch.setattr(chanmama, "load_cookies", lambda _driver: False)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"login",
|
||||
lambda _driver: (_ for _ in ()).throw(AssertionError("must not login")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chanmama,
|
||||
"refresh_then_export_accounts",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("must not scrape")
|
||||
),
|
||||
)
|
||||
|
||||
assert chanmama.main() == COOKIE_SKIP_EXIT_CODE
|
||||
assert "profile/cookie session is invalid" in evidence.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_acceptance_captcha_is_non_interactive(monkeypatch, tmp_path):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
|
||||
class Element:
|
||||
def is_displayed(self):
|
||||
return True
|
||||
|
||||
class Driver:
|
||||
current_url = "https://www.chanmama.com/login.html"
|
||||
page_source = ""
|
||||
|
||||
def find_element(self, *_args):
|
||||
return Element()
|
||||
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert chanmama.handle_captcha(Driver()) is False
|
||||
assert "captcha or interactive verification" in evidence.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_cli_preserves_cookie_skip_exit_code(monkeypatch):
|
||||
monkeypatch.setattr(chanmama, "main", lambda: COOKIE_SKIP_EXIT_CODE)
|
||||
|
||||
with pytest.raises(SystemExit) as raised:
|
||||
chanmama.cli()
|
||||
|
||||
assert raised.value.code == COOKIE_SKIP_EXIT_CODE
|
||||
@@ -0,0 +1,840 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import collection_completeness as cc
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
|
||||
|
||||
def test_normalize_title_handles_nfkc_case_and_zero_width():
|
||||
assert cc.normalize_title("ＭａｃＢｏｏｋ\u200b 通勤包！") == "macbook通勤包"
|
||||
|
||||
|
||||
def test_coerce_url_unwraps_feishu_markdown_links():
|
||||
value = "[查看笔记](https://www.xiaohongshu.com/discovery/item/abc123?x=1)"
|
||||
assert cc.coerce_url(value) == "https://www.xiaohongshu.com/discovery/item/abc123?x=1"
|
||||
|
||||
|
||||
def test_match_tasks_prefers_note_id_over_changed_title():
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书中的旧标题",
|
||||
"note_url": "https://www.xiaohongshu.com/explore/abc123",
|
||||
"note_id": "abc123",
|
||||
}]
|
||||
cards = [
|
||||
{"title": "完全不同的新标题", "href": "https://www.xiaohongshu.com/explore/abc123", "note_id": "abc123"},
|
||||
{"title": "飞书中的旧标题", "href": "https://www.xiaohongshu.com/explore/other", "note_id": "other"},
|
||||
]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="xhs")
|
||||
|
||||
assert matched["r1"]["note_id"] == "abc123"
|
||||
assert matched["r1"]["match_method"] == "content_id"
|
||||
|
||||
|
||||
def test_match_tasks_accepts_unique_truncated_title_but_rejects_ambiguous_title():
|
||||
task = {"record_id": "r1", "target_title": "男生长期主义通勤双肩包分享"}
|
||||
unique = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "夏日轻量斜挎包"},
|
||||
]
|
||||
ambiguous = [
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
{"title": "男生长期主义通勤双肩包"},
|
||||
]
|
||||
|
||||
assert cc.match_tasks_to_cards([task], unique, platform="douyin")["r1"]
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], ambiguous, platform="douyin")
|
||||
|
||||
|
||||
def test_title_similarity_ignores_episode_prefix_and_hashtag_suffix():
|
||||
target = "第20集：Pocket4拍旋焦，无后期也能出片？"
|
||||
platform_title = (
|
||||
"Pocket4拍旋焦，无后期也能出片？ #旋焦 #pocket4 #摄影装备 "
|
||||
"无论Pocket3还是Pocket4都可以轻松拍出旋焦效果！"
|
||||
)
|
||||
|
||||
assert cc.title_similarity(platform_title, target) >= 0.98
|
||||
|
||||
|
||||
def test_title_similarity_rejects_different_episode_numbers():
|
||||
assert cc.title_similarity(
|
||||
"第19集：Pocket4拍旋焦，无后期也能出片？",
|
||||
"第20集：Pocket4拍旋焦，无后期也能出片？",
|
||||
) == 0.0
|
||||
|
||||
|
||||
def test_match_tasks_rejects_duplicate_exact_titles_with_different_ids():
|
||||
task = {"record_id": "r1", "target_title": "完全相同的目标标题"}
|
||||
cards = [
|
||||
{"title": "完全相同的目标标题", "note_id": "video-1"},
|
||||
{"title": "完全相同的目标标题", "note_id": "video-2"},
|
||||
]
|
||||
|
||||
assert "r1" not in cc.match_tasks_to_cards([task], cards, platform="douyin")
|
||||
|
||||
|
||||
def test_one_card_is_not_reused_for_two_different_creator_tasks():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "loose",
|
||||
"target_title": "commuter backpack review today",
|
||||
"note_url": "https://v.douyin.com/loose-source/",
|
||||
},
|
||||
{
|
||||
"record_id": "exact",
|
||||
"target_title": "commuter backpack review",
|
||||
"note_url": "https://v.douyin.com/exact-source/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "commuter backpack review", "note_id": "observed-1"}]
|
||||
|
||||
matched = cc.match_tasks_to_cards(tasks, cards, platform="douyin")
|
||||
|
||||
assert set(matched) == {"exact"}
|
||||
|
||||
|
||||
def test_same_source_url_can_share_one_card_across_styles():
|
||||
tasks = [
|
||||
{
|
||||
"record_id": "style-1",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
{
|
||||
"record_id": "style-2",
|
||||
"target_title": "同一篇跨款式笔记",
|
||||
"note_url": "https://v.douyin.com/same-short-link/",
|
||||
},
|
||||
]
|
||||
cards = [{"title": "同一篇跨款式笔记", "note_id": "observed-1"}]
|
||||
|
||||
assert set(cc.match_tasks_to_cards(tasks, cards, platform="douyin")) == {
|
||||
"style-1", "style-2",
|
||||
}
|
||||
|
||||
|
||||
def test_known_mismatched_content_ids_cannot_fall_back_to_same_title():
|
||||
task = {
|
||||
"record_id": "r1",
|
||||
"target_title": "完全相同标题",
|
||||
"note_id": "video-a",
|
||||
}
|
||||
card = {"title": "完全相同标题", "note_id": "video-b"}
|
||||
|
||||
assert cc.match_tasks_to_cards([task], [card], platform="douyin") == {}
|
||||
|
||||
|
||||
def test_xingtu_api_and_dom_duplicate_are_merged_before_matching():
|
||||
cards = [
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": 19481,
|
||||
"note_id": "video-1",
|
||||
"source": "show_items_api",
|
||||
},
|
||||
{
|
||||
"title": "同一条视频",
|
||||
"play_count": "1.9w",
|
||||
"note_id": "",
|
||||
"page": 1,
|
||||
},
|
||||
]
|
||||
|
||||
deduped = cc.deduplicate_observed_cards(cards)
|
||||
matched = cc.match_tasks_to_cards(
|
||||
[{"record_id": "r1", "target_title": "同一条视频"}],
|
||||
deduped,
|
||||
platform="douyin",
|
||||
)
|
||||
|
||||
assert len(deduped) == 1
|
||||
assert matched["r1"]["note_id"] == "video-1"
|
||||
|
||||
|
||||
def test_merge_partial_summary_keeps_existing_success_rows():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total": 2, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "matched": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total": 1, "matched": 1, "filled": 1,
|
||||
"results": [
|
||||
{"record_id": "retry", "status": "success", "matched": True, "write_ok": True},
|
||||
],
|
||||
}
|
||||
|
||||
merged = cc.merge_style_summary(existing, partial)
|
||||
|
||||
assert merged["total"] == 2
|
||||
assert {row["record_id"] for row in merged["results"]} == {"ok", "retry"}
|
||||
assert merged["matched"] == 2
|
||||
assert merged["filled"] == 2
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_summary_contract_detects_missing_or_write_failure():
|
||||
summary = {
|
||||
"total": 2,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success", "matched": True, "write_ok": True},
|
||||
{"record_id": "bad", "status": "write_failure", "matched": True, "write_ok": False},
|
||||
],
|
||||
}
|
||||
finalized = cc.finalize_summary(summary, dry_run=False)
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] == 1
|
||||
|
||||
missing = cc.finalize_summary({"total": 2, "results": summary["results"][:1]}, dry_run=False)
|
||||
assert missing["complete"] is False
|
||||
assert missing["missing_results"] == 1
|
||||
|
||||
|
||||
def _style_for_platform(platform: str):
|
||||
return {
|
||||
"index": 1,
|
||||
"name": "款式A",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "name"},
|
||||
"creator_id": {"field_id": "creator_id"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"read_count_7d": {"field_id": "read7"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_extractors_preserve_note_url_and_content_id(monkeypatch):
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": "标题", "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
xhs_task = pgy.extract_target_tasks(_style_for_platform("小红书"))[0]
|
||||
dy_task = xingtu.extract_target_tasks(_style_for_platform("抖音"))[0]
|
||||
|
||||
assert xhs_task["note_url"].endswith("/abc123")
|
||||
assert xhs_task["note_id"] == "abc123"
|
||||
assert dy_task["note_url"].endswith("/7654321")
|
||||
assert dy_task["note_id"] == "7654321"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "filled_url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.douyin.com/video/7654321"),
|
||||
(xingtu, "抖音", "http://8.99 分享文案 https://v.douyin.com/abc123/"),
|
||||
],
|
||||
)
|
||||
def test_filled_publish_link_triggers_collection_without_platform_validation(
|
||||
monkeypatch, module, platform, filled_url,
|
||||
):
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"name": "达人",
|
||||
"creator_id": "creator-id",
|
||||
"title": "已发布标题",
|
||||
"pub": "2026-07-20",
|
||||
"platform": platform,
|
||||
"url": filled_url,
|
||||
"read7": None,
|
||||
}
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_: [row])
|
||||
|
||||
task = module.extract_target_tasks(_style_for_platform(platform))[0]
|
||||
|
||||
assert task["input_error"] is None
|
||||
|
||||
|
||||
def test_extractors_can_collect_missing_title_when_direct_url_has_content_id(monkeypatch):
|
||||
xhs_style = _style_for_platform("小红书")
|
||||
dy_style = _style_for_platform("抖音")
|
||||
xhs_row = {
|
||||
"record_id": "xhs-r", "name": "达人", "creator_id": "xhs-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "小红书",
|
||||
"url": "https://www.xiaohongshu.com/explore/abc123", "read7": None,
|
||||
}
|
||||
dy_row = {
|
||||
"record_id": "dy-r", "name": "达人", "creator_id": "dy-id",
|
||||
"title": None, "pub": "2026-07-20", "platform": "抖音",
|
||||
"url": "https://www.douyin.com/video/7654321", "read7": None,
|
||||
}
|
||||
monkeypatch.setattr(pgy, "list_records_by_table", lambda *_: [xhs_row])
|
||||
monkeypatch.setattr(xingtu, "list_records_by_table", lambda *_: [dy_row])
|
||||
|
||||
assert pgy.extract_target_tasks(xhs_style)[0]["note_id"] == "abc123"
|
||||
assert xingtu.extract_target_tasks(dy_style)[0]["note_id"] == "7654321"
|
||||
assert xhs_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
assert dy_style["_extract_stats"]["skipped_incomplete"] == 0
|
||||
|
||||
|
||||
def test_missing_critical_mapping_is_an_incomplete_summary(monkeypatch):
|
||||
style = _style_for_platform("抖音")
|
||||
del style["field_map"]["note_url"]
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"list_records_by_table",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("mapping should fail first")),
|
||||
)
|
||||
|
||||
_, summaries = xingtu.collect_tasks_across_styles([style], None)
|
||||
finalized = cc.finalize_summary(summaries[1], dry_run=True)
|
||||
|
||||
assert "note_url" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
class _FakeInput:
|
||||
def __init__(self):
|
||||
self.filled = []
|
||||
|
||||
def click(self):
|
||||
return None
|
||||
|
||||
def fill(self, value):
|
||||
self.filled.append(value)
|
||||
|
||||
def press(self, _key):
|
||||
return None
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, input_):
|
||||
self.first = input_
|
||||
|
||||
def filter(self, **_kwargs):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeSearchPage:
|
||||
def __init__(self):
|
||||
self.input = _FakeInput()
|
||||
self.has_result_calls = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
def evaluate(self, script, arg=None):
|
||||
if "const rows" in script:
|
||||
self.has_result_calls += 1
|
||||
return self.has_result_calls >= 2
|
||||
if "const inputs" in script:
|
||||
return ""
|
||||
raise AssertionError(script)
|
||||
|
||||
def locator(self, _selector):
|
||||
return _FakeLocator(self.input)
|
||||
|
||||
def wait_for_url(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def test_pgy_search_retry_keeps_creator_id_query():
|
||||
page = _FakeSearchPage()
|
||||
|
||||
pgy._ensure_search_results(page, query="xhs-id-123", result_name="达人昵称", max_retries=2)
|
||||
|
||||
assert page.input.filled == ["xhs-id-123"]
|
||||
|
||||
|
||||
def test_apply_matched_task_reports_write_failure(monkeypatch):
|
||||
task = {
|
||||
"record_id": "r1", "publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 1, "base_token": "base", "table_id": "table", "field_map": {},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(pgy, "pick_read_field", lambda *_: ("read_count_7d", "fid", "7天曝光量"))
|
||||
monkeypatch.setattr(pgy, "write_back", lambda *_: False)
|
||||
|
||||
outcome = pgy.apply_matched_task(task, {"read_count": "123", "title": "标题"}, summary, dry_run=False)
|
||||
|
||||
assert outcome["status"] == "write_failure"
|
||||
assert outcome["write_ok"] is False
|
||||
|
||||
|
||||
def test_default_creator_batch_has_no_forced_hour_pause():
|
||||
assert pgy.DEFAULT_BATCH_SIZE == 0
|
||||
assert xingtu.DEFAULT_BATCH_SIZE == 0
|
||||
|
||||
|
||||
def test_repeatable_style_argument_and_selection_keep_every_requested_style():
|
||||
parser = argparse.ArgumentParser()
|
||||
cc.add_repeatable_style_argument(parser, "styles")
|
||||
args = parser.parse_args(["--style", "1", "--style", "3"])
|
||||
styles = [{"index": 1}, {"index": 2}, {"index": 3}]
|
||||
|
||||
assert args.style == [1, 3]
|
||||
assert cc.select_requested_styles(styles, args.style) == [styles[0], styles[2]]
|
||||
with pytest.raises(ValueError, match="99"):
|
||||
cc.select_requested_styles(styles, [1, 99])
|
||||
|
||||
|
||||
def test_pgy_stable_cards_retry_an_initial_empty_render(monkeypatch):
|
||||
sequence = [[], [{"title": "标题", "read_count": "1"}], [{"title": "标题", "read_count": "1"}]]
|
||||
monkeypatch.setattr(pgy, "parse_cards_on_page", lambda _page: sequence.pop(0))
|
||||
|
||||
class Page:
|
||||
waits = 0
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
self.waits += 1
|
||||
|
||||
page = Page()
|
||||
assert pgy.parse_cards_stable(page) == [{"title": "标题", "read_count": "1"}]
|
||||
assert page.waits == 2
|
||||
|
||||
|
||||
def test_xingtu_paginates_before_falling_back_to_search(monkeypatch):
|
||||
pages = [
|
||||
[{"title": "无关视频", "play_count": "10"}],
|
||||
[{"title": "目标视频标题", "play_count": "20"}],
|
||||
]
|
||||
calls = {"next": 0}
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "parse_videos_stable", lambda _page: pages.pop(0))
|
||||
|
||||
def next_page(_page):
|
||||
calls["next"] += 1
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu, "go_next_page", next_page)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [{"record_id": "r1", "target_title": "目标视频标题"}], max_pages=2,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == "20"
|
||||
assert calls["next"] == 1
|
||||
|
||||
|
||||
def test_xingtu_normalizes_show_items_api_video():
|
||||
card = xingtu.normalize_xingtu_api_item({
|
||||
"item_id": 7654321,
|
||||
"item_title": "目标视频标题",
|
||||
"play": 168000,
|
||||
"like": 1200,
|
||||
"comment": 34,
|
||||
"share": 56,
|
||||
"url": "https://www.douyin.com/video/7654321",
|
||||
})
|
||||
|
||||
assert card == {
|
||||
"title": "目标视频标题",
|
||||
"play_count": 168000,
|
||||
"like_count": 1200,
|
||||
"comment_count": 34,
|
||||
"share_count": 56,
|
||||
"href": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}
|
||||
|
||||
|
||||
def test_xingtu_show_items_capture_collects_and_deduplicates_cards():
|
||||
class Response:
|
||||
url = "https://www.xingtu.cn/gw/api/author/get_author_show_items_v2"
|
||||
|
||||
def json(self):
|
||||
return {
|
||||
"latest_item_info": [
|
||||
{"item_id": "1", "title": "视频一", "play": 10},
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
"latest_star_item_info": [
|
||||
{"item_id": "2", "item_title": "视频二", "play": 20},
|
||||
],
|
||||
}
|
||||
|
||||
class Page:
|
||||
def on(self, event, callback):
|
||||
assert event == "response"
|
||||
self.callback = callback
|
||||
|
||||
page = Page()
|
||||
cards = xingtu.setup_show_items_capture(page)
|
||||
page.callback(Response())
|
||||
|
||||
assert [card["note_id"] for card in cards] == ["1", "2"]
|
||||
|
||||
|
||||
def test_xingtu_matches_api_cards_without_dom_pagination(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("DOM should not be needed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"is_blocked",
|
||||
lambda _page: (_ for _ in ()).throw(AssertionError("page should not be needed")),
|
||||
)
|
||||
tasks = [{
|
||||
"record_id": "r1",
|
||||
"target_title": "飞书旧标题",
|
||||
"note_url": "https://www.douyin.com/video/7654321",
|
||||
"note_id": "7654321",
|
||||
}]
|
||||
api_cards = [{
|
||||
"title": "平台上的新标题",
|
||||
"play_count": 168000,
|
||||
"note_id": "7654321",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
object(), tasks, max_pages=5, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["play_count"] == 168000
|
||||
assert found["r1"]["match_method"] == "content_id"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
def test_xingtu_api_fuzzy_candidate_does_not_preempt_later_exact_dom(monkeypatch):
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{"title": "commuter backpack review", "note_id": "exact-card"}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda *_: False)
|
||||
task = {"record_id": "r1", "target_title": "commuter backpack review"}
|
||||
api_cards = [{
|
||||
"title": "commuter backpack review today",
|
||||
"note_id": "fuzzy-card",
|
||||
"source": "show_items_api",
|
||||
}]
|
||||
|
||||
found, _, _ = xingtu.find_videos_for_tasks(
|
||||
object(), [task], max_pages=2, api_cards=api_cards,
|
||||
)
|
||||
|
||||
assert found["r1"]["note_id"] == "exact-card"
|
||||
|
||||
|
||||
def test_browser_session_loss_detection_is_specific():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
assert pgy.is_browser_session_lost(closed)
|
||||
assert xingtu.is_browser_session_lost(closed)
|
||||
assert not pgy.is_browser_session_lost(RuntimeError("ordinary parse failure"))
|
||||
|
||||
|
||||
def test_xingtu_nested_session_loss_is_reraised():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.reraise_if_browser_session_lost(closed)
|
||||
xingtu.reraise_if_browser_session_lost(RuntimeError("ordinary lookup miss"))
|
||||
|
||||
|
||||
def test_xingtu_detail_and_video_search_propagate_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Detail:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
def new_page(self):
|
||||
return Detail()
|
||||
|
||||
class SearchPage:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, _script, _arg=None):
|
||||
return {"found": True, "url": "https://www.xingtu.cn/ad/creator/author-homepage/douyin-video/1"}
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.open_creator_detail(SearchPage(), "达人", "creator-id")
|
||||
|
||||
class Locator:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class VideoPage:
|
||||
def locator(self, _selector):
|
||||
return Locator()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
xingtu.submit_video_search(VideoPage(), "标题")
|
||||
|
||||
|
||||
def test_pgy_fallback_detail_click_propagates_closed_session():
|
||||
closed = RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
class Clicker:
|
||||
first = None
|
||||
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise closed
|
||||
|
||||
class Context:
|
||||
pages = []
|
||||
|
||||
class Page:
|
||||
context = Context()
|
||||
|
||||
def evaluate(self, *_args, **_kwargs):
|
||||
return {"ok": False, "reason": "no-row"}
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Clicker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has been closed"):
|
||||
pgy.open_blogger_detail(Page(), "达人", None)
|
||||
|
||||
|
||||
def test_bilibili_triggered_blocked_inputs_are_visible_terminal_rows():
|
||||
summary = bili.finalize_bili_summary({
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{
|
||||
"record_id": "r1", "status": "blocked_input",
|
||||
"matched": False, "reason": "publish_time_missing", "ok": False,
|
||||
}],
|
||||
})
|
||||
|
||||
assert summary["blocked_input"] == 1
|
||||
assert summary["unresolved"] == 0
|
||||
assert summary["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_missing_url_does_not_trigger_collection(monkeypatch):
|
||||
style = {
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"publish_time": {"field_id": "pub"},
|
||||
"read_count_7d": {"field_id": "s7"},
|
||||
"read_count_14d": {"field_id": "s14"},
|
||||
"read_count_21d": {"field_id": "s21"},
|
||||
"read_count_28d": {"field_id": "s28"},
|
||||
"month_end": {"field_id": "sm"},
|
||||
},
|
||||
}
|
||||
rows = [
|
||||
{
|
||||
"record_id": "no-link",
|
||||
"platform": "B站",
|
||||
"creator": "未发布达人",
|
||||
"url": "",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
{
|
||||
"record_id": "published",
|
||||
"platform": "B站",
|
||||
"creator": "已发布达人",
|
||||
"url": "https://www.bilibili.com/video/BV1234567890",
|
||||
"pub": "2026-07-20",
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(bili, "list_all_records", lambda *_args: rows)
|
||||
monkeypatch.setattr(bili, "fetch_play_count", lambda *_args: 321)
|
||||
monkeypatch.setattr(bili, "write_record", lambda *_args, **_kwargs: True)
|
||||
|
||||
summary = bili.process_style(
|
||||
style,
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=bili.date(2026, 7, 23),
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
assert summary["source_b_records"] == 2
|
||||
assert summary["skipped_no_url"] == 1
|
||||
assert summary["total_b_records"] == 1
|
||||
assert summary["success"] == 1
|
||||
assert summary["blocked_input"] == 0
|
||||
assert [row["record_id"] for row in summary["details"]] == ["published"]
|
||||
|
||||
|
||||
def test_bilibili_partial_retry_preserves_old_success():
|
||||
existing = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 2,
|
||||
"details": [
|
||||
{"record_id": "ok", "status": "success", "ok": True},
|
||||
{"record_id": "retry", "status": "retryable_failure", "ok": False},
|
||||
],
|
||||
}
|
||||
partial = {
|
||||
"style": "款式A", "index": 1, "total_b_records": 1,
|
||||
"details": [{"record_id": "retry", "status": "success", "ok": True}],
|
||||
}
|
||||
|
||||
merged = bili.merge_bili_summary(existing, partial)
|
||||
|
||||
assert merged["total_b_records"] == 2
|
||||
assert {row["record_id"] for row in merged["details"]} == {"ok", "retry"}
|
||||
assert merged["complete"] is True
|
||||
|
||||
|
||||
def test_bilibili_top_level_error_cannot_finalize_as_complete():
|
||||
finalized = bili.finalize_bili_summary({
|
||||
"style": "款式A",
|
||||
"index": 1,
|
||||
"total_b_records": 0,
|
||||
"details": [],
|
||||
"error": "missing platform/url field_id",
|
||||
"unresolved": 1,
|
||||
"retryable_failures": 1,
|
||||
"complete": False,
|
||||
})
|
||||
|
||||
assert finalized["complete"] is False
|
||||
assert finalized["unresolved"] >= 1
|
||||
|
||||
|
||||
def test_bilibili_missing_slot_mapping_is_system_error_not_blocked_input():
|
||||
summary = bili.process_style(
|
||||
{
|
||||
"style": "款式A",
|
||||
"name": "款式A",
|
||||
"index": 1,
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"platform": {"field_id": "platform"},
|
||||
"note_url": {"field_id": "url"},
|
||||
},
|
||||
},
|
||||
only_record_ids=None,
|
||||
dry_run=True,
|
||||
delay=0,
|
||||
session=object(),
|
||||
state={},
|
||||
force_today=None,
|
||||
first_run=False,
|
||||
)
|
||||
|
||||
finalized = bili.finalize_bili_summary(summary, dry_run=True)
|
||||
assert "publish_time" in finalized["error"]
|
||||
assert "read_count_7d" in finalized["error"]
|
||||
assert finalized["complete"] is False
|
||||
|
||||
|
||||
def test_self_douyin_missing_mapping_and_write_failure_are_not_complete(monkeypatch):
|
||||
missing = self_dy.scrape_one_style(
|
||||
{"name": "款式A", "index": 1, "base_token": "base", "table_id": "table", "field_map": {}},
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
assert cc.finalize_summary(missing)["complete"] is False
|
||||
|
||||
style = _style_for_platform("抖音")
|
||||
row = {
|
||||
"record_id": "r1",
|
||||
"platform": "抖音",
|
||||
"name": "自营达人",
|
||||
"title": "目标作品",
|
||||
"url": "https://www.douyin.com/video/1",
|
||||
"pub": "2026-07-20",
|
||||
}
|
||||
|
||||
class Page:
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, _ms):
|
||||
return None
|
||||
|
||||
class Session:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def fetch(self, _url, page_action, wait):
|
||||
page_action(Page())
|
||||
|
||||
monkeypatch.setattr(self_dy, "DynamicSession", Session)
|
||||
monkeypatch.setattr(self_dy, "list_records_by_table", lambda *_: [row])
|
||||
monkeypatch.setattr(self_dy, "dismiss_popups", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "restore_cookies", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "maybe_wait_for_login", lambda *_: None)
|
||||
monkeypatch.setattr(self_dy, "save_state", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
self_dy,
|
||||
"scroll_and_collect_posts",
|
||||
lambda *_args, **_kwargs: [{"title": "目标作品", "play_count": 123}],
|
||||
)
|
||||
monkeypatch.setattr(self_dy, "pick_read_field", lambda *_: ("read_count_7d", "read7", "7天曝光量"))
|
||||
monkeypatch.setattr(self_dy, "write_back", lambda *_: False)
|
||||
|
||||
failed = cc.finalize_summary(
|
||||
self_dy.scrape_one_style(
|
||||
style,
|
||||
login_timeout=1,
|
||||
headless=True,
|
||||
only_record_ids=None,
|
||||
dry_run=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert failed["results"][0]["status"] == "write_failure"
|
||||
assert failed["complete"] is False
|
||||
|
||||
|
||||
def test_run_all_rejects_stale_or_incomplete_summary(tmp_path: Path):
|
||||
path = tmp_path / "summary.json"
|
||||
path.write_text("[]", encoding="utf-8")
|
||||
started_at = path.stat().st_mtime + 1
|
||||
|
||||
assert run_all.validate_summary_payload([], {1}, path, started_at)[0] is False
|
||||
|
||||
path.write_text('[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}]', encoding="utf-8")
|
||||
assert run_all.validate_summary_payload(
|
||||
[{"index": 1, "total": 2, "results": [{"record_id": "a", "status": "success"}]}],
|
||||
{1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is False
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
analyze_comments,
|
||||
analyze_note,
|
||||
daily_report_card,
|
||||
friday_relogin_parallel,
|
||||
)
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
evidence = tmp_path / "content-evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
return evidence
|
||||
|
||||
|
||||
def test_content_write_back_skips_before_lark_cli(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = _enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_called(*args, **kwargs):
|
||||
raise AssertionError("lark-cli wrapper must not be called")
|
||||
|
||||
monkeypatch.setattr(chanmama_scraper, "_call_lark_json", fail_if_called)
|
||||
assert chanmama_scraper._write_back(
|
||||
"base_secret", "tbl_test", "rec_test", "fld_test", 123
|
||||
)
|
||||
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["operation"].endswith("record-upsert")
|
||||
|
||||
|
||||
def test_weekly_summary_skips_table_upsert_but_reports_success(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
weekly_summary_all,
|
||||
"PATHS",
|
||||
SimpleNamespace(tmp_root=tmp_path),
|
||||
)
|
||||
|
||||
def fail_if_called(*args, **kwargs):
|
||||
raise AssertionError("weekly lark-cli wrapper must not be called")
|
||||
|
||||
monkeypatch.setattr(weekly_summary_all, "call_lark_json", fail_if_called)
|
||||
assert weekly_summary_all.write_to_target_table(
|
||||
"base_secret",
|
||||
"tbl_test",
|
||||
"fld_time",
|
||||
"fld_summary",
|
||||
"style-test",
|
||||
"https://example.test/doc",
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_card_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"message_id": "om_test"}),
|
||||
stderr="",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run)
|
||||
results = daily_report_card.send_card_to_recipients(
|
||||
{"schema": "2.0", "body": {"elements": []}},
|
||||
("ou_other_a", "ou_other_b"),
|
||||
"2026-08-01",
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(commands) == 1
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_comment_summary_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
|
||||
monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run)
|
||||
analyze_comments.send_feishu_summary("summary", "ou_other")
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_note_report_sends_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
commands = []
|
||||
monkeypatch.setattr(analyze_note.sys, "platform", "linux")
|
||||
monkeypatch.setattr(
|
||||
analyze_note,
|
||||
"build_card_payload",
|
||||
lambda *args, **kwargs: {"schema": "2.0"},
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
commands.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||||
|
||||
monkeypatch.setattr(analyze_note.subprocess, "run", fake_run)
|
||||
analyze_note.send_feishu_report(
|
||||
{},
|
||||
{},
|
||||
{},
|
||||
[],
|
||||
[],
|
||||
"analysis",
|
||||
"ou_other",
|
||||
tmp_path / "report.md",
|
||||
)
|
||||
|
||||
command = commands[0]
|
||||
assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def test_relogin_notifications_resolve_only_to_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
assert (
|
||||
friday_relogin_parallel.recipient_for_platform("pgy", "ou_other")
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
|
||||
|
||||
def test_run_all_rebinds_browser_environment_to_child_entry(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
observed["target"] = Path(target)
|
||||
observed["base"] = dict(base_environment)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_SCRIPT_ID": "content_marketing:pgy_xhs_scraper_v2.py",
|
||||
"GYXX_BROWSER_CDP_PORT": "child-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
|
||||
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(run_all, "environment_for_child_script", fake_rebind)
|
||||
target = run_all.BASE_DIR / "pgy_xhs_scraper_v2.py"
|
||||
|
||||
environment = run_all.build_child_environment(
|
||||
target,
|
||||
{
|
||||
"KEEP": "yes",
|
||||
"GYXX_BROWSER_CDP_PORT": "parent-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "parent-profile",
|
||||
},
|
||||
)
|
||||
|
||||
assert observed["target"] == target
|
||||
assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "parent-port"
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
|
||||
assert environment["PYTHONIOENCODING"] == "utf-8"
|
||||
assert environment["PYTHONUNBUFFERED"] == "1"
|
||||
@@ -0,0 +1,100 @@
|
||||
from argparse import Namespace
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.daily_creator_exposure_scope import (
|
||||
DEFAULT_MAX_AGE_DAYS,
|
||||
DEFAULT_PUBLISHED_FROM,
|
||||
EXCLUDED_STYLE_NAMES,
|
||||
classify_publish_scope,
|
||||
select_daily_styles,
|
||||
)
|
||||
|
||||
|
||||
def test_daily_scope_excludes_requested_styles():
|
||||
styles = [
|
||||
{"index": 1, "name": "保留款"},
|
||||
{"index": 2, "name": "盖亚微单"},
|
||||
{"index": 3, "name": "逐星GT"},
|
||||
{"index": 4, "name": "晨星2"},
|
||||
{"index": 5, "name": "觅光"},
|
||||
]
|
||||
assert [style["name"] for style in select_daily_styles(styles)] == ["保留款"]
|
||||
assert EXCLUDED_STYLE_NAMES == frozenset({"盖亚微单", "逐星GT", "晨星2", "觅光"})
|
||||
|
||||
|
||||
def test_publish_scope_starts_on_july_first_and_stops_at_30_days():
|
||||
collected_on = date(2026, 7, 31)
|
||||
assert classify_publish_scope(
|
||||
date(2026, 6, 30),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) == "before_publish_cutoff"
|
||||
assert classify_publish_scope(
|
||||
date(2026, 7, 1),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) == "collection_window_complete"
|
||||
assert classify_publish_scope(
|
||||
date(2026, 7, 2),
|
||||
collected_on,
|
||||
published_from=DEFAULT_PUBLISHED_FROM,
|
||||
max_age_days=DEFAULT_MAX_AGE_DAYS,
|
||||
) is None
|
||||
|
||||
|
||||
def test_run_all_daily_scope_passes_filters_and_no_retry_to_children():
|
||||
args = Namespace(dry_run=False, style=[1, 2], daily_scope=True, no_retry=False)
|
||||
command = run_all.build_process_command("pgy", args)
|
||||
assert "--skip-field-prepare" in command
|
||||
assert command[command.index("--published-from") + 1] == "2026-07-01"
|
||||
assert command[command.index("--max-age-days") + 1] == "30"
|
||||
assert "--no-retry" in command
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "platform", "url"),
|
||||
[
|
||||
(pgy, "小红书", "https://www.xiaohongshu.com/explore/eligible"),
|
||||
(xingtu, "抖音", "https://www.douyin.com/video/eligible"),
|
||||
],
|
||||
)
|
||||
def test_platform_task_extraction_applies_daily_publish_window(
|
||||
module, platform, url, monkeypatch
|
||||
):
|
||||
style = {
|
||||
"index": 1,
|
||||
"name": "保留款",
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"field_map": {
|
||||
"creator_name": {"field_id": "creator"},
|
||||
"note_title": {"field_id": "title"},
|
||||
"publish_time": {"field_id": "published"},
|
||||
"note_url": {"field_id": "url"},
|
||||
"platform": {"field_id": "platform"},
|
||||
"daily_exposure": {"field_id": "daily", "field_name": "2026-07-29曝光量"},
|
||||
},
|
||||
}
|
||||
base = {"creator": "达人", "title": "标题", "url": url, "platform": platform}
|
||||
records = [
|
||||
{"record_id": "old", **base, "published": "2026-06-30"},
|
||||
{"record_id": "eligible", **base, "published": "2026-07-02"},
|
||||
{"record_id": "missing", **base, "published": None},
|
||||
]
|
||||
monkeypatch.setattr(module, "list_records_by_table", lambda *_args: records)
|
||||
tasks = module.extract_target_tasks(
|
||||
style,
|
||||
published_from=date(2026, 7, 1),
|
||||
max_age_days=30,
|
||||
collection_date=date(2026, 7, 29),
|
||||
)
|
||||
assert [task["record_id"] for task in tasks] == ["eligible"]
|
||||
assert style["_extract_stats"]["skipped_publish_scope"] == 1
|
||||
assert style["_extract_stats"]["skipped_incomplete"] == 1
|
||||
@@ -0,0 +1,105 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_dashboard_analytics import (
|
||||
build_dashboard_facts,
|
||||
)
|
||||
|
||||
|
||||
class DailyDashboardAnalyticsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tracking = [
|
||||
{"style_name": "大流量", "visitors": 1000, "cart_users": 100, "sales": 20, "refund_orders": 2, "light": "🟡"},
|
||||
{"style_name": "高转化", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8, "light": "🔴"},
|
||||
{"style_name": "极小样本", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0, "light": "🔴"},
|
||||
{"style_name": "缺数据", "visitors": None, "cart_users": None, "sales": None, "refund_orders": None, "light": "无判断"},
|
||||
]
|
||||
self.platform = [
|
||||
{"style_name": "大流量", "platform": "天猫", "visitors": 900, "cart_users": 90, "sales": 18, "refund_orders": 2},
|
||||
{"style_name": "高转化", "platform": "天猫", "visitors": 100, "cart_users": 30, "sales": 20, "refund_orders": 8},
|
||||
{"style_name": "极小样本", "platform": "天猫", "visitors": 10, "cart_users": 5, "sales": 10, "refund_orders": 0},
|
||||
]
|
||||
self.personas = [
|
||||
{
|
||||
"style_name": "大流量", "platform": "天猫", "data_date": date(2026, 7, 15),
|
||||
"gender_top": "女性 60%", "age_top": "26-35岁 45%",
|
||||
"city_top": "一线城市 35%", "buying_power_top": "高级白领 40%",
|
||||
}
|
||||
]
|
||||
|
||||
def test_all_styles_receive_same_metrics_and_diagnosis(self):
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas,
|
||||
{"大流量": {"creator_count": 3, "directions": ["通勤"]}},
|
||||
)
|
||||
self.assertEqual(len(facts["styles"]), 4)
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertEqual(high["conversion_rate"], 0.2)
|
||||
self.assertEqual(high["cart_rate"], 0.3)
|
||||
self.assertEqual(high["refund_rate"], 0.4)
|
||||
missing = next(row for row in facts["styles"] if row["style_name"] == "缺数据")
|
||||
self.assertEqual(missing["diagnosis"], "无数据")
|
||||
|
||||
def test_top_conversion_excludes_tiny_visitor_base(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
self.assertEqual(facts["rankings"]["conversion"][0]["style_name"], "高转化")
|
||||
self.assertEqual(facts["rankings"]["visitors"][0]["style_name"], "大流量")
|
||||
self.assertEqual(facts["rankings"]["refund_rate"][0]["style_name"], "高转化")
|
||||
|
||||
def test_platform_winners_keep_missing_persona_explicit(self):
|
||||
facts = build_dashboard_facts("2026-07-15", self.tracking, self.platform, self.personas, {})
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["platform"], "天猫")
|
||||
self.assertEqual(card["top_conversion"]["style_name"], "高转化")
|
||||
self.assertEqual(card["top_conversion"]["persona_text"], "无数据")
|
||||
self.assertEqual(card["top_visitors"]["style_name"], "大流量")
|
||||
self.assertIn("推测", card["top_visitors"]["interest_hypothesis"])
|
||||
self.assertIn("提转化", card["top_visitors"]["optimization"])
|
||||
|
||||
def test_daily_new_notes_overview_replaces_view_delta_and_creator_ranking(self):
|
||||
notes = [{"note_id": 7, "style_name": "大流量", "creator": "达人A", "platform": "小红书", "view_count": 500}]
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {}, daily_notes=notes
|
||||
)
|
||||
self.assertEqual(facts["daily_new_notes"], notes)
|
||||
self.assertNotIn("creator_connections", facts["rankings"])
|
||||
self.assertNotIn("daily_views", facts["rankings"])
|
||||
|
||||
def test_daily_view_ranking_and_platform_cards_receive_stored_images(self):
|
||||
images = {
|
||||
("高转化", "天猫"): {"local_image_path": "same.jpg", "image_platform": "天猫"},
|
||||
("大流量", "天猫"): {"local_image_path": "visitor.jpg", "image_platform": "天猫"},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_images=images,
|
||||
)
|
||||
card = facts["platform_personas"][0]
|
||||
self.assertEqual(card["top_conversion"]["local_image_path"], "same.jpg")
|
||||
self.assertEqual(card["top_visitors"]["local_image_path"], "visitor.jpg")
|
||||
|
||||
def test_matrix_groups_by_style_category_without_total_exposure(self):
|
||||
signals = {
|
||||
"高转化": {"persona": True, "note_count": 3, "recent_note_exposure": 43000, "comment_count": 80,
|
||||
"review_count": 10, "review_negative_rate": 0.3, "creative_ctr": 0.08},
|
||||
}
|
||||
facts = build_dashboard_facts(
|
||||
"2026-07-15", self.tracking, self.platform, self.personas, {},
|
||||
style_categories={
|
||||
"大流量": "都市机能", "高转化": "智性通勤",
|
||||
"极小样本": "智性通勤", "缺数据": "都市机能",
|
||||
},
|
||||
style_signals=signals,
|
||||
)
|
||||
self.assertEqual([group["style_category"] for group in facts["style_groups"]], ["都市机能", "智性通勤"])
|
||||
self.assertEqual([row["style_name"] for row in facts["style_groups"][0]["styles"]], ["大流量", "缺数据"])
|
||||
high = next(row for row in facts["styles"] if row["style_name"] == "高转化")
|
||||
self.assertNotIn("total_note_exposure", high)
|
||||
self.assertEqual(high["recent_note_exposure"], 43000)
|
||||
self.assertNotIn("total_note_exposure", facts["definitions"])
|
||||
self.assertIn("评价风险", high["diagnosis"])
|
||||
self.assertIn("画像", high["evidence_summary"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
import copy
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.content_marketing import bilibili_scraper as bili
|
||||
from gyxx_flow.modules.content_marketing import feishu_mapping
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
|
||||
TARGET_DATE = date(2026, 7, 28)
|
||||
TARGET_NAME = "2026-07-28曝光量"
|
||||
|
||||
|
||||
def _mapping():
|
||||
return {"tables": [{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {}, "all_field_names": []}]}
|
||||
|
||||
|
||||
def test_daily_exposure_field_name_uses_collection_date():
|
||||
assert feishu_mapping.daily_exposure_field_name(TARGET_DATE) == TARGET_NAME
|
||||
|
||||
|
||||
def test_build_field_map_only_maps_exact_requested_daily_field():
|
||||
fmap, names = feishu_mapping.build_field_map([
|
||||
{"id": "old", "name": "2026-07-27曝光量"}, {"id": "today", "name": TARGET_NAME},
|
||||
], target_date=TARGET_DATE)
|
||||
assert fmap["daily_exposure"] == {"field_id": "today", "field_name": TARGET_NAME}
|
||||
assert names == ["2026-07-27曝光量", TARGET_NAME]
|
||||
|
||||
|
||||
def test_ensure_daily_field_reuses_existing_field(monkeypatch):
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [{"id": "field-today", "name": TARGET_NAME}])
|
||||
calls = []
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
|
||||
assert result["ok"] is True and result["created"] == [] and calls == []
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"] == "field-today"
|
||||
|
||||
|
||||
def test_ensure_daily_field_creates_number_field_then_refreshes(monkeypatch):
|
||||
mapping = _mapping()
|
||||
fields = iter([[], [{"id": "new-field", "name": TARGET_NAME}]])
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: next(fields))
|
||||
calls = []
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True})
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False)
|
||||
assert result["ok"] is True and result["created"] == ["款式A"]
|
||||
payload = calls[0][calls[0].index("--json") + 1]
|
||||
assert TARGET_NAME in payload and '"type": "number"' in payload
|
||||
|
||||
|
||||
def test_ensure_daily_field_dry_run_does_not_call_feishu(monkeypatch):
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: (_ for _ in ()).throw(AssertionError("must not list")))
|
||||
monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda *_: (_ for _ in ()).throw(AssertionError("must not create")))
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, dry_run=True, write=False)
|
||||
assert result["ok"] is True
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"].startswith("dryrun:")
|
||||
|
||||
|
||||
def test_acceptance_skips_field_create_and_keeps_sentinel_in_memory(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
evidence = tmp_path / "evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
mapping = _mapping()
|
||||
monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [])
|
||||
monkeypatch.setattr(
|
||||
feishu_mapping,
|
||||
"call_lark_json",
|
||||
lambda *_: (_ for _ in ()).throw(AssertionError("must not create")),
|
||||
)
|
||||
|
||||
result = feishu_mapping.ensure_daily_exposure_fields(
|
||||
mapping,
|
||||
target_date=TARGET_DATE,
|
||||
write=False,
|
||||
)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["created"] == []
|
||||
assert result["skipped_creations"] == ["款式A"]
|
||||
assert mapping["tables"][0]["field_map"]["daily_exposure"][
|
||||
"field_id"
|
||||
].startswith("acceptance-skipped:")
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["operation"] == "content.mapping.daily-exposure-field-create"
|
||||
|
||||
|
||||
def test_acceptance_hydrates_cached_mapping_without_persisting_sentinel(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
cache = tmp_path / "mapping.json"
|
||||
cache.write_text(json.dumps(_mapping(), ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
hydrated = feishu_mapping._hydrate_acceptance_daily_field(
|
||||
json.loads(cache.read_text(encoding="utf-8")),
|
||||
self_operated=False,
|
||||
)
|
||||
|
||||
assert hydrated["tables"][0]["field_map"]["daily_exposure"][
|
||||
"field_id"
|
||||
].startswith("acceptance-skipped:")
|
||||
persisted = json.loads(cache.read_text(encoding="utf-8"))
|
||||
assert "daily_exposure" not in persisted["tables"][0]["field_map"]
|
||||
|
||||
|
||||
def test_collaboration_slot_pickers_use_daily_field_without_publish_time():
|
||||
fmap = {"daily_exposure": {"field_id": "today-field", "field_name": TARGET_NAME}}
|
||||
expected = ("daily_exposure", "today-field", TARGET_NAME)
|
||||
assert pgy.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
|
||||
assert xingtu.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected
|
||||
|
||||
|
||||
def test_self_operated_mapping_removes_daily_field(monkeypatch):
|
||||
source = {"tables": [{"field_map": {
|
||||
"daily_exposure": {"field_id": "daily", "field_name": TARGET_NAME},
|
||||
"read_count_7d": {"field_id": "s7", "field_name": "7天曝光量"},
|
||||
}}]}
|
||||
monkeypatch.setattr(feishu_mapping, "load_mapping", lambda *_args, **_kwargs: copy.deepcopy(source))
|
||||
for mapping in (pgy.load_mapping(True), xingtu.load_mapping(True), bili.load_mapping(bili.DEFAULT_DATA_DIR, True)):
|
||||
assert "daily_exposure" not in mapping["tables"][0]["field_map"]
|
||||
@@ -0,0 +1,442 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import daily_marketing_report as daily
|
||||
from gyxx_flow.modules.content_marketing.data.tools import analyze_comments
|
||||
|
||||
|
||||
class DailyMarketingReportTests(unittest.TestCase):
|
||||
def test_hermes_url_accepts_api_base_or_full_completion_path(self):
|
||||
full_url = "http://127.0.0.1:8642/v1/chat/completions"
|
||||
|
||||
self.assertEqual(
|
||||
analyze_comments._chat_completions_url("http://127.0.0.1:8642/v1"),
|
||||
full_url,
|
||||
)
|
||||
self.assertEqual(
|
||||
analyze_comments._chat_completions_url(full_url),
|
||||
full_url,
|
||||
)
|
||||
|
||||
def test_sales_forecast_uses_seven_day_total_and_recent_three_day_velocity(self):
|
||||
forecast = daily.estimate_sales_forecast([10, 12, 14, 16, 18, 20, 22])
|
||||
|
||||
self.assertEqual(forecast["sales_7d_total"], 112)
|
||||
self.assertEqual(forecast["sales_forecast_7d"], 140)
|
||||
self.assertAlmostEqual(forecast["sales_forecast_change"], 7 / 13, places=4)
|
||||
|
||||
def test_sales_forecast_requires_complete_seven_day_history(self):
|
||||
self.assertEqual(
|
||||
daily.estimate_sales_forecast([10, 12, 14]),
|
||||
{
|
||||
"sales_7d_total": None,
|
||||
"sales_forecast_7d": None,
|
||||
"sales_forecast_change": None,
|
||||
},
|
||||
)
|
||||
|
||||
def test_default_recipients_include_all_four_daily_report_owners(self):
|
||||
self.assertEqual(
|
||||
daily.DEFAULT_RECIPIENT_OPEN_IDS,
|
||||
(
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"ou_fa8d81a16527ad06352dbecc575285b8",
|
||||
"ou_2eda5eec112109ae6d19a1f6813eadcb",
|
||||
"ou_89bcff110ccbb09a23548dc0fb3d880c",
|
||||
),
|
||||
)
|
||||
|
||||
def test_hermes_transport_error_is_not_accepted_as_report(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "Hermes 分析失败"):
|
||||
daily.require_valid_hermes_report("API call failed after 3 retries: Connection error.")
|
||||
self.assertEqual(daily.require_valid_hermes_report("整体经营分析\n销量保持稳定"), "整体经营分析\n销量保持稳定")
|
||||
|
||||
def test_light_uses_highest_absolute_visitor_or_cart_change(self):
|
||||
self.assertEqual(daily.classify_light(0.05, -0.08), "🟢")
|
||||
self.assertEqual(daily.classify_light(0.12, 0.02), "🟡")
|
||||
self.assertEqual(daily.classify_light(-0.31, 0.01), "🔴")
|
||||
self.assertEqual(daily.classify_light(None, None), "无判断")
|
||||
|
||||
def test_comment_signal_requires_minimum_sample(self):
|
||||
comments = ["光影行星宙斯怎么买"] * 49
|
||||
self.assertIsNone(daily.calculate_comment_signal("宙斯", comments))
|
||||
|
||||
comments.append("好看")
|
||||
signal = daily.calculate_comment_signal("宙斯", comments)
|
||||
self.assertEqual(signal["sample_size"], 50)
|
||||
self.assertEqual(signal["brand_mention_rate"], 0.98)
|
||||
self.assertEqual(signal["purchase_intent_rate"], 0.98)
|
||||
|
||||
def test_prompt_uses_only_the_fresh_user_instruction_and_database_facts(self):
|
||||
prompt = daily.build_daily_prompt(
|
||||
report_date="2026-07-15",
|
||||
today_notes=[],
|
||||
tracking_rows=[],
|
||||
next_day_rows=[],
|
||||
comment_rows=[],
|
||||
)
|
||||
self.assertIn("以下为数据库查询结果", prompt)
|
||||
self.assertIn('"报告日期": "2026-07-15"', prompt)
|
||||
self.assertIn("增长型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("流量机会型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("潜力型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("风险型SKU", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("A类：爆款增长SKU", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_fashion_bag_data_analyst_role(self):
|
||||
self.assertIn("专业的时尚包袋品牌电商运营数据分析师", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("销售、电商运营及达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("发现增长机会和风险", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("SKU营销运营日报", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("展示关键数据", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_fresh_prompt_contains_the_complete_requested_analysis_structure(self):
|
||||
prompt = daily.SYSTEM_PROMPT
|
||||
for phrase in (
|
||||
"今日整体经营判断",
|
||||
"SKU经营表现分析",
|
||||
"达人营销效果分析",
|
||||
"产品节奏判断",
|
||||
"蓄水期",
|
||||
"放量期",
|
||||
"稳定期",
|
||||
"衰退期",
|
||||
"增长SKU TOP3",
|
||||
"风险SKU TOP3",
|
||||
"比较昨日、七日基准",
|
||||
"用图表展示",
|
||||
"曝光量",
|
||||
"建议动作",
|
||||
"对销量影响",
|
||||
"所有建议必须具体到SKU",
|
||||
):
|
||||
self.assertIn(phrase, prompt)
|
||||
for retired in ("A类：爆款增长SKU", "B类：潜力增长SKU", "D类：需要优化SKU"):
|
||||
self.assertNotIn(retired, prompt)
|
||||
|
||||
def test_database_payload_keeps_all_available_sources(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"reviews": [{"sample_size": 3}]})
|
||||
for phrase in ("所有SKU电商运营数据", "达人营销内容数据", "评论与补充经营数据", "reviews"):
|
||||
self.assertIn(phrase, prompt)
|
||||
|
||||
def test_note_label_ignores_placeholder_creator_name(self):
|
||||
note = {"id": 7, "creator_name": "/", "title": "瑞白通勤包实测"}
|
||||
self.assertEqual(daily.note_label(note), "瑞白通勤包实测")
|
||||
|
||||
def test_cooperation_metrics_hide_unverifiable_engagement(self):
|
||||
invalid = daily._sanitize_cooperation_row({
|
||||
"publish_time": None,
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 21430000,
|
||||
})
|
||||
self.assertIsNone(invalid["engagement_count_num"])
|
||||
self.assertEqual(invalid["publish_status"], "发布时间无数据")
|
||||
self.assertIn("口径异常", invalid["engagement_data_status"])
|
||||
|
||||
valid = daily._sanitize_cooperation_row({
|
||||
"publish_time": "2026-07-14",
|
||||
"exposure_count": 38000,
|
||||
"engagement_count_num": 2143,
|
||||
})
|
||||
self.assertEqual(valid["engagement_count_num"], 2143)
|
||||
self.assertEqual(valid["publish_status"], "已记录发布时间")
|
||||
|
||||
def test_load_report_facts_tracks_all_styles_for_report_date(self):
|
||||
metrics = {
|
||||
"宙斯": {"light": "🟢"},
|
||||
}
|
||||
with patch.object(daily, "_note_rows", return_value=[]), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯", "蓝鹊"]) as all_styles, \
|
||||
patch.object(daily, "_style_metrics", return_value=metrics) as style_metrics:
|
||||
with patch.object(daily, "_load_enrichment", return_value={"platform_metrics": []}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={"styles": ["宙斯", "蓝鹊"]}):
|
||||
_, tracking, _, _, enrichment = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
all_styles.assert_called_once_with(date(2026, 7, 15))
|
||||
self.assertEqual([row["style_name"] for row in tracking], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(tracking[1]["light"], "无判断")
|
||||
self.assertEqual(style_metrics.call_args_list[0].args[1], ["宙斯", "蓝鹊"])
|
||||
self.assertEqual(enrichment["platform_metrics"], [])
|
||||
self.assertEqual(enrichment["candidate_styles"], [])
|
||||
self.assertEqual(enrichment["dashboard"]["styles"], ["宙斯", "蓝鹊"])
|
||||
|
||||
def test_report_date_uses_yesterday_notes_for_daily_new_notes(self):
|
||||
notes = [
|
||||
{"id": 1, "publish_date": "2026-07-15", "style_name": "宙斯", "creator_name": "达人甲", "platform": "小红书", "publish_time": "2026-07-15 10:00", "title": "今天", "view_count": 10, "engagement_rate": None},
|
||||
{"id": 2, "publish_date": "2026-07-14", "style_name": "宙斯", "creator_name": "达人乙", "platform": "抖音", "publish_time": "2026-07-14 11:00", "title": "昨天", "view_count": 20, "engagement_rate": None},
|
||||
]
|
||||
with patch.object(daily, "_note_rows", return_value=notes), \
|
||||
patch.object(daily, "_all_style_names", return_value=["宙斯"]), \
|
||||
patch.object(daily, "_style_metrics", return_value={}), \
|
||||
patch.object(daily, "_load_enrichment", return_value={}), \
|
||||
patch.object(daily, "_platform_metrics", return_value=[]), \
|
||||
patch.object(daily, "_persona_context", return_value=[]), \
|
||||
patch.object(daily, "load_creator_connections", return_value={}), \
|
||||
patch.object(daily, "load_style_images", return_value={}), \
|
||||
patch.object(daily, "load_style_categories", return_value={}), \
|
||||
patch.object(daily, "_cooperation_context", return_value=[]), \
|
||||
patch.object(daily, "_style_comment_context", return_value=[]), \
|
||||
patch.object(daily, "_review_context", return_value=[]), \
|
||||
patch.object(daily, "_creative_context", return_value=[]), \
|
||||
patch.object(daily, "_comment_rows", return_value=[]), \
|
||||
patch.object(daily, "build_dashboard_facts", return_value={}) as dashboard:
|
||||
daily_notes, _, _, _, _ = daily.load_report_facts(date(2026, 7, 15))
|
||||
|
||||
self.assertEqual([row["note_id"] for row in daily_notes], [2])
|
||||
self.assertEqual([row["note_id"] for row in dashboard.call_args.kwargs["daily_notes"]], [2])
|
||||
|
||||
def test_prompt_requires_all_styles_without_top_ten_truncation(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [])
|
||||
self.assertIn("所有SKU", prompt)
|
||||
self.assertNotIn("只保留异常程度最高的10个", prompt)
|
||||
self.assertIn("SKU经营表现分析", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("加购率", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_uses_the_new_daily_output_contract(self):
|
||||
self.assertIn("电商运营数据", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("达人营销数据", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("TOP20", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("一页A4", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("↑↓展示趋势", daily.SYSTEM_PROMPT)
|
||||
self.assertNotIn("→稳定", daily.SYSTEM_PROMPT)
|
||||
self.assertIn("电商销量表现 + 访客数 + 转化率 + 达人笔记曝光 + 用户反馈", daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_prompt_payload_keeps_yesterday_and_seven_day_comparisons(self):
|
||||
rows = [{
|
||||
"style_name": "蓝鹊",
|
||||
"sales": 10,
|
||||
"sales_yesterday": 8,
|
||||
"avg_sales_7d": 7.5,
|
||||
"sales_change_yesterday": 0.25,
|
||||
"sales_change": 0.3333,
|
||||
}]
|
||||
prompt = daily.build_daily_prompt("2026-07-16", [], rows, [], [])
|
||||
self.assertIn('"sales_yesterday": 8', prompt)
|
||||
self.assertIn('"avg_sales_7d": 7.5', prompt)
|
||||
self.assertIn('"sales_change_yesterday": 0.25', prompt)
|
||||
|
||||
def test_style_metrics_calculates_yesterday_and_seven_day_changes(self):
|
||||
class FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def execute(self, query, params):
|
||||
self.query = query
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return [(
|
||||
"蓝鹊",
|
||||
120, 24, 10, 2,
|
||||
100, 20, 8, 1,
|
||||
90, 18, 7.5, 1.2,
|
||||
[4, 5, 6, 7, 8, 9, 10],
|
||||
)]
|
||||
|
||||
class FakeConnection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return FakeCursor()
|
||||
|
||||
with patch.object(daily, "get_conn", return_value=FakeConnection()):
|
||||
row = daily._style_metrics(date(2026, 7, 16), ["蓝鹊"])["蓝鹊"]
|
||||
|
||||
self.assertEqual(row["sales_yesterday"], 8)
|
||||
self.assertEqual(row["sales_change_yesterday"], 0.25)
|
||||
self.assertEqual(row["sales_change"], 0.3333)
|
||||
self.assertEqual(row["visitor_change_yesterday"], 0.2)
|
||||
self.assertEqual(row["conversion_rate"], round(10 / 120, 4))
|
||||
self.assertEqual(row["conversion_rate_yesterday"], round(8 / 100, 4))
|
||||
self.assertEqual(row["sales_7d_total"], 49)
|
||||
self.assertEqual(row["sales_forecast_7d"], 63)
|
||||
|
||||
def test_prompt_does_not_reuse_retired_dashboard_sections(self):
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], {"dashboard": {
|
||||
"rankings": {"sales": []},
|
||||
"platform_personas": [{"platform": "天猫"}],
|
||||
"daily_new_notes": [],
|
||||
"styles": [{"style_name": "蓝鹊", "total_note_exposure": 1000, "evidence_summary": "经营、画像、营销笔记"}],
|
||||
"definitions": {"conversion_rate": "销量/访客"},
|
||||
}})
|
||||
self.assertNotIn("按风格分组的综合经营矩阵", prompt)
|
||||
self.assertNotIn("平台高转化与高访客画像", prompt)
|
||||
self.assertNotIn("全款式经营看板摘要", prompt)
|
||||
|
||||
def test_report_normalization_and_all_style_validation(self):
|
||||
report = daily.normalize_report(
|
||||
"健康度：偏弱。36 个款式中\n蓝鹊同比增长79%\n宙斯环比下降16%",
|
||||
35,
|
||||
)
|
||||
self.assertNotIn("同比", report)
|
||||
self.assertIn("环比下降16%", report)
|
||||
self.assertIn("35个款式中", report)
|
||||
self.assertEqual(daily.normalize_report("销量 0 / 退款 0（无数据）"), "销量 0 / 退款 0（当日均为0）")
|
||||
cleaned = daily.normalize_report("投放尚未发布，带动效应快速衰减；待发布后核对")
|
||||
self.assertNotIn("未发布", cleaned)
|
||||
self.assertNotIn("带动效应", cleaned)
|
||||
self.assertIn("发布时间无数据", cleaned)
|
||||
self.assertNotIn("驱动访客", daily.normalize_report("该内容未驱动访客回升"))
|
||||
daily.validate_all_styles(report, [{"style_name": "蓝鹊"}, {"style_name": "宙斯"}])
|
||||
with self.assertRaisesRegex(RuntimeError, "缺少款式"):
|
||||
daily.validate_all_styles(report, [{"style_name": "白鹭"}])
|
||||
|
||||
def test_report_normalization_removes_database_and_code_style_language(self):
|
||||
raw = """数据已收到，我先用脚本做一遍聚合。
|
||||
# 品牌SKU电商运营日报
|
||||
|
||||
• 最佳渠道：无法判定。今日 `昨日新发布内容` 为空，cooperations 仅1条，publish_time=null、engagement_data_status=无数据。
|
||||
|
||||
5. 数据治理与内容补位：（a）对齐 publish_time；（b）修复互动数据；（c）补齐数据。
|
||||
|
||||
报告字数提示：如需压缩请告知。
|
||||
完整数据索引：如需导出 CSV 请告诉我。
|
||||
"""
|
||||
report = daily.normalize_report(raw)
|
||||
|
||||
self.assertTrue(report.startswith("# 品牌SKU电商运营日报"))
|
||||
for forbidden in ("我先用脚本", "cooperations", "publish_time", "engagement_data_status", "null", "`", "（a）", "报告字数提示", "请告诉我"):
|
||||
self.assertNotIn(forbidden, report)
|
||||
self.assertIn("达人合作记录", report)
|
||||
self.assertIn("发布时间无数据", report)
|
||||
self.assertIn("互动数据无数据", report)
|
||||
self.assertIn("（1）", report)
|
||||
|
||||
def test_system_prompt_forbids_database_fields_and_conversational_tail(self):
|
||||
for phrase in (
|
||||
"禁止输出数据库表名、字段名、JSON键名",
|
||||
"禁止出现反引号、null",
|
||||
"禁止描述生成过程",
|
||||
"禁止在结尾追问",
|
||||
):
|
||||
self.assertIn(phrase, daily.SYSTEM_PROMPT)
|
||||
|
||||
def test_status_groups_are_replaced_with_complete_deterministic_lists(self):
|
||||
report = "二、种草追踪池\n\nB. 全量状态（固定4行）\n🔴（1）：蓝鹊\n\n三、昨日笔记次日追踪"
|
||||
rows = [
|
||||
{"style_name": "蓝鹊", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
{"style_name": "云卷2", "light": "无判断"},
|
||||
]
|
||||
result = daily.replace_status_groups(report, rows)
|
||||
self.assertIn("B. 全量状态（4款）", result)
|
||||
self.assertIn("🔴（1）：蓝鹊", result)
|
||||
self.assertIn("🟡（1）：宙斯", result)
|
||||
self.assertIn("🟢（1）：白鹭", result)
|
||||
self.assertIn("无判断（1）：云卷2", result)
|
||||
self.assertIn("三、昨日笔记次日追踪", result)
|
||||
|
||||
def test_status_groups_are_inserted_when_model_omits_marker(self):
|
||||
report = "一、核心经营结论\n正常\n\n三、达人投放与电商同期关联\n无数据"
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}, {"style_name": "白鹭", "light": "🟢"}]
|
||||
result = daily.replace_status_groups(report, rows, "C. 补充经营信号\n- 店铺大盘：无数据")
|
||||
self.assertIn("B. 全量状态（2款）", result)
|
||||
self.assertIn("🔴（1）：蓝鹊", result)
|
||||
self.assertIn("🟢（1）：白鹭", result)
|
||||
self.assertLess(result.index("B. 全量状态"), result.index("三、达人投放"))
|
||||
|
||||
def test_prompt_passes_available_enrichment_without_restoring_old_sections(self):
|
||||
enrichment = {
|
||||
"platform_metrics": [{"style_name": "蓝鹊", "platform": "天猫", "sales": 14}],
|
||||
"cooperations": [{"style_name": "蓝鹊", "content_direction": "通勤"}],
|
||||
"personas": [{"style_name": "蓝鹊", "gender_top": "女性用户 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "ctr": 4.2}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12"}],
|
||||
}
|
||||
prompt = daily.build_daily_prompt("2026-07-15", [], [], [], [], enrichment)
|
||||
for source_key in ("platform_metrics", "cooperations", "personas", "creatives", "reviews", "shop_overview"):
|
||||
self.assertIn(source_key, prompt)
|
||||
self.assertIn("评论与补充经营数据", prompt)
|
||||
self.assertNotIn("补充经营信号", prompt)
|
||||
self.assertNotIn("全量状态", prompt)
|
||||
|
||||
def test_enrichment_section_validation_requires_available_categories(self):
|
||||
enrichment = {"personas": [{}], "creatives": [{}], "reviews": [{}], "shop_overview": [{}]}
|
||||
complete = "条件信号：人物画像；主图；商品评价；店铺大盘"
|
||||
daily.validate_enrichment_sections(complete, enrichment)
|
||||
with self.assertRaisesRegex(RuntimeError, "主图"):
|
||||
daily.validate_enrichment_sections("人物画像 商品评价 店铺大盘", enrichment)
|
||||
|
||||
def test_build_condition_signals_covers_all_available_sources(self):
|
||||
enrichment = {
|
||||
"candidate_styles": ["蓝鹊"],
|
||||
"personas": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-15", "gender_top": "女性 60%"}],
|
||||
"creatives": [{"style_name": "蓝鹊", "platform": "天猫", "data_date": "2026-07-12", "impressions": 1000, "clicks": 50, "ctr": 5.0}],
|
||||
"reviews": [{"style_name": "蓝鹊", "sample_size": 6, "negative_count": 1, "negative_rate": 0.1667, "window_start": "2026-07-09", "window_end": "2026-07-15"}],
|
||||
"shop_overview": [{"platform": "天猫", "week_end": "2026-07-12", "visitors": 10000, "deal_amount": 20000}],
|
||||
}
|
||||
text = daily.build_condition_signals(enrichment)
|
||||
for phrase in ("C. 补充经营信号", "人物画像", "主图", "商品评价", "店铺大盘", "2026-07-12"):
|
||||
self.assertIn(phrase, text)
|
||||
|
||||
def test_persona_top_value_ignores_invalid_percentages(self):
|
||||
payload = {"buying_power": [{"name": "异常", "value": 133}, {"name": "L4", "value": 42}]}
|
||||
self.assertEqual(daily._top_persona_value(payload, "buying_power"), "L4 42.0%")
|
||||
|
||||
def test_enriched_report_rejects_missing_platform_claim_and_invented_new_product(self):
|
||||
daily.validate_enriched_report("平台拆分完整，均为同期合作")
|
||||
with self.assertRaisesRegex(RuntimeError, "平台拆分"):
|
||||
daily.validate_enriched_report("无平台拆分数据")
|
||||
with self.assertRaisesRegex(RuntimeError, "新品"):
|
||||
daily.validate_enriched_report("同期出现多个新品合作")
|
||||
|
||||
def test_platform_persona_section_uses_database_winners_not_model_text(self):
|
||||
report = ("### 三、平台人群与营销机会\n天猫高转化款为错误款式\n\n"
|
||||
"### 四、明日业务动作\n复查数据\n\n"
|
||||
"三、平台人群与营销机会\n重复错误款式")
|
||||
enrichment = {"dashboard": {
|
||||
"daily_views_status": "无数据：未保存每日增量快照",
|
||||
"platform_personas": [{
|
||||
"platform": "天猫",
|
||||
"top_conversion": {
|
||||
"style_name": "布谷", "conversion_rate": 0.0259, "visitors": 617,
|
||||
"persona_text": "无数据", "interest_hypothesis": "兴趣推测：无数据",
|
||||
"optimization": "先补采人群画像",
|
||||
},
|
||||
"top_visitors": {
|
||||
"style_name": "星云2", "conversion_rate": 0.0114, "visitors": 3504,
|
||||
"persona_text": "男性用户 62.7%", "interest_hypothesis": "兴趣推测：数码3C",
|
||||
"optimization": "优化详情页",
|
||||
},
|
||||
}],
|
||||
}}
|
||||
result = daily.replace_platform_persona_section(report, enrichment)
|
||||
self.assertIn("转化最高为布谷（转化率2.6%", result)
|
||||
self.assertIn("访客最高为星云2（访客3,504", result)
|
||||
self.assertIn("兴趣推测：数码3C", result)
|
||||
self.assertNotIn("错误款式", result)
|
||||
self.assertNotIn("重复错误款式", result)
|
||||
self.assertIn("四、明日业务动作", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
def test_platform_persona_section_supports_new_daily_report_boundary(self):
|
||||
report = ("### 三、平台人群与营销机会\n模型内容\n\n"
|
||||
"### 四、今日重点SKU分析\n重点款内容\n")
|
||||
result = daily.replace_platform_persona_section(report, {"dashboard": {}})
|
||||
self.assertIn("三、平台人群与营销机会", result)
|
||||
self.assertIn("四、今日重点SKU分析", result)
|
||||
self.assertNotIn("模型内容", result)
|
||||
self.assertEqual(result.count("三、平台人群与营销机会"), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,26 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DailyMarketingReportScheduleTests(unittest.TestCase):
|
||||
def test_python_scheduler_runs_latest_daily_report(self):
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
workflow = next(
|
||||
item
|
||||
for item in catalog.workflows
|
||||
if item.workflow_id == "content.marketing_report.daily"
|
||||
)
|
||||
schedule = catalog.schedule_for(workflow.workflow_id)
|
||||
|
||||
self.assertEqual(workflow.steps[0].entry, "daily_marketing_report.py")
|
||||
self.assertEqual(workflow.steps[0].args, ("--send",))
|
||||
self.assertEqual(schedule.kind, "daily")
|
||||
self.assertEqual(schedule.at, "10:00")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,202 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_report_card import (
|
||||
build_analysis_card,
|
||||
build_daily_report_card,
|
||||
build_dashboard_card,
|
||||
prepare_feishu_image,
|
||||
send_card_to_recipients,
|
||||
send_daily_report_cards,
|
||||
upload_dashboard_image,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportCardTests(unittest.TestCase):
|
||||
def test_dashboard_and_analysis_are_separate_cards_with_full_text(self):
|
||||
report = """
|
||||
|
||||
第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断：下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
1. 蓝鹊：销量第一。
|
||||
2. 星云2：转化领先。
|
||||
3. 宙斯：访客增长。
|
||||
4. 南斯：退款偏高。
|
||||
5. 凌云3：需要优化。
|
||||
6. 波塞冬：继续观察。
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升，48小时内排查。
|
||||
"""
|
||||
rows = [{"style_name": "蓝鹊", "light": "🔴"}]
|
||||
|
||||
dashboard = build_dashboard_card("2026-07-17", rows, "img_v3_test")
|
||||
analysis = build_analysis_card("2026-07-17", report)
|
||||
|
||||
dashboard_text = json.dumps(dashboard, ensure_ascii=False)
|
||||
analysis_text = json.dumps(analysis, ensure_ascii=False)
|
||||
self.assertIn("img_v3_test", dashboard_text)
|
||||
self.assertNotIn("总销量下降15%", dashboard_text)
|
||||
self.assertNotIn("img_v3_test", analysis_text)
|
||||
self.assertNotIn("![全域种草数据看板]", analysis_text)
|
||||
self.assertIn("波塞冬：继续观察", analysis_text)
|
||||
self.assertIn("退款订单比上升", analysis_text)
|
||||
|
||||
def test_build_card_uses_card_2_schema_and_dashboard(self):
|
||||
report = """一、核心经营结论
|
||||
1. 波塞冬edge访客明显下滑。
|
||||
2. 蓝鹊销量同期上行。
|
||||
|
||||
二、产品经营分析
|
||||
略
|
||||
|
||||
四、明日业务动作
|
||||
1. 复查波塞冬edge天猫流量。
|
||||
2. 跟进蓝鹊投放承接。
|
||||
"""
|
||||
rows = [
|
||||
{"style_name": "波塞冬edge", "light": "🔴"},
|
||||
{"style_name": "宙斯", "light": "🟡"},
|
||||
{"style_name": "白鹭", "light": "🟢"},
|
||||
]
|
||||
card = build_daily_report_card("2026-07-15", report, rows, "img_v3_test")
|
||||
|
||||
self.assertEqual(card["schema"], "2.0")
|
||||
self.assertEqual(card["config"]["width_mode"], "fill")
|
||||
self.assertIn("Hermes", card["header"]["subtitle"]["content"])
|
||||
elements = card["body"]["elements"]
|
||||
self.assertEqual([item["tag"] for item in elements], ["column_set", "img", "markdown", "markdown"])
|
||||
self.assertEqual(elements[1]["img_key"], "img_v3_test")
|
||||
metrics_text = json.dumps(elements[0], ensure_ascii=False)
|
||||
self.assertIn("红灯款式", metrics_text)
|
||||
self.assertIn("1", metrics_text)
|
||||
self.assertNotIn("二、产品经营分析", json.dumps(card, ensure_ascii=False))
|
||||
|
||||
def test_prepare_feishu_image_resizes_to_supported_width(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source = Path(tmp) / "source.png"
|
||||
target = Path(tmp) / "target.png"
|
||||
Image.new("RGB", (1600, 1000), "white").save(source)
|
||||
prepare_feishu_image(source, target)
|
||||
with Image.open(target) as image:
|
||||
self.assertEqual(image.width, 1500)
|
||||
self.assertLessEqual(image.height / image.width, 16 / 9)
|
||||
|
||||
def test_build_card_reads_new_management_report_headings(self):
|
||||
report = """## 1. 今日经营概览
|
||||
销售：↑12%，蓝鹊贡献主要增量。
|
||||
整体判断：增长状态。
|
||||
|
||||
## 2. SKU健康排名 TOP20
|
||||
略
|
||||
|
||||
## 6. 今日运营建议
|
||||
1. 商品动作：复核蓝鹊库存。
|
||||
2. 电商动作：优化宙斯详情页。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("蓝鹊贡献主要增量", card_text)
|
||||
self.assertIn("复核蓝鹊库存", card_text)
|
||||
|
||||
def test_build_card_reads_fresh_partial_prompt_report_headings(self):
|
||||
report = """第一部分 整体经营分析
|
||||
总销量下降15%。
|
||||
整体趋势判断：下滑状态。
|
||||
|
||||
第二部分 SKU分析排序
|
||||
略
|
||||
|
||||
第三部分 关键预警与建议
|
||||
退款订单比上升，48小时内排查。
|
||||
追加增长型SKU预算。
|
||||
"""
|
||||
card = build_daily_report_card("2026-07-17", report, [], "img_v3_test")
|
||||
card_text = json.dumps(card, ensure_ascii=False)
|
||||
self.assertIn("总销量下降15%", card_text)
|
||||
self.assertIn("48小时内排查", card_text)
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
)
|
||||
def test_upload_dashboard_uses_named_image_file_field(self, run):
|
||||
run.return_value = type("Result", (), {
|
||||
"returncode": 0, "stdout": json.dumps({"image_key": "img_v3_test"}), "stderr": ""
|
||||
})()
|
||||
key = upload_dashboard_image(Path("charts/dashboard.png"))
|
||||
self.assertEqual(key, "img_v3_test")
|
||||
command = run.call_args.args[0]
|
||||
self.assertEqual(command[command.index("--file") + 1], "image=./dashboard.png")
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools."
|
||||
"daily_report_card.subprocess.run"
|
||||
)
|
||||
def test_send_card_to_all_recipients(self, run):
|
||||
run.side_effect = [
|
||||
type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})()
|
||||
for i in range(3)
|
||||
]
|
||||
recipients = ["ou_one", "ou_two", "ou_three"]
|
||||
results = send_card_to_recipients({"schema": "2.0"}, recipients, "2026-07-15")
|
||||
|
||||
self.assertEqual([item["message_id"] for item in results], ["om_0", "om_1", "om_2"])
|
||||
self.assertEqual(run.call_count, 3)
|
||||
commands = [call.args[0] for call in run.call_args_list]
|
||||
self.assertEqual([cmd[cmd.index("--user-id") + 1] for cmd in commands], recipients)
|
||||
self.assertTrue(all("--msg-type" in cmd and "interactive" in cmd for cmd in commands))
|
||||
keys = [cmd[cmd.index("--idempotency-key") + 1] for cmd in commands]
|
||||
self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys))
|
||||
self.assertTrue(all(len(key) <= 50 for key in keys))
|
||||
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients"
|
||||
)
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.upload_dashboard_image",
|
||||
return_value="img_v3_test",
|
||||
)
|
||||
@patch(
|
||||
"gyxx_flow.modules.content_marketing.data.tools.daily_report_card.prepare_feishu_image"
|
||||
)
|
||||
def test_daily_report_sends_dashboard_then_analysis_as_two_messages_per_recipient(
|
||||
self, prepare, upload, send
|
||||
):
|
||||
send.side_effect = [
|
||||
[{"message_id": "om_board_1"}, {"message_id": "om_board_2"}],
|
||||
[{"message_id": "om_text_1"}, {"message_id": "om_text_2"}],
|
||||
]
|
||||
recipients = ["ou_one", "ou_two"]
|
||||
|
||||
results = send_daily_report_cards(
|
||||
"2026-07-17",
|
||||
"第一部分 整体经营分析\n完整分析内容",
|
||||
[{"light": "🟢"}],
|
||||
Path("charts/dashboard.png"),
|
||||
recipients,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[item["message_id"] for item in results],
|
||||
["om_board_1", "om_board_2", "om_text_1", "om_text_2"],
|
||||
)
|
||||
self.assertEqual(send.call_count, 2)
|
||||
first_card = send.call_args_list[0].args[0]
|
||||
second_card = send.call_args_list[1].args[0]
|
||||
self.assertIn("img_v3_test", json.dumps(first_card, ensure_ascii=False))
|
||||
self.assertIn("完整分析内容", json.dumps(second_card, ensure_ascii=False))
|
||||
self.assertEqual(send.call_args_list[0].kwargs["message_kind"], "dashboard")
|
||||
self.assertEqual(send.call_args_list[1].kwargs["message_kind"], "analysis")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,104 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.daily_report_charts import (
|
||||
_split_style_groups,
|
||||
build_overview_cards,
|
||||
classify_style_action,
|
||||
generate_dashboard,
|
||||
select_chart_styles,
|
||||
)
|
||||
|
||||
|
||||
class DailyReportChartTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.rows = [
|
||||
{"style_name": "大盘下滑", "light": "🔴", "visitors": 1000, "avg_visitors_7d": 2000, "cart_users": 80, "avg_cart_users_7d": 100, "visitor_change": -0.5, "cart_change": -0.2, "sales_change": -0.4, "refund_change": 0.1},
|
||||
{"style_name": "正向放量", "light": "🔴", "visitors": 1500, "avg_visitors_7d": 1000, "cart_users": 180, "avg_cart_users_7d": 100, "visitor_change": 0.5, "cart_change": 0.8, "sales_change": 0.3, "refund_change": -0.1},
|
||||
{"style_name": "小基数", "light": "🔴", "visitors": 20, "avg_visitors_7d": 100, "cart_users": 1, "avg_cart_users_7d": 2, "visitor_change": -0.8, "cart_change": -0.5, "sales_change": None, "refund_change": None},
|
||||
{"style_name": "黄灯A", "light": "🟡", "visitors": 800, "avg_visitors_7d": 1000, "cart_users": 70, "avg_cart_users_7d": 80, "visitor_change": -0.2, "cart_change": -0.125, "sales_change": -0.1, "refund_change": 0.2},
|
||||
{"style_name": "绿灯", "light": "🟢", "visitors": 500, "avg_visitors_7d": 510, "cart_users": 40, "avg_cart_users_7d": 41, "visitor_change": -0.02, "cart_change": -0.02, "sales_change": 0.0, "refund_change": 0.0},
|
||||
]
|
||||
|
||||
def test_selection_covers_negative_positive_and_small_base(self):
|
||||
names = [row["style_name"] for row in select_chart_styles(self.rows, limit=4)]
|
||||
self.assertIn("大盘下滑", names)
|
||||
self.assertIn("正向放量", names)
|
||||
self.assertIn("小基数", names)
|
||||
|
||||
def test_generate_dashboard_writes_png(self):
|
||||
enrichment = {"platform_metrics": [
|
||||
{"style_name": "大盘下滑", "platform": "天猫", "visitors": 700},
|
||||
{"style_name": "大盘下滑", "platform": "京东", "visitors": 300},
|
||||
]}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "dashboard.png"
|
||||
generate_dashboard("2026-07-15", self.rows, enrichment, path)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertGreater(path.stat().st_size, 10_000)
|
||||
with Image.open(path) as image:
|
||||
self.assertEqual(image.size, (1600, 2500))
|
||||
|
||||
def test_style_groups_are_split_into_balanced_category_columns(self):
|
||||
groups = [
|
||||
{"style_category": "都市机能", "styles": [{}] * 12},
|
||||
{"style_category": "户外机能", "styles": [{}] * 3},
|
||||
{"style_category": "都市新贵", "styles": [{}] * 3},
|
||||
{"style_category": "智性通勤", "styles": [{}] * 14},
|
||||
{"style_category": "都市新旅", "styles": [{}]},
|
||||
{"style_category": "都市运动", "styles": [{}] * 3},
|
||||
]
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual([group["style_category"] for group in left], ["都市机能", "户外机能", "都市新贵"])
|
||||
self.assertEqual([group["style_category"] for group in right], ["智性通勤", "都市新旅", "都市运动"])
|
||||
|
||||
groups.append({"style_category": "未分类", "styles": [{}] * 2})
|
||||
left, right = _split_style_groups(groups)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in left), 18)
|
||||
self.assertEqual(sum(len(group["styles"]) for group in right), 20)
|
||||
|
||||
def test_overview_compares_today_yesterday_and_seven_day_average(self):
|
||||
rows = [
|
||||
{
|
||||
"style_name": "蓝鹊", "sales": 12, "sales_yesterday": 10, "avg_sales_7d": 8,
|
||||
"visitors": 120, "visitors_yesterday": 100, "avg_visitors_7d": 90,
|
||||
},
|
||||
{
|
||||
"style_name": "宙斯", "sales": 18, "sales_yesterday": 20, "avg_sales_7d": 15,
|
||||
"visitors": 180, "visitors_yesterday": 200, "avg_visitors_7d": 150,
|
||||
},
|
||||
]
|
||||
cards = build_overview_cards(rows, {"daily_new_notes": [{"view_count": 515}]})
|
||||
sales = next(card for card in cards if card["key"] == "sales")
|
||||
conversion = next(card for card in cards if card["key"] == "conversion")
|
||||
content = next(card for card in cards if card["key"] == "content")
|
||||
|
||||
self.assertEqual((sales["today"], sales["yesterday"], sales["baseline_7d"]), (30, 30, 23))
|
||||
self.assertEqual(conversion["today"], 0.1)
|
||||
self.assertEqual(content["today"], 515)
|
||||
self.assertIsNone(content["baseline_7d"])
|
||||
|
||||
def test_style_action_matches_prompt_growth_opportunity_and_risk_rules(self):
|
||||
growth = classify_style_action({
|
||||
"sales_change_yesterday": 0.2, "visitor_change_yesterday": 0.1,
|
||||
"conversion_change_yesterday": 0.001, "refund_change_yesterday": -0.1,
|
||||
})
|
||||
opportunity = classify_style_action({
|
||||
"sales_change_yesterday": -0.1, "visitor_change_yesterday": 0.2,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0,
|
||||
})
|
||||
risk = classify_style_action({
|
||||
"sales_change_yesterday": -0.2, "visitor_change_yesterday": -0.1,
|
||||
"conversion_change_yesterday": -0.01, "refund_change_yesterday": 0.4,
|
||||
})
|
||||
|
||||
self.assertEqual(growth, ("增长型", "加达人/备货"))
|
||||
self.assertEqual(opportunity, ("流量机会", "优化详情页"))
|
||||
self.assertEqual(risk, ("风险型", "查退款/控投放"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,137 @@
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
friday_relogin_parallel as relogin,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction
|
||||
|
||||
|
||||
def test_five_expected_platforms_are_configured():
|
||||
assert set(relogin.PLATFORMS) == {
|
||||
"bilibili", "douyin", "pgy", "xingtu", "xiaohongshu"
|
||||
}
|
||||
assert relogin.PLATFORMS["xingtu"].required_cookie == "sessionid"
|
||||
assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid"
|
||||
|
||||
|
||||
def test_manual_relogin_can_select_only_xingtu():
|
||||
assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"]
|
||||
assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS)
|
||||
|
||||
|
||||
def test_default_screenshot_recipients_are_routed_by_platform():
|
||||
assert relogin.DEFAULT_RECIPIENTS == {
|
||||
"pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"xingtu": "ou_24cc944d6e43c69c59d6560ad4e2ae6e",
|
||||
"douyin": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"bilibili": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
"xiaohongshu": "ou_7ad5fc8012e2f741afc5346e05ffd447",
|
||||
}
|
||||
assert relogin.recipient_for_platform("pgy") == "ou_24cc944d6e43c69c59d6560ad4e2ae6e"
|
||||
assert relogin.recipient_for_platform("douyin") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("bilibili") == "ou_7ad5fc8012e2f741afc5346e05ffd447"
|
||||
assert relogin.recipient_for_platform("pgy", "ou_override") == "ou_override"
|
||||
|
||||
|
||||
def test_failed_platforms_are_grouped_by_their_responsible_recipient():
|
||||
grouped = relogin.group_platforms_by_recipient(
|
||||
["pgy", "douyin", "bilibili", "xingtu", "xiaohongshu"]
|
||||
)
|
||||
assert grouped == {
|
||||
"ou_24cc944d6e43c69c59d6560ad4e2ae6e": ["pgy", "xingtu"],
|
||||
"ou_7ad5fc8012e2f741afc5346e05ffd447": [
|
||||
"douyin", "bilibili", "xiaohongshu"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_login_windows_are_classified_by_title():
|
||||
assert relogin._platform_from_window_title("账号登录 - Google Chrome") == "bilibili"
|
||||
assert relogin._platform_from_window_title("小红书蒲公英 - Google Chrome") == "pgy"
|
||||
assert relogin._platform_from_window_title("抖音授权 - Google Chrome") == "xingtu"
|
||||
assert relogin._platform_from_window_title("抖音 - 记录美好生活") == "douyin"
|
||||
assert relogin._platform_from_window_title("小红书 - 你的生活兴趣社区") == "xiaohongshu"
|
||||
assert relogin._platform_from_window_title("Codex") is None
|
||||
|
||||
|
||||
def test_retry_only_failed_platforms_for_at_most_three_rounds():
|
||||
calls = []
|
||||
|
||||
def fake_round(names, attempt):
|
||||
calls.append((tuple(names), attempt))
|
||||
if attempt == 1:
|
||||
return {name: name != "xingtu" for name in names}
|
||||
if attempt == 2:
|
||||
return {name: False for name in names}
|
||||
return {name: True for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["bilibili", "pgy", "xingtu", "xiaohongshu"],
|
||||
max_attempts=3,
|
||||
run_round=fake_round,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(("bilibili", "pgy", "xingtu", "xiaohongshu"), 1),
|
||||
(("xingtu",), 2),
|
||||
(("xingtu",), 3),
|
||||
]
|
||||
assert all(result.values())
|
||||
|
||||
|
||||
def test_stops_after_third_failed_round():
|
||||
calls = []
|
||||
|
||||
def always_fail(names, attempt):
|
||||
calls.append(attempt)
|
||||
return {name: False for name in names}
|
||||
|
||||
result = relogin.run_with_retries(
|
||||
["pgy"], max_attempts=3, run_round=always_fail
|
||||
)
|
||||
|
||||
assert calls == [1, 2, 3]
|
||||
assert result == {"pgy": False}
|
||||
|
||||
|
||||
def test_timeout_terminates_the_complete_process_tree(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Process:
|
||||
pid = 1234
|
||||
|
||||
def terminate(self):
|
||||
calls.append("fallback-terminate")
|
||||
|
||||
def wait(self, timeout):
|
||||
calls.append(("wait", timeout))
|
||||
|
||||
monkeypatch.setattr(
|
||||
relogin.subprocess,
|
||||
"run",
|
||||
lambda command, **kwargs: calls.append((command, kwargs)),
|
||||
)
|
||||
|
||||
relogin._terminate_process_tree(Process())
|
||||
|
||||
assert calls[0][0] == ["taskkill", "/PID", "1234", "/T", "/F"]
|
||||
assert calls[1] == ("wait", 10)
|
||||
|
||||
|
||||
def test_transaction_rolls_back_files_and_profile(tmp_path):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
storage = tmp_path / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
state = relogin_transaction.begin((cookie, storage), profile)
|
||||
cookie.write_text("partial", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
relogin_transaction.rollback(state)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
@@ -0,0 +1,367 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy
|
||||
from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu
|
||||
from gyxx_flow.modules.content_marketing.creator_task_grouping import (
|
||||
chunk_creator_groups,
|
||||
group_tasks_by_creator,
|
||||
)
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _workflow(workflow_id):
|
||||
catalog = WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
return next(item for item in catalog.workflows if item.workflow_id == workflow_id)
|
||||
|
||||
|
||||
def test_group_tasks_uses_creator_id_and_merges_missing_id_by_unique_name():
|
||||
tasks = [
|
||||
{"creator_name": "同一达人", "creator_id": " 123 ", "record_id": "a"},
|
||||
{"creator_name": "同一达人 ", "creator_id": None, "record_id": "b"},
|
||||
{"creator_name": "另一个达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 2
|
||||
assert groups[0]["creator_id"] == "123"
|
||||
assert [task["record_id"] for task in groups[0]["tasks"]] == ["a", "b"]
|
||||
|
||||
|
||||
def test_group_tasks_does_not_guess_when_same_name_has_conflicting_ids():
|
||||
tasks = [
|
||||
{"creator_name": "重名达人", "creator_id": "1", "record_id": "a"},
|
||||
{"creator_name": "重名达人", "creator_id": "2", "record_id": "b"},
|
||||
{"creator_name": "重名达人", "creator_id": None, "record_id": "c"},
|
||||
]
|
||||
|
||||
groups = group_tasks_by_creator(tasks)
|
||||
|
||||
assert len(groups) == 3
|
||||
|
||||
|
||||
def test_zero_batch_size_means_one_unlimited_batch():
|
||||
groups = [{"creator_name": str(i), "tasks": []} for i in range(3)]
|
||||
assert chunk_creator_groups(groups, 0) == [groups]
|
||||
|
||||
|
||||
def _assert_cross_style_collection(module, monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {"a": 1}},
|
||||
{"index": 2, "name": "款式B", "base_token": "base-b", "table_id": "table-b", "field_map": {"b": 2}},
|
||||
]
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
return [{
|
||||
"record_id": f"record-{style['index']}",
|
||||
"creator_name": "同一达人",
|
||||
"creator_id": "creator-1",
|
||||
"target_title": f"标题{style['index']}",
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(module, "extract_target_tasks", fake_extract)
|
||||
tasks, summaries = module.collect_tasks_across_styles(styles, None)
|
||||
|
||||
assert len(tasks) == 2
|
||||
assert len(group_tasks_by_creator(tasks)) == 1
|
||||
assert tasks[0]["style_context"]["table_id"] == "table-a"
|
||||
assert tasks[1]["style_context"]["table_id"] == "table-b"
|
||||
assert summaries[1]["total"] == 1
|
||||
assert summaries[2]["total"] == 1
|
||||
|
||||
|
||||
def test_pgy_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(pgy, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_collects_tasks_across_styles(monkeypatch):
|
||||
_assert_cross_style_collection(xingtu, monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_record_scope_is_applied_per_style(monkeypatch):
|
||||
styles = [
|
||||
{"index": 1, "name": "款式A", "base_token": "a", "table_id": "a", "field_map": {}},
|
||||
{"index": 2, "name": "款式B", "base_token": "b", "table_id": "b", "field_map": {}},
|
||||
]
|
||||
observed = {}
|
||||
|
||||
def fake_extract(style, only_record_ids):
|
||||
observed[style["index"]] = only_record_ids
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(xingtu, "extract_target_tasks", fake_extract)
|
||||
|
||||
xingtu.collect_tasks_across_styles(styles, {1: {"same-id"}, 2: {"other-id"}})
|
||||
|
||||
assert observed == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def test_xingtu_load_retry_scope_keeps_style_and_record_pair(tmp_path):
|
||||
summary_path = tmp_path / "summary.json"
|
||||
summary_path.write_text(json.dumps([
|
||||
{"index": 1, "results": [
|
||||
{"record_id": "same-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
{"index": 2, "results": [
|
||||
{"record_id": "same-id", "reason": "title_unmatched"},
|
||||
{"record_id": "other-id", "reason": "creator_search_circuit_open"},
|
||||
]},
|
||||
]), encoding="utf-8")
|
||||
|
||||
scope = xingtu.load_retry_scope(summary_path, "creator_search_circuit_open")
|
||||
|
||||
assert scope == {1: {"same-id"}, 2: {"other-id"}}
|
||||
|
||||
|
||||
def _assert_matched_task_uses_own_writeback_context(module, metric_key, monkeypatch):
|
||||
calls = []
|
||||
task = {
|
||||
"record_id": "record-b",
|
||||
"publish_time": object(),
|
||||
"style_context": {
|
||||
"index": 2,
|
||||
"base_token": "base-b",
|
||||
"table_id": "table-b",
|
||||
"field_map": {"slot": "style-b-slot"},
|
||||
},
|
||||
}
|
||||
summary = {"matched": 0, "filled": 0, "skipped_no_pubtime": 0, "results": []}
|
||||
monkeypatch.setattr(module, "pick_read_field", lambda fmap, publish_time: ("read_count_7d", "field-b", "7天曝光量"))
|
||||
monkeypatch.setattr(module, "write_back", lambda *args: calls.append(args) or True)
|
||||
|
||||
module.apply_matched_task(task, {metric_key: "1.2万", "title": "命中标题"}, summary, dry_run=False)
|
||||
|
||||
assert calls == [("base-b", "table-b", "record-b", "field-b", 12000)]
|
||||
assert summary["matched"] == 1
|
||||
assert summary["filled"] == 1
|
||||
|
||||
|
||||
def test_pgy_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(pgy, "read_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_match_writes_to_task_style_context(monkeypatch):
|
||||
_assert_matched_task_uses_own_writeback_context(xingtu, "play_count", monkeypatch)
|
||||
|
||||
|
||||
def test_xingtu_search_and_open_creator_submits_only_one_search(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fail_search(_page, creator_name, creator_id):
|
||||
calls.append((creator_name, creator_id))
|
||||
raise TimeoutError("no result")
|
||||
|
||||
monkeypatch.setattr(xingtu, "search_creator", fail_search)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"open_creator_detail",
|
||||
lambda *_args, **_kwargs: pytest.fail("detail must not open after search failure"),
|
||||
)
|
||||
|
||||
with pytest.raises(TimeoutError, match="no result"):
|
||||
xingtu.search_and_open_creator_once(object(), "同一达人", "creator-1")
|
||||
|
||||
assert calls == [("同一达人", "creator-1")]
|
||||
|
||||
|
||||
def test_xingtu_search_budget_blocks_duplicate_without_failure_circuit():
|
||||
budget = xingtu.CreatorSearchBudget(max_consecutive_failures=3)
|
||||
|
||||
assert budget.begin(("id", "1")) is True
|
||||
assert budget.begin(("id", "1")) is False
|
||||
budget.failed("first")
|
||||
assert budget.begin(("id", "2")) is True
|
||||
budget.failed("second")
|
||||
assert budget.begin(("id", "3")) is True
|
||||
assert budget.failed("third") is False
|
||||
assert budget.begin(("id", "4")) is True
|
||||
assert budget.calls == 4
|
||||
assert budget.circuit_reason is None
|
||||
|
||||
budget.exhaust("search_quota_exhausted")
|
||||
assert budget.begin(("id", "5")) is False
|
||||
|
||||
|
||||
def test_xingtu_wait_for_results_detects_search_quota_exhausted():
|
||||
class Page:
|
||||
def evaluate(self, *_args):
|
||||
return "quota"
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pytest.fail("quota detection must return immediately")
|
||||
|
||||
with pytest.raises(xingtu.SearchQuotaExhausted):
|
||||
xingtu.wait_for_visible_creator_result(Page(), "达人", timeout_ms=1000)
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_is_not_treated_as_global_block():
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return "no_creation_tab"
|
||||
|
||||
assert xingtu.is_blocked(Page()) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reason", ["verify_text", "verify_iframe", "verify_div"])
|
||||
def test_xingtu_explicit_verification_still_opens_global_block(reason):
|
||||
class Page:
|
||||
def evaluate(self, _script):
|
||||
return reason
|
||||
|
||||
assert xingtu.is_blocked(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_missing_creation_tab_falls_back_to_current_creator_page():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def click(self, **_kwargs):
|
||||
raise TimeoutError("missing tab")
|
||||
|
||||
class Page:
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def evaluate(self, _script):
|
||||
return False
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
assert xingtu.open_creation_ability(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_current_creator_page_cards_are_used_as_fallback(monkeypatch):
|
||||
class Page:
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(xingtu, "is_blocked", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
xingtu,
|
||||
"parse_videos_stable",
|
||||
lambda _page: [{
|
||||
"title": "MacBook党必看！提升幸福感的开工好物",
|
||||
"play_count": "1.2万",
|
||||
"note_id": "",
|
||||
}],
|
||||
)
|
||||
monkeypatch.setattr(xingtu, "go_next_page", lambda _page: False)
|
||||
monkeypatch.setattr(xingtu, "submit_video_search", lambda _page, _title: False)
|
||||
|
||||
found, _candidates, page_limit_hit = xingtu.find_videos_for_tasks(
|
||||
Page(),
|
||||
[{
|
||||
"record_id": "record-1",
|
||||
"target_title": "MacBook党必看！提升幸福感的开工好物",
|
||||
}],
|
||||
)
|
||||
|
||||
assert found["record-1"]["play_count"] == "1.2万"
|
||||
assert found["record-1"]["match_method"] == "title"
|
||||
assert page_limit_hit is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
"BrowserContext.new_page: Protocol error (Target.createTarget): Failed to open a new tab",
|
||||
"Protocol error (Target.createTarget)",
|
||||
],
|
||||
)
|
||||
def test_xingtu_new_tab_creation_failure_rebuilds_browser_session(message):
|
||||
assert xingtu.is_browser_session_lost(RuntimeError(message)) is True
|
||||
|
||||
|
||||
def test_scheduled_collection_has_no_automatic_retry_and_syncs_once():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
|
||||
assert "data/tools/retry_failed.py" not in entries
|
||||
assert entries.count("data/tools/sync_metrics_to_cmt_notes.py") == 1
|
||||
assert entries.index("run_all.py") < entries.index(
|
||||
"data/tools/sync_metrics_to_cmt_notes.py"
|
||||
)
|
||||
|
||||
|
||||
def test_daily_collection_uses_dedicated_self_operated_pipeline():
|
||||
steps = _workflow("content.metrics.daily").steps
|
||||
entries = [step.entry for step in steps]
|
||||
run_step = next(step for step in steps if step.entry == "run_all.py")
|
||||
|
||||
assert "--daily-scope" in run_step.args
|
||||
assert "--include-self-operated" not in run_step.args
|
||||
assert "data/tools/refresh_self_mapping.py" in entries
|
||||
assert "self_bilibili_scraper.py" in entries
|
||||
assert "chanmama_scraper.py" in entries
|
||||
refresh_index = entries.index("data/tools/refresh_self_mapping.py")
|
||||
bili_index = entries.index("self_bilibili_scraper.py")
|
||||
douyin_index = entries.index("chanmama_scraper.py")
|
||||
assert refresh_index < bili_index < douyin_index
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "entry_url", "error_type", "operation"),
|
||||
[
|
||||
(pgy, pgy.HOME_URL, pgy.AcceptanceCookieSkip, "content.pugongying.login"),
|
||||
(
|
||||
xingtu,
|
||||
xingtu.MARKET_URL,
|
||||
xingtu.AcceptanceCookieSkip,
|
||||
"content.xingtu.login",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_acceptance_invalid_session_skips_before_qr_login(
|
||||
module,
|
||||
entry_url,
|
||||
error_type,
|
||||
operation,
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
):
|
||||
evidence_file = tmp_path / "evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_FEISHU_TABLE_WRITE_DISABLED", "1")
|
||||
monkeypatch.setenv("GYXX_COOKIE_INVALID_SKIP", "1")
|
||||
monkeypatch.setenv(
|
||||
"GYXX_NOTIFICATION_RECIPIENT_OPEN_ID",
|
||||
"ou_8ee224968aa26a74c7d30ba27fed5eeb",
|
||||
)
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence_file))
|
||||
monkeypatch.setattr(module, "is_logged_in", lambda _page: False)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"wait_for_scan_login",
|
||||
lambda *_args, **_kwargs: pytest.fail("QR login must remain disabled"),
|
||||
)
|
||||
|
||||
class Page:
|
||||
def __init__(self):
|
||||
self.urls = []
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
self.urls.append(url)
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
page = Page()
|
||||
with pytest.raises(error_type):
|
||||
module.ensure_login(page, login_timeout=30)
|
||||
|
||||
assert page.urls == [entry_url]
|
||||
records = [
|
||||
json.loads(line)
|
||||
for line in evidence_file.read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
assert records[-1]["event"] == "cookie_skipped"
|
||||
assert records[-1]["operation"] == operation
|
||||
assert records[-1]["details"]["status"] == "SKIPPED_COOKIE"
|
||||
@@ -0,0 +1,367 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.content_marketing import (
|
||||
pgy_xhs_scraper_v2,
|
||||
xingtu_scraper_v2,
|
||||
)
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
relogin_douyin,
|
||||
relogin_pgy,
|
||||
relogin_xingtu,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_failed_relogin_restores_cookie_and_profile(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "state.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
assert not cookie.exists()
|
||||
assert not profile.exists()
|
||||
|
||||
cookie.write_text("partial-new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
module.restore_previous_state(cookie_backup, profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "state.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu])
|
||||
def test_successful_relogin_keeps_new_cookie(tmp_path, monkeypatch, module):
|
||||
cookie = tmp_path / "cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
monkeypatch.setattr(module, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(module, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = module.reset_state()
|
||||
cookie.write_text("new-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
|
||||
module.finish_successful_relogin(profile_backup)
|
||||
|
||||
assert cookie.read_text(encoding="utf-8") == "new-cookie"
|
||||
assert cookie_backup.exists()
|
||||
assert not profile_backup.exists()
|
||||
|
||||
|
||||
def test_xingtu_scan_login_opens_customer_sso_and_clicks_douyin():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
|
||||
|
||||
def __init__(self):
|
||||
self.goto_url = None
|
||||
self.selectors = []
|
||||
self.douyin = FakeLocator()
|
||||
|
||||
def goto(self, url, **kwargs):
|
||||
self.goto_url = url
|
||||
self.url = url
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.douyin
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
page = FakePage()
|
||||
xingtu_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert page.goto_url == xingtu_scraper_v2.CUSTOMER_LOGIN_URL
|
||||
assert 'img[src*="aweme.png"]' in page.selectors
|
||||
assert page.douyin.clicked
|
||||
|
||||
|
||||
def test_xingtu_index_page_is_recognized_as_logged_in():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def wait_for(self, timeout):
|
||||
assert timeout == 1200
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/index"
|
||||
|
||||
def get_by_text(self, _text, exact=False):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is True
|
||||
|
||||
|
||||
def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie():
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "stale"}]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
raise AssertionError("public homepage must be rejected before marker checks")
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_market_page_without_business_ui_rejects_stale_cookie():
|
||||
class Locator:
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def wait_for(self, **_kwargs):
|
||||
raise xingtu_scraper_v2.PlaywrightTimeoutError("missing")
|
||||
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return [{"name": "sessionid", "value": "stale"}]
|
||||
|
||||
class Page:
|
||||
url = "https://www.xingtu.cn/ad/creator/market"
|
||||
context = Context()
|
||||
|
||||
def get_by_text(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
def locator(self, *_args, **_kwargs):
|
||||
return Locator()
|
||||
|
||||
assert xingtu_scraper_v2.is_logged_in(Page()) is False
|
||||
|
||||
|
||||
def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is False
|
||||
|
||||
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
|
||||
assert xingtu_scraper_v2.has_valid_login_cookie_file(cookie) is True
|
||||
|
||||
|
||||
def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowed(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
|
||||
class Session:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fetch(self, *_args, **_kwargs):
|
||||
# Scrapling can log a page_action error and still return a response.
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
|
||||
|
||||
assert xingtu_scraper_v2.login_only(1) == 1
|
||||
|
||||
|
||||
def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
cookie = tmp_path / "xingtu.json"
|
||||
|
||||
class Session:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def fetch(self, *_args, **_kwargs):
|
||||
raise RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session())
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None)
|
||||
|
||||
def verify():
|
||||
cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8")
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify)
|
||||
|
||||
assert xingtu_scraper_v2.login_only(1) == 0
|
||||
|
||||
|
||||
def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch):
|
||||
class Context:
|
||||
def __init__(self):
|
||||
self.pages = []
|
||||
|
||||
def new_page(self):
|
||||
raise AssertionError("a surviving replacement page should be reused")
|
||||
|
||||
class Page:
|
||||
def __init__(self, context, url, *, closes_on_wait=False):
|
||||
self.context = context
|
||||
self.url = url
|
||||
self.closed = False
|
||||
self.closes_on_wait = closes_on_wait
|
||||
|
||||
def is_closed(self):
|
||||
return self.closed
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
if self.closes_on_wait:
|
||||
self.closed = True
|
||||
|
||||
def goto(self, url, **_kwargs):
|
||||
self.url = url
|
||||
|
||||
context = Context()
|
||||
qr_page = Page(
|
||||
context,
|
||||
"https://open.douyin.com/platform/oauth/pc/auth",
|
||||
closes_on_wait=True,
|
||||
)
|
||||
market_page = Page(context, xingtu_scraper_v2.MARKET_URL)
|
||||
context.pages = [qr_page, market_page]
|
||||
saved = []
|
||||
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page
|
||||
)
|
||||
monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page))
|
||||
|
||||
result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2)
|
||||
|
||||
assert result is market_page
|
||||
assert saved == [market_page]
|
||||
|
||||
|
||||
def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "xingtu_cookies.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text("old-cookie", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_xingtu, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_xingtu, "PROFILE_DIR", profile)
|
||||
|
||||
cookie_backup, profile_backup = relogin_xingtu.reset_state()
|
||||
profile.mkdir()
|
||||
(profile / "partial.txt").write_text("partial", encoding="utf-8")
|
||||
|
||||
assert relogin_xingtu.recover_interrupted_state() is True
|
||||
assert cookie.read_text(encoding="utf-8") == "old-cookie"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
assert not (profile / "partial.txt").exists()
|
||||
assert cookie_backup is not None
|
||||
assert profile_backup is not None
|
||||
|
||||
|
||||
def test_pgy_scan_login_clicks_qr_switch_image():
|
||||
class FakeLocator:
|
||||
def __init__(self):
|
||||
self.first = self
|
||||
self.clicked = False
|
||||
self.evaluated = False
|
||||
|
||||
def count(self):
|
||||
return 1
|
||||
|
||||
def is_visible(self):
|
||||
return True
|
||||
|
||||
def click(self, timeout):
|
||||
self.clicked = True
|
||||
|
||||
def evaluate(self, script):
|
||||
assert script == "e => e.click()"
|
||||
self.evaluated = True
|
||||
|
||||
class FakePage:
|
||||
def __init__(self):
|
||||
self.selectors = []
|
||||
self.qr_switch = FakeLocator()
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
pass
|
||||
|
||||
def get_by_text(self, _pattern):
|
||||
return FakeLocator()
|
||||
|
||||
def locator(self, selector):
|
||||
self.selectors.append(selector)
|
||||
return self.qr_switch
|
||||
|
||||
page = FakePage()
|
||||
pgy_xhs_scraper_v2.open_scan_login(page)
|
||||
|
||||
assert 'img[src*="qr_code"]' in page.selectors
|
||||
assert page.qr_switch.evaluated
|
||||
|
||||
|
||||
def test_failed_douyin_relogin_restores_cookie_storage_and_profile(tmp_path, monkeypatch):
|
||||
cookie = tmp_path / "douyin_cookies.json"
|
||||
storage = tmp_path / "douyin_storage_state.json"
|
||||
profile = tmp_path / "profile"
|
||||
cookie.write_text('[{"name":"sessionid","value":"old"}]', encoding="utf-8")
|
||||
storage.write_text("old-storage", encoding="utf-8")
|
||||
profile.mkdir()
|
||||
(profile / "old.txt").write_text("old-profile", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(relogin_douyin, "COOKIE_FILE", cookie)
|
||||
monkeypatch.setattr(relogin_douyin, "STATE_FILE", storage)
|
||||
monkeypatch.setattr(relogin_douyin, "PROFILE_DIR", profile)
|
||||
monkeypatch.setattr(relogin_douyin.subprocess, "call", lambda *_args, **_kwargs: 1)
|
||||
monkeypatch.setattr(relogin_douyin.sys, "argv", ["relogin_douyin.py"])
|
||||
|
||||
assert relogin_douyin.main() == 1
|
||||
assert cookie.read_text(encoding="utf-8") == '[{"name":"sessionid","value":"old"}]'
|
||||
assert storage.read_text(encoding="utf-8") == "old-storage"
|
||||
assert (profile / "old.txt").read_text(encoding="utf-8") == "old-profile"
|
||||
@@ -0,0 +1,84 @@
|
||||
import io
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.adapters import RuntimeServicePolicy
|
||||
from gyxx_flow.modules.content_marketing.data.tools import db
|
||||
from gyxx_flow.modules.content_marketing.data.tools import (
|
||||
sync_metrics_to_cmt_notes as sync_metrics,
|
||||
)
|
||||
|
||||
ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "content_marketing"
|
||||
)
|
||||
|
||||
|
||||
class LocalDatabaseConfigTests(unittest.TestCase):
|
||||
def test_config_requires_every_final_connection_value(self):
|
||||
complete = {
|
||||
"PG_HOST": "127.0.0.1",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "gyxx_super_data",
|
||||
"PG_USER": "gyxx_flow",
|
||||
"PG_PASSWORD": "secret",
|
||||
}
|
||||
self.assertEqual(
|
||||
db.database_config_from_env(complete),
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 5432,
|
||||
"dbname": "gyxx_super_data",
|
||||
"user": "gyxx_flow",
|
||||
"password": "secret",
|
||||
},
|
||||
)
|
||||
for missing in complete:
|
||||
values = {key: value for key, value in complete.items() if key != missing}
|
||||
with self.subTest(missing=missing), self.assertRaisesRegex(RuntimeError, missing):
|
||||
db.database_config_from_env(values)
|
||||
|
||||
def test_runtime_policy_uses_explicit_cloud_postgres(self):
|
||||
environment = RuntimeServicePolicy().apply(
|
||||
{
|
||||
"PG_HOST": "db.example.com",
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "data_hub",
|
||||
"PG_USER": "data_hub",
|
||||
"GYXX_POSTGRES_PASSWORD": "secret",
|
||||
}
|
||||
)
|
||||
config = db.database_config_from_env(environment)
|
||||
self.assertEqual(config["host"], "db.example.com")
|
||||
self.assertEqual(config["dbname"], "data_hub")
|
||||
self.assertEqual(config["user"], "data_hub")
|
||||
|
||||
def test_database_scripts_do_not_define_remote_connection_defaults(self):
|
||||
paths = (
|
||||
ROOT / "data" / "tools" / "db.py",
|
||||
ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py",
|
||||
ROOT / "data" / "tools" / "sync_cooperations.py",
|
||||
ROOT / "data" / "tools" / "generate_creator_report.py",
|
||||
)
|
||||
for path in paths:
|
||||
source = path.read_text(encoding="utf-8-sig")
|
||||
with self.subTest(path=path.name):
|
||||
self.assertNotIn("8.148.185.119", source)
|
||||
self.assertNotIn("data_hub", source)
|
||||
|
||||
def test_sync_console_replaces_characters_unsupported_by_gbk(self):
|
||||
raw = io.BytesIO()
|
||||
stream = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
|
||||
sync_metrics.configure_console_output(stream, None)
|
||||
stream.write("𝑻")
|
||||
stream.flush()
|
||||
|
||||
self.assertTrue(raw.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,398 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import gyxx_flow.modules.content_marketing.data.tools.retry_failed as retry
|
||||
from gyxx_flow.modules.content_marketing import run_all
|
||||
|
||||
|
||||
def test_requested_records_must_be_successful_even_when_process_exits_zero():
|
||||
data = {
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "retryable_failure", "matched": False},
|
||||
]
|
||||
}
|
||||
|
||||
ok, failed = retry.requested_records_succeeded(data, ["a", "b"])
|
||||
|
||||
assert ok is False
|
||||
assert failed == ["b"]
|
||||
|
||||
|
||||
def test_success_status_cannot_hide_write_failure_and_blocked_input_is_terminal():
|
||||
contradictory = {
|
||||
"results": [{
|
||||
"record_id": "a", "status": "success", "matched": True,
|
||||
"write_ok": False,
|
||||
}]
|
||||
}
|
||||
assert retry.requested_records_succeeded(contradictory, ["a"]) == (False, ["a"])
|
||||
|
||||
blocked = {
|
||||
"total": 1,
|
||||
"details": [{
|
||||
"record_id": "b", "status": "blocked_input", "ok": False,
|
||||
"reason": "url_missing",
|
||||
}],
|
||||
}
|
||||
assert retry.extract_failed_records(blocked, "bili") == []
|
||||
|
||||
|
||||
def test_canonical_filename_requires_current_style_name():
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
|
||||
assert retry.canonical_result_filename(style, "pgy", False) == "02-极星pro_v2.json"
|
||||
assert retry.canonical_result_filename(style, "xt", False) == "02-极星pro_xingtu_v2.json"
|
||||
assert retry.canonical_result_filename(style, "bili", True) == "02-极星pro_self_bilibili_v2.json"
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict | list) -> None:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def test_find_failed_uses_exact_current_style_file_and_ignores_legacy_name(tmp_path):
|
||||
style = {"index": 2, "name": "极星pro"}
|
||||
canonical = tmp_path / "02-极星pro_v2.json"
|
||||
legacy = tmp_path / "02-盖亚微单_v2.json"
|
||||
payload = {
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
}
|
||||
_write_json(canonical, payload)
|
||||
_write_json(legacy, {"index": 2, "total": 1, "matched": 0, "results": []})
|
||||
old = time.time() - 48 * 3600
|
||||
os.utime(legacy, (old, old))
|
||||
|
||||
failed = retry.find_failed(
|
||||
"pgy", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed == {}
|
||||
|
||||
|
||||
def test_find_failed_treats_missing_result_rows_as_incomplete(tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
path = tmp_path / retry.canonical_result_filename(style, "xt", False)
|
||||
_write_json(path, {"index": 1, "total": 2, "matched": 0, "filled": 0, "results": []})
|
||||
|
||||
failed = retry.find_failed(
|
||||
"xt", stale_hours=4, include_half=True,
|
||||
styles=[style], v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert failed[1]["reason"] == "ALL_FAILED"
|
||||
assert failed[1]["failed_rids"] == []
|
||||
|
||||
|
||||
def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
calls = []
|
||||
|
||||
def fake_call(_cmd, cwd, env=None):
|
||||
del env
|
||||
calls.append(cwd)
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"filled": 0,
|
||||
"results": [{
|
||||
"record_id": "bad",
|
||||
"status": "retryable_failure",
|
||||
"matched": False,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
monkeypatch.setattr(retry.time, "sleep", lambda _seconds: None)
|
||||
|
||||
ok = retry.run_one_style(
|
||||
"pgy", 1, max_attempts=2, record_ids=["bad"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False)
|
||||
|
||||
def fake_call(_cmd, cwd, env=None):
|
||||
del env
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
|
||||
assert retry.run_one_style(
|
||||
"pgy", 1, max_attempts=1, record_ids=["ok"],
|
||||
style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
|
||||
|
||||
def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_path):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
result_path = tmp_path / retry.canonical_result_filename(style, "xt", True)
|
||||
observed = {}
|
||||
|
||||
def fake_call(cmd, cwd, env=None):
|
||||
del env
|
||||
observed["cmd"] = cmd
|
||||
_write_json(result_path, {
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success", "matched": True}],
|
||||
})
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "call", fake_call)
|
||||
monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {})
|
||||
|
||||
assert retry.run_one_style(
|
||||
"xt", 1, max_attempts=1, self_operated=True,
|
||||
record_ids=["ok"], style=style, v2_dir=tmp_path,
|
||||
) is True
|
||||
assert observed["cmd"][1].endswith("self_douyin_scraper.py")
|
||||
assert "--self-operated" not in observed["cmd"]
|
||||
|
||||
|
||||
def test_full_summary_declared_failure_cannot_be_treated_as_complete():
|
||||
complete, reason = retry._summary_completed({
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "ok",
|
||||
"status": "success",
|
||||
"matched": True,
|
||||
"write_ok": True,
|
||||
}],
|
||||
"write_failures": 1,
|
||||
})
|
||||
|
||||
assert complete is False
|
||||
assert reason == "summary declares unresolved failures"
|
||||
|
||||
|
||||
def test_retry_main_returns_nonzero_when_verified_retry_still_fails(monkeypatch):
|
||||
style = {"index": 1, "name": "款式A"}
|
||||
monkeypatch.setattr(retry, "load_current_styles", lambda self_operated=False: [style])
|
||||
monkeypatch.setattr(
|
||||
retry, "find_failed",
|
||||
lambda *_args, **_kwargs: {1: {"reason": "HALF", "failed_rids": ["bad"]}},
|
||||
)
|
||||
monkeypatch.setattr(retry, "run_one_style", lambda *_args, **_kwargs: False)
|
||||
monkeypatch.setattr(retry.sys, "argv", ["retry_failed.py", "--platform", "pgy"])
|
||||
|
||||
assert retry.main() == 1
|
||||
|
||||
|
||||
def test_validate_summary_payload_rejects_stale_missing_duplicate_and_unresolved(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
valid = [{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"filled": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success", "matched": True},
|
||||
{"record_id": "b", "status": "success", "matched": True},
|
||||
],
|
||||
}]
|
||||
_write_json(path, valid)
|
||||
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime - 1,
|
||||
)[0] is True
|
||||
assert run_all.validate_summary_payload(
|
||||
valid, {1}, path, path.stat().st_mtime + 1,
|
||||
)[0] is False
|
||||
assert run_all.validate_summary_payload(valid, {1, 2}, path, 0)[0] is False
|
||||
|
||||
duplicate = [{
|
||||
"index": 1, "total": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "a", "status": "success"},
|
||||
],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(duplicate, {1}, path, 0)[0] is False
|
||||
|
||||
unresolved = [{
|
||||
"index": 1, "total": 1,
|
||||
"results": [{
|
||||
"record_id": "a", "status": "retryable_failure", "matched": False,
|
||||
}],
|
||||
}]
|
||||
assert run_all.validate_summary_payload(unresolved, {1}, path, 0)[0] is False
|
||||
|
||||
|
||||
def test_validate_selected_style_allows_unrelated_rows_in_merged_global_summary(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": time.time() + 10,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "ok", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"total": 1,
|
||||
"results": [{
|
||||
"record_id": "old-failure",
|
||||
"status": "retryable_failure",
|
||||
}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1}, path, path.stat().st_mtime - 1,
|
||||
)
|
||||
|
||||
assert valid is True, reason
|
||||
|
||||
|
||||
def test_validate_summary_rejects_expected_style_left_over_from_old_run(tmp_path):
|
||||
path = tmp_path / "summary.json"
|
||||
started_at = 100.0
|
||||
payload = [
|
||||
{
|
||||
"index": 1,
|
||||
"run_started_at": 101.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "new", "status": "success"}],
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"run_started_at": 90.0,
|
||||
"total": 1,
|
||||
"results": [{"record_id": "old", "status": "success"}],
|
||||
},
|
||||
]
|
||||
_write_json(path, payload)
|
||||
|
||||
valid, reason = run_all.validate_summary_payload(
|
||||
payload, {1, 2}, path, started_at,
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "style 2" in reason
|
||||
assert "current run" in reason
|
||||
|
||||
|
||||
def test_aggregate_counts_all_unresolved_rows_as_errors():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": [{
|
||||
"index": 1,
|
||||
"total": 3,
|
||||
"matched": 1,
|
||||
"filled": 1,
|
||||
"results": [
|
||||
{"record_id": "ok", "status": "success"},
|
||||
{"record_id": "miss", "matched": False},
|
||||
{"record_id": "detail", "reason": "no_detail_page"},
|
||||
],
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["pgy"]["unresolved"] == 2
|
||||
assert aggregate["platforms"]["pgy"]["errors"] == 2
|
||||
|
||||
|
||||
def test_aggregate_sums_declared_failure_categories_when_unresolved_is_absent():
|
||||
aggregate = run_all.aggregate({
|
||||
"xt": [{
|
||||
"index": 1,
|
||||
"total": 2,
|
||||
"matched": 2,
|
||||
"results": [
|
||||
{"record_id": "a", "status": "success"},
|
||||
{"record_id": "b", "status": "success"},
|
||||
],
|
||||
"retryable_failures": 1,
|
||||
"write_failures": 1,
|
||||
}],
|
||||
})
|
||||
|
||||
assert aggregate["platforms"]["xt"]["unresolved"] == 2
|
||||
|
||||
|
||||
def test_aggregate_keeps_partial_counts_when_validation_marks_payload_invalid():
|
||||
aggregate = run_all.aggregate({
|
||||
"pgy": {
|
||||
"error": "invalid summary: unresolved",
|
||||
"payload": [{
|
||||
"index": 1,
|
||||
"total": 1,
|
||||
"matched": 0,
|
||||
"results": [{
|
||||
"record_id": "bad", "status": "retryable_failure",
|
||||
}],
|
||||
}],
|
||||
},
|
||||
})
|
||||
|
||||
platform = aggregate["platforms"]["pgy"]
|
||||
assert platform["loaded"] is True
|
||||
assert platform["unresolved"] == 1
|
||||
assert platform["validation_error"] == "invalid summary: unresolved"
|
||||
|
||||
|
||||
def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monkeypatch, tmp_path):
|
||||
args = SimpleNamespace(dry_run=True, style=[1])
|
||||
stdout = tmp_path / "stdout.log"
|
||||
stderr = tmp_path / "stderr.log"
|
||||
observed = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
run_all, "start_process",
|
||||
lambda *_args, **_kwargs: (object(), stdout, stderr),
|
||||
)
|
||||
monkeypatch.setattr(run_all, "wait_all", lambda _procs: {"pgy": 0})
|
||||
|
||||
def fake_load(key, self_operated=False, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"error": "invalid summary: stale"}
|
||||
|
||||
monkeypatch.setattr(run_all, "load_summary", fake_load)
|
||||
monkeypatch.setattr(run_all, "print_report", lambda *_args: None)
|
||||
monkeypatch.setattr(run_all, "_atomic_write_json", lambda *_args: None)
|
||||
|
||||
result = run_all.run_round(args, ["pgy"])
|
||||
|
||||
assert result["exit_codes"] == {"pgy": 0}
|
||||
assert result["_non_zero_platforms"] == ["pgy"]
|
||||
assert observed["expected_indices"] == {1}
|
||||
assert isinstance(observed["started_at"], float)
|
||||
|
||||
|
||||
def test_run_all_preserves_cookie_skip_exit_code():
|
||||
assert run_all.aggregate_exit_code(True, [75, 75]) == 75
|
||||
assert run_all.aggregate_exit_code(True, [75, 1]) == 1
|
||||
assert run_all.aggregate_exit_code(True, []) == 1
|
||||
assert run_all.aggregate_exit_code(False, []) == 0
|
||||
@@ -0,0 +1,10 @@
|
||||
from gyxx_flow.modules.content_marketing import self_douyin_scraper
|
||||
|
||||
|
||||
def test_self_douyin_accepts_feishu_select_and_multiselect_values():
|
||||
assert self_douyin_scraper.is_douyin_platform("抖音") is True
|
||||
assert self_douyin_scraper.is_douyin_platform(["抖音"]) is True
|
||||
assert self_douyin_scraper.is_douyin_platform("Douyin") is True
|
||||
assert self_douyin_scraper.is_douyin_platform(["小红书", "抖音"]) is True
|
||||
assert self_douyin_scraper.is_douyin_platform("小红书") is False
|
||||
assert self_douyin_scraper.is_douyin_platform(None) is False
|
||||
@@ -0,0 +1,58 @@
|
||||
import unittest
|
||||
|
||||
from gyxx_flow.modules.content_marketing.data.tools.sync_style_categories import (
|
||||
extract_style_mapping,
|
||||
normalize_style_name,
|
||||
)
|
||||
|
||||
|
||||
class SyncStyleCategoriesTests(unittest.TestCase):
|
||||
def test_normalizes_feishu_lifecycle_names_to_database_names(self):
|
||||
cases = {
|
||||
"盖亚微单Pro生命进程（新）": "盖亚微单",
|
||||
"星迹&星迹2生命进程（新）": "星迹2",
|
||||
"宙斯双肩包生命进程（新）": "宙斯",
|
||||
"极星Pro生命进程（新）": "极星pro",
|
||||
"瑞白双肩包生命进程（新）": "瑞白",
|
||||
"逐星gt生命进程（新）": "逐星GT",
|
||||
"拾影相机包生命进程 （新）": "拾影斜挎相机包",
|
||||
"极星托特生命进程（新）": "极星托特",
|
||||
}
|
||||
for source, expected in cases.items():
|
||||
with self.subTest(source=source):
|
||||
self.assertEqual(normalize_style_name(source), expected)
|
||||
|
||||
def test_extracts_mapping_and_ignores_blank_rows(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
[
|
||||
"[盖亚斜挎生命进程（新）](https://example.feishu.cn/base/TokenA)",
|
||||
["都市机能"],
|
||||
],
|
||||
[None, ["都市运动"]],
|
||||
["[极星托特生命进程（新）](https://example.feishu.cn/base/TokenB)", ["智性通勤"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
self.assertEqual(
|
||||
extract_style_mapping(payload),
|
||||
{"盖亚斜挎": "都市机能", "极星托特": "智性通勤"},
|
||||
)
|
||||
|
||||
def test_rejects_conflicting_categories_for_same_style(self):
|
||||
payload = {
|
||||
"data": {
|
||||
"data": [
|
||||
["[布谷生命进程（新）](https://example/a)", ["都市运动"]],
|
||||
["[布谷生命进程（新）](https://example/b)", ["都市机能"]],
|
||||
]
|
||||
}
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "布谷"):
|
||||
extract_style_mapping(payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,70 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly
|
||||
|
||||
|
||||
class WeeklyTmallPersonaTests(unittest.TestCase):
|
||||
def test_missing_persona_forbids_invented_percentages(self):
|
||||
text = weekly.format_tmall_persona(None)
|
||||
self.assertIn("不得虚构具体比例", text)
|
||||
|
||||
def test_format_tmall_persona_includes_date_and_dimensions(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [
|
||||
{"name": "男性用户", "value": 66.97},
|
||||
{"name": "女性用户", "value": 28.26},
|
||||
],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [{"name": "准一线城市", "value": 31.14}],
|
||||
"buying_power": [{"name": "购买力L5", "value": 30.65}],
|
||||
},
|
||||
}
|
||||
|
||||
text = weekly.format_tmall_persona(persona)
|
||||
|
||||
self.assertIn("2026-07-15", text)
|
||||
self.assertIn("男性用户 66.97%", text)
|
||||
self.assertIn("25-29岁 26.97%", text)
|
||||
self.assertIn("准一线城市 31.14%", text)
|
||||
self.assertIn("购买力L5 30.65%", text)
|
||||
|
||||
def test_generate_summary_injects_latest_tmall_persona_into_single_note_prompt(self):
|
||||
persona = {
|
||||
"collect_date": "2026-07-15",
|
||||
"item_id": "900176117674",
|
||||
"payload": {
|
||||
"gender": [{"name": "男性用户", "value": 66.97}],
|
||||
"age": [{"name": "25-29岁", "value": 26.97}],
|
||||
"city_tier": [],
|
||||
"buying_power": [],
|
||||
},
|
||||
}
|
||||
notes = [{
|
||||
"title": "测试笔记",
|
||||
"platform": "douyin",
|
||||
"creator_name": "达人甲",
|
||||
"view_count": 100,
|
||||
"like_count": 10,
|
||||
"comment_count": 2,
|
||||
"favorite_count": 3,
|
||||
"share_count": 1,
|
||||
"report_path": None,
|
||||
}]
|
||||
|
||||
with patch.object(weekly, "get_latest_tmall_persona", return_value=persona) as get_persona, \
|
||||
patch.object(weekly, "call_hermes_analyzer", side_effect=["单篇分析", "## 三、周度总结"]) as call:
|
||||
weekly.generate_summary("星云2", notes)
|
||||
|
||||
get_persona.assert_called_once_with("星云2")
|
||||
single_prompt = call.call_args_list[0].args[1]
|
||||
self.assertIn("天猫该款最新人物画像", single_prompt)
|
||||
self.assertIn("男性用户 66.97%", single_prompt)
|
||||
self.assertIn("内容触达人群", single_prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Test bootstrap for the flattened product-commerce module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MODULE_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
|
||||
# The migrated scripts intentionally retain their original sibling imports.
|
||||
if str(MODULE_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(MODULE_ROOT))
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
COMMAND_PATH = (
|
||||
PROJECT_ROOT
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "commands"
|
||||
/ "run_alerts.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_command(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
observed: dict[str, object],
|
||||
) -> ModuleType:
|
||||
legacy = ModuleType("run_alerts_with_retry")
|
||||
|
||||
def fake_main() -> int:
|
||||
observed["argv"] = tuple(sys.argv)
|
||||
return 0
|
||||
|
||||
legacy.main = fake_main # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "run_alerts_with_retry", legacy)
|
||||
spec = importlib.util.spec_from_file_location("test_run_alerts_command", COMMAND_PATH)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_alert_command_uses_canonical_acceptance_recipient(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
monkeypatch.delenv("GYXX_ALERT_RECIPIENT_OPEN_ID", raising=False)
|
||||
module = _load_command(monkeypatch, observed)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"] == (
|
||||
sys.argv[0],
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID,
|
||||
)
|
||||
|
||||
|
||||
def test_alert_command_accepts_canonical_production_recipient(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01")
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
monkeypatch.setenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", "ou_configured")
|
||||
monkeypatch.setenv("GYXX_ALERT_RECIPIENT_OPEN_ID", "ou_legacy")
|
||||
module = _load_command(monkeypatch, observed)
|
||||
|
||||
assert module.main() == 0
|
||||
assert observed["argv"][-1] == "ou_configured"
|
||||
@@ -0,0 +1,362 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import check_nine_day_decline as decline
|
||||
import pytest
|
||||
import run_alerts_with_retry as retry
|
||||
|
||||
PAIR = ("tm", "测试款")
|
||||
|
||||
|
||||
def _segment(sales: int, *, days: int = 3) -> dict:
|
||||
return {
|
||||
"sales": sales,
|
||||
"present_days": days,
|
||||
"erp_style_codes": ["STYLE-1"],
|
||||
}
|
||||
|
||||
|
||||
def _event() -> dict:
|
||||
return {
|
||||
"style_name": "测试款",
|
||||
"platform": "tm",
|
||||
"erp_style_codes": ["STYLE-1"],
|
||||
"window_start": "2026-07-24",
|
||||
"window_end": "2026-08-01",
|
||||
"seg1_sales": 300,
|
||||
"seg2_sales": 210,
|
||||
"seg3_sales": 90,
|
||||
"drop_pct_1to2": 30.0,
|
||||
"drop_pct_2to3": 57.1,
|
||||
"drop_pct_1to3": 70.0,
|
||||
}
|
||||
|
||||
|
||||
def _prepare_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"check_nine_day_decline.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_owner",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(decline, "managed_data_path", lambda value: Path(value))
|
||||
monkeypatch.setattr(decline, "append_log", lambda line: None)
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
|
||||
|
||||
def test_detect_decline_uses_three_inclusive_postgres_sum_windows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
observed: list[list[tuple[date, date]]] = []
|
||||
|
||||
def fake_load(windows):
|
||||
observed.append(windows)
|
||||
return [
|
||||
{PAIR: _segment(300)},
|
||||
{PAIR: _segment(210)},
|
||||
{PAIR: _segment(90)},
|
||||
]
|
||||
|
||||
monkeypatch.setattr(decline, "_load_segment_snapshots", fake_load)
|
||||
|
||||
events = decline.detect_decline("2026-08-01", 9)
|
||||
|
||||
assert observed == [
|
||||
[
|
||||
(date(2026, 7, 24), date(2026, 7, 26)),
|
||||
(date(2026, 7, 27), date(2026, 7, 29)),
|
||||
(date(2026, 7, 30), date(2026, 8, 1)),
|
||||
]
|
||||
]
|
||||
assert len(events) == 1
|
||||
assert events[0]["window_start"] == "2026-07-24"
|
||||
assert events[0]["window_end"] == "2026-08-01"
|
||||
assert [events[0][f"seg{index}_sales"] for index in range(1, 4)] == [
|
||||
300,
|
||||
210,
|
||||
90,
|
||||
]
|
||||
assert events[0]["segment_windows"] == [
|
||||
{"start": "2026-07-24", "end": "2026-07-26", "days": 3},
|
||||
{"start": "2026-07-27", "end": "2026-07-29", "days": 3},
|
||||
{"start": "2026-07-30", "end": "2026-08-01", "days": 3},
|
||||
]
|
||||
|
||||
|
||||
def test_detect_decline_rejects_incomplete_three_day_segment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_load_segment_snapshots",
|
||||
lambda windows: [
|
||||
{PAIR: _segment(300)},
|
||||
{PAIR: _segment(210, days=2)},
|
||||
{PAIR: _segment(90)},
|
||||
],
|
||||
)
|
||||
|
||||
assert decline.detect_decline("2026-08-01", 9) == []
|
||||
|
||||
|
||||
def test_segment_loader_uses_postgres_range_sum_on_one_connection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fake_db = ModuleType("db")
|
||||
connection = object()
|
||||
observed: list[tuple[object, str, str]] = []
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
return connection
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def fake_fetch(conn, start, end):
|
||||
observed.append((conn, start, end))
|
||||
return {}
|
||||
|
||||
fake_db.get_conn = ConnectionContext # type: ignore[attr-defined]
|
||||
fake_db.fetch_daily_for_range = fake_fetch # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "db", fake_db)
|
||||
windows = decline._segment_windows("2026-08-01", 9)
|
||||
|
||||
assert decline._load_segment_snapshots(windows) == [{}, {}, {}]
|
||||
assert observed == [
|
||||
(connection, "2026-07-24", "2026-07-26"),
|
||||
(connection, "2026-07-27", "2026-07-29"),
|
||||
(connection, "2026-07-30", "2026-08-01"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response", "expected"),
|
||||
[
|
||||
({"id": "chatcmpl-not-a-receipt"}, None),
|
||||
({"message_id": "om_direct123"}, "om_direct123"),
|
||||
({"data": {"message_id": "om_nested123"}}, "om_nested123"),
|
||||
(
|
||||
{"choices": [{"message": {"content": "飞书回执 om_content123"}}]},
|
||||
"om_content123",
|
||||
),
|
||||
(
|
||||
{"choices": [{"message": {"content": "发送成功，但没有回执"}}]},
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_feishu_message_id_requires_real_receipt_shape(
|
||||
response: dict,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
assert decline.extract_feishu_message_id(response) == expected
|
||||
|
||||
|
||||
def test_main_fails_when_postgres_detection_read_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"detect_decline",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("postgres unavailable")),
|
||||
)
|
||||
|
||||
assert decline.main() == 1
|
||||
|
||||
|
||||
def test_main_zero_events_has_explicit_postcondition(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [])
|
||||
|
||||
assert decline.main() == 0
|
||||
output = capsys.readouterr().out
|
||||
assert "detection_source=postgresql events=0" in output
|
||||
assert "notification=not_required persistence=not_required" in output
|
||||
|
||||
|
||||
def test_main_dedup_query_failure_is_fail_closed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_already_notified",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("db timeout")),
|
||||
)
|
||||
notify = pytest.fail
|
||||
monkeypatch.setattr(decline, "notify_hermes", notify)
|
||||
|
||||
assert decline.main() == 5
|
||||
|
||||
|
||||
def test_main_does_not_accept_arbitrary_non_error_hermes_text(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
lambda *args, **kwargs: {
|
||||
"id": "chatcmpl-123",
|
||||
"choices": [{"message": {"content": "发送成功"}}],
|
||||
},
|
||||
)
|
||||
persist = pytest.fail
|
||||
monkeypatch.setattr(decline, "_persist_events", persist)
|
||||
|
||||
assert decline.main() == 6
|
||||
|
||||
|
||||
def test_main_returns_nonzero_when_receipt_persistence_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"_persist_events",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("write failed")),
|
||||
)
|
||||
|
||||
assert decline.main() == 4
|
||||
|
||||
|
||||
def test_main_success_requires_and_persists_feishu_receipt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
_prepare_main(monkeypatch, tmp_path)
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()])
|
||||
monkeypatch.setattr(decline, "_already_notified", lambda *args: False)
|
||||
monkeypatch.setattr(
|
||||
decline,
|
||||
"notify_hermes",
|
||||
lambda *args, **kwargs: {"message_id": "om_real123"},
|
||||
)
|
||||
|
||||
def fake_persist(*args, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(decline, "_persist_events", fake_persist)
|
||||
|
||||
assert decline.main() == 0
|
||||
assert observed["message_id"] == "om_real123"
|
||||
assert observed["openid"] == "ou_owner"
|
||||
output = capsys.readouterr().out
|
||||
assert "notification=sent feishu_message_id=om_real123" in output
|
||||
assert "persistence=committed history_rows=1" in output
|
||||
|
||||
|
||||
class _FailedCollector:
|
||||
def __init__(self, *args):
|
||||
self.pages: list[tuple] = []
|
||||
|
||||
def add_page(self, *args, **kwargs):
|
||||
self.pages.append((args, kwargs))
|
||||
|
||||
def write(self):
|
||||
return None
|
||||
|
||||
def summary_line(self):
|
||||
return "failure"
|
||||
|
||||
|
||||
def test_retry_wrapper_does_not_repeat_after_possible_success(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"run_alerts_with_retry.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_owner",
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(retry, "FailedCollector", _FailedCollector)
|
||||
monkeypatch.setattr(retry, "consolidate_failed_logs", lambda value: None)
|
||||
monkeypatch.setattr(retry, "build_child_environment", lambda *args: {})
|
||||
monkeypatch.setattr(
|
||||
retry.time,
|
||||
"sleep",
|
||||
lambda seconds: pytest.fail("non-retryable result must not sleep"),
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return subprocess.CompletedProcess(command, 4)
|
||||
|
||||
monkeypatch.setattr(retry.subprocess, "run", fake_run)
|
||||
|
||||
assert retry.main() == 4
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_retry_wrapper_can_retry_fail_closed_dedup_read(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
return_codes = iter((5, 0))
|
||||
sleeps: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"run_alerts_with_retry.py",
|
||||
"--end-date",
|
||||
"2026-08-01",
|
||||
"--openid",
|
||||
"ou_owner",
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(retry, "FailedCollector", _FailedCollector)
|
||||
monkeypatch.setattr(retry, "consolidate_failed_logs", lambda value: None)
|
||||
monkeypatch.setattr(retry, "build_child_environment", lambda *args: {})
|
||||
monkeypatch.setattr(retry.time, "sleep", sleeps.append)
|
||||
monkeypatch.setattr(
|
||||
retry.subprocess,
|
||||
"run",
|
||||
lambda command, **kwargs: subprocess.CompletedProcess(
|
||||
command,
|
||||
next(return_codes),
|
||||
),
|
||||
)
|
||||
|
||||
assert retry.main() == 0
|
||||
assert sleeps == [retry.RUN_SLEEP_SECONDS]
|
||||
@@ -0,0 +1,210 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import backfill_poseidon_sales as backfill
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def test_backfill_uses_layered_runtime_roots():
|
||||
assert backfill.DOWNLOAD_ROOT == backfill.RAW_DATA_ROOT
|
||||
assert backfill.PLATFORM_OUTPUT_ROOT == backfill.NORMALIZED_DATA_ROOT
|
||||
assert backfill.FINAL_OUTPUT_ROOT == backfill.CURATED_DATA_ROOT
|
||||
assert backfill.REPORT_OUTPUT_ROOT == backfill.EXPORTS_DATA_ROOT
|
||||
|
||||
|
||||
def test_extract_jd_uses_existing_pipeline_metric_definitions():
|
||||
frame = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"时间": "2026-07-01",
|
||||
"商品ID": "10035289085755",
|
||||
"访客数": "1,234",
|
||||
"加购人数": "23",
|
||||
"成交单量": "7",
|
||||
"成交商品件数": "9",
|
||||
"取消及售后退款单量": "2",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result = backfill.extract_metrics("jd", frame, ["10035289085755"], "2026-07-01")
|
||||
|
||||
assert result["matched_ids"] == ["10035289085755"]
|
||||
assert result["metrics"] == {
|
||||
"sales": 7,
|
||||
"visitors": 1234,
|
||||
"cart_users": 23,
|
||||
"refund_orders": 2,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_dy_ignores_weekly_period_and_uses_deal_people():
|
||||
frame = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"统计周期": "2026/07/01-2026/07/07",
|
||||
"商品编码": "3830202762700915010",
|
||||
"商品点击人数": "999",
|
||||
"加购人数": "99",
|
||||
"成交人数": "88",
|
||||
"成交订单数": "100",
|
||||
"退款订单数(支付时间)": "9",
|
||||
},
|
||||
{
|
||||
"统计周期": "2026/07/07",
|
||||
"商品编码": "3830202762700915010",
|
||||
"商品点击人数": "42",
|
||||
"加购人数": "10",
|
||||
"成交人数": "3",
|
||||
"成交订单数": "4",
|
||||
"退款订单数(支付时间)": "1",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
result = backfill.extract_metrics("dy", frame, ["3830202762700915010"], "2026-07-07")
|
||||
|
||||
assert result["matched_row_count"] == 1
|
||||
assert result["metrics"] == {
|
||||
"sales": 3,
|
||||
"visitors": 42,
|
||||
"cart_users": 10,
|
||||
"refund_orders": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_tm_sums_configured_ids_and_uses_paid_items():
|
||||
frame = pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"统计日期": "2026-07-25",
|
||||
"商品ID": "1061408417997",
|
||||
"商品访客数": "395",
|
||||
"商品加购人数": "17",
|
||||
"支付买家数": "3",
|
||||
"支付件数": "4",
|
||||
},
|
||||
{
|
||||
"统计日期": "2026-07-25",
|
||||
"商品ID": "1063521443835",
|
||||
"商品访客数": "5",
|
||||
"商品加购人数": "1",
|
||||
"支付买家数": "1",
|
||||
"支付件数": "2",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
result = backfill.extract_metrics(
|
||||
"tm",
|
||||
frame,
|
||||
["1060420778652", "1061408417997", "1063521443835"],
|
||||
"2026-07-25",
|
||||
)
|
||||
|
||||
assert result["matched_ids"] == ["1061408417997", "1063521443835"]
|
||||
assert result["missing_ids"] == ["1060420778652"]
|
||||
assert result["metrics"] == {
|
||||
"sales": 6,
|
||||
"visitors": 400,
|
||||
"cart_users": 18,
|
||||
"refund_orders": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_build_final_payload_preserves_excel_lineage():
|
||||
parsed = {
|
||||
"jd": {
|
||||
"metrics": {"sales": 1, "visitors": 10, "cart_users": 2, "refund_orders": 0},
|
||||
"matched_ids": ["jd-1"],
|
||||
"missing_ids": [],
|
||||
"matched_row_count": 1,
|
||||
"source_file": "jd.xlsx",
|
||||
},
|
||||
"dy": {
|
||||
"metrics": {"sales": 2, "visitors": 20, "cart_users": 3, "refund_orders": 1},
|
||||
"matched_ids": ["dy-1"],
|
||||
"missing_ids": [],
|
||||
"matched_row_count": 1,
|
||||
"source_file": "dy.xlsx",
|
||||
},
|
||||
"tm": {
|
||||
"metrics": {"sales": 3, "visitors": 30, "cart_users": 4, "refund_orders": 0},
|
||||
"matched_ids": ["tm-1"],
|
||||
"missing_ids": [],
|
||||
"matched_row_count": 1,
|
||||
"source_file": "tm.xls",
|
||||
},
|
||||
}
|
||||
|
||||
payload = backfill.build_final_payload("2026-07-01", parsed, ["10515"])
|
||||
|
||||
assert payload["style_name"] == backfill.STYLE_NAME
|
||||
assert payload["erp"]["style_codes"] == ["10515"]
|
||||
assert payload["jd"]["sales"] == 1
|
||||
assert payload["dy"]["source"] == "dy.xlsx"
|
||||
assert payload["tm"]["matched_product_ids"] == ["tm-1"]
|
||||
|
||||
|
||||
def test_render_final_markdown_contains_all_three_platforms():
|
||||
parsed = {
|
||||
"jd": {
|
||||
"metrics": {"sales": 1, "visitors": 10, "cart_users": 2, "refund_orders": 0},
|
||||
"matched_ids": ["jd-1"], "missing_ids": [], "matched_row_count": 1, "source_file": "jd.xlsx",
|
||||
},
|
||||
"dy": {
|
||||
"metrics": {"sales": 2, "visitors": 20, "cart_users": 3, "refund_orders": 1},
|
||||
"matched_ids": ["dy-1"], "missing_ids": [], "matched_row_count": 1, "source_file": "dy.xlsx",
|
||||
},
|
||||
"tm": {
|
||||
"metrics": {"sales": 3, "visitors": 30, "cart_users": 4, "refund_orders": 0},
|
||||
"matched_ids": ["tm-1"], "missing_ids": [], "matched_row_count": 1, "source_file": "tm.xls",
|
||||
},
|
||||
}
|
||||
payload = backfill.build_final_payload("2026-07-16", parsed, ["10515"])
|
||||
|
||||
markdown = backfill.render_final_markdown(payload)
|
||||
|
||||
assert "| jd | 1 | 0 | 10 | 2 |" in markdown
|
||||
assert "| dy | 2 | 1 | 20 | 3 |" in markdown
|
||||
assert "| tm | 3 | 0 | 30 | 4 |" in markdown
|
||||
|
||||
|
||||
def test_write_final_summary_updates_json_and_markdown(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(backfill, "FINAL_OUTPUT_ROOT", tmp_path)
|
||||
parsed = {
|
||||
platform: {
|
||||
"metrics": {"sales": 1, "visitors": 10, "cart_users": 2, "refund_orders": 0},
|
||||
"matched_ids": [platform], "missing_ids": [], "matched_row_count": 1, "source_file": f"{platform}.xlsx",
|
||||
}
|
||||
for platform in ("jd", "dy", "tm")
|
||||
}
|
||||
payload = backfill.build_final_payload("2026-07-16", parsed, ["10515"])
|
||||
|
||||
json_path = backfill.write_final_summary(payload)
|
||||
|
||||
markdown_path = json_path.with_suffix(".md")
|
||||
assert json_path.exists()
|
||||
assert markdown_path.exists()
|
||||
assert "| dy | 1 | 0 | 10 | 2 |" in markdown_path.read_text(encoding="utf-8-sig")
|
||||
|
||||
|
||||
def test_build_db_rows_expands_every_platform():
|
||||
parsed = {
|
||||
platform: {
|
||||
"metrics": {"sales": index, "visitors": 10 * index, "cart_users": index, "refund_orders": 0},
|
||||
"matched_ids": [platform], "missing_ids": [], "matched_row_count": 1, "source_file": f"{platform}.xlsx",
|
||||
}
|
||||
for index, platform in enumerate(("jd", "dy", "tm"), start=1)
|
||||
}
|
||||
payload = backfill.build_final_payload("2026-07-16", parsed, ["10515"])
|
||||
|
||||
rows = backfill.build_db_rows([payload])
|
||||
|
||||
assert len(rows) == 3
|
||||
assert {(row["metric_date"], row["platform"]) for row in rows} == {
|
||||
("2026-07-16", "jd"),
|
||||
("2026-07-16", "dy"),
|
||||
("2026-07-16", "tm"),
|
||||
}
|
||||
assert next(row for row in rows if row["platform"] == "tm")["sales"] == 3
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import collect_erp_yesterday_metrics as erp
|
||||
import collect_jd_persona_to_bitable as jd_persona
|
||||
from runtime_paths import playwright_launch_options
|
||||
|
||||
|
||||
class _Chromium:
|
||||
def __init__(self) -> None:
|
||||
self.kwargs = None
|
||||
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
return object()
|
||||
|
||||
|
||||
def test_jd_persona_launches_the_managed_persistent_profile(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = tmp_path / "jd-profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
chromium = _Chromium()
|
||||
|
||||
context = jd_persona._launch_jd_context(chromium, headless=True)
|
||||
|
||||
assert context is not None
|
||||
assert chromium.kwargs["user_data_dir"] == str(profile.resolve())
|
||||
assert chromium.kwargs["headless"] is True
|
||||
|
||||
|
||||
def test_jd_persona_reuses_logged_in_page_without_credentials() -> None:
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
class Page:
|
||||
url = "https://shop.jd.com/jdm/home"
|
||||
|
||||
def goto(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
def wait_for_timeout(self, _milliseconds):
|
||||
return None
|
||||
|
||||
def locator(self, _selector):
|
||||
return SimpleNamespace(
|
||||
inner_text=lambda **_kwargs: "京东商家后台 商品明细",
|
||||
)
|
||||
|
||||
jd_persona._ensure_jd_session(
|
||||
Page(),
|
||||
shop="",
|
||||
password="",
|
||||
step_login=lambda _page, shop, password: calls.append((shop, password)),
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_erp_uses_managed_profile_when_bound_cdp_is_not_running(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = tmp_path / "erp-profile"
|
||||
monkeypatch.setattr(erp, "_cdp_is_listening", lambda _url: False)
|
||||
|
||||
cdp_url, user_data_dir = erp.resolve_browser_session(
|
||||
{"cdp_url": "http://127.0.0.1:9222"},
|
||||
{
|
||||
"GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22063",
|
||||
"GYXX_BROWSER_PROFILE_DIR": str(profile),
|
||||
},
|
||||
)
|
||||
|
||||
assert cdp_url is None
|
||||
assert user_data_dir == str(profile)
|
||||
|
||||
|
||||
def test_erp_attaches_only_to_the_live_managed_cdp(monkeypatch) -> None:
|
||||
monkeypatch.setattr(erp, "_cdp_is_listening", lambda _url: True)
|
||||
|
||||
cdp_url, user_data_dir = erp.resolve_browser_session(
|
||||
{"cdp_url": "http://127.0.0.1:9222", "user_data_dir": "legacy"},
|
||||
{
|
||||
"GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22063",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "managed",
|
||||
},
|
||||
)
|
||||
|
||||
assert cdp_url == "http://127.0.0.1:22063"
|
||||
assert user_data_dir is None
|
||||
|
||||
|
||||
def test_erp_main_fails_before_browser_when_no_style_is_collectable(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
target_date=date(2026, 8, 3),
|
||||
mode="daily",
|
||||
refresh_config=False,
|
||||
input="unused-styles.json",
|
||||
)
|
||||
monkeypatch.setattr(erp, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_config",
|
||||
lambda _path: {"login": {"url": "https://erp.test", "username": "placeholder", "password": "placeholder"}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_erp_styles",
|
||||
lambda *_args: ([], {"source": "lark", "collectable": [], "skipped": []}),
|
||||
)
|
||||
monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None)
|
||||
monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
erp.DynamicFetcher,
|
||||
"fetch",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("browser must not start")
|
||||
),
|
||||
)
|
||||
|
||||
assert erp.main() == 1
|
||||
|
||||
|
||||
def test_erp_main_fails_when_collector_summary_contains_failures(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
config="unused.json",
|
||||
target_date=date(2026, 8, 3),
|
||||
mode="daily",
|
||||
refresh_config=False,
|
||||
input="unused-styles.json",
|
||||
data_dir=str(tmp_path),
|
||||
platform="all",
|
||||
no_cdp=False,
|
||||
timeout_ms=1000,
|
||||
limit=0,
|
||||
report_timeout=1,
|
||||
report_attempts=1,
|
||||
include_slow_codes=False,
|
||||
hold_seconds=0,
|
||||
headless=True,
|
||||
)
|
||||
styles = [{"style_name": "style-a", "erp_style_codes": ["erp-a"], "brand": erp.DEFAULT_BRAND}]
|
||||
monkeypatch.setattr(erp, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_config",
|
||||
lambda _path: {"login": {"url": "https://erp.test", "username": "placeholder", "password": "placeholder"}},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"load_erp_styles",
|
||||
lambda *_args: (styles, {"source": "lark", "collectable": ["style-a"], "skipped": []}),
|
||||
)
|
||||
monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None)
|
||||
monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None)
|
||||
monkeypatch.setattr(erp, "managed_data_path", lambda _path: tmp_path)
|
||||
monkeypatch.setattr(erp, "resolve_browser_session", lambda _config: (None, None))
|
||||
|
||||
def fake_login_action(**kwargs):
|
||||
state = kwargs["state"]
|
||||
|
||||
def action(_page):
|
||||
state["ok"] = True
|
||||
|
||||
return action
|
||||
|
||||
monkeypatch.setattr(erp, "make_page_action", fake_login_action)
|
||||
monkeypatch.setattr(
|
||||
erp,
|
||||
"collect_in_page",
|
||||
lambda *_args, **_kwargs: {
|
||||
"reports": 1,
|
||||
"valid_codes": 1,
|
||||
"has_failures": True,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
erp.DynamicFetcher,
|
||||
"fetch",
|
||||
lambda _url, **kwargs: kwargs["page_action"](object()),
|
||||
)
|
||||
|
||||
assert erp.main() == 1
|
||||
|
||||
|
||||
def test_erp_date_inputs_are_required_and_read_back() -> None:
|
||||
source = inspect.getsource(erp.set_platform_and_code)
|
||||
|
||||
assert "const begin = await waitFor('#order_date_begin')" in source
|
||||
assert "const end = await waitFor('#order_date_end')" in source
|
||||
assert "date range did not persist" in source
|
||||
|
||||
|
||||
def test_playwright_channel_is_optional_and_environment_driven() -> None:
|
||||
assert playwright_launch_options({}) == {}
|
||||
assert playwright_launch_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == {
|
||||
"channel": "chrome"
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import insert_bitable_records as insert
|
||||
import pytest
|
||||
|
||||
|
||||
def test_record_id_parser_accepts_nested_upsert_receipt() -> None:
|
||||
envelope = {
|
||||
"ok": True,
|
||||
"data": {"record": {"record_id_list": ["rec-confirmed"]}},
|
||||
}
|
||||
|
||||
assert insert._record_id_from_envelope(envelope) == "rec-confirmed"
|
||||
|
||||
|
||||
def test_lark_upsert_rejects_success_without_record_id(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
monkeypatch.setattr(insert.shutil, "which", lambda _name: "lark-cli.cmd")
|
||||
monkeypatch.setattr(
|
||||
insert.subprocess,
|
||||
"run",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout='{"ok": true, "data": {}}',
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="without a confirmed record_id"):
|
||||
insert.run_lark_upsert("base", "table", {"field": "value"})
|
||||
|
||||
|
||||
def _daily_record(style: str, source_id: str) -> dict:
|
||||
return {
|
||||
"数据日期": "2026-08-03",
|
||||
"平台": "京东",
|
||||
"平台代码": "jd",
|
||||
"款式": style,
|
||||
"SourceID": source_id,
|
||||
"metric": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_any_feishu_insert_failure_makes_whole_command_fail(monkeypatch) -> None:
|
||||
style_map = {
|
||||
style: {
|
||||
"base_token": "base",
|
||||
"table_id": "table",
|
||||
"enabled": True,
|
||||
"field_map": {"metric": "metric", "SourceID": "SourceID"},
|
||||
}
|
||||
for style in ("style-a", "style-b")
|
||||
}
|
||||
monkeypatch.setattr(insert, "get_bitable_style_map", lambda: style_map)
|
||||
monkeypatch.setattr(
|
||||
insert,
|
||||
"load_records",
|
||||
lambda *_args, **_kwargs: [
|
||||
_daily_record("style-a", "source-a"),
|
||||
_daily_record("style-b", "source-b"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(insert, "_find_record_id_by_source_id", lambda *_args, **_kwargs: None)
|
||||
|
||||
def upsert(_base, _table, fields, **_kwargs):
|
||||
if fields["SourceID"] == "source-b":
|
||||
raise RuntimeError("write failed")
|
||||
return "rec-a"
|
||||
|
||||
monkeypatch.setattr(insert, "run_lark_upsert", upsert)
|
||||
|
||||
assert insert.main(["--date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_empty_feishu_export_is_not_success(monkeypatch) -> None:
|
||||
monkeypatch.setattr(insert, "get_bitable_style_map", lambda: {"style-a": {}})
|
||||
monkeypatch.setattr(insert, "load_records", lambda *_args, **_kwargs: [])
|
||||
|
||||
assert insert.main(["--date", "2026-08-03"]) == 1
|
||||
@@ -0,0 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import aggregate_daily_final as aggregate
|
||||
import export_bitable_records as export
|
||||
|
||||
|
||||
def test_daily_aggregate_and_export_share_curated_summary_root() -> None:
|
||||
assert export.SUMMARY_ROOT == aggregate.CURATED_DATA_ROOT
|
||||
|
||||
|
||||
def test_daily_export_keeps_platform_details_in_raw_root() -> None:
|
||||
assert export.PLATFORM_ROOT == export.DATA_ROOT
|
||||
assert export.PLATFORM_ROOT != export.SUMMARY_ROOT
|
||||
@@ -0,0 +1,428 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import aggregate_daily_final as aggregate
|
||||
import db as product_db
|
||||
import import_product_daily as product_import
|
||||
import orchestrate_daily_collection as daily
|
||||
import pytest
|
||||
from commands import import_daily as import_daily_command
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __enter__(self):
|
||||
return self.value
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
class _Cursor:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.execute_args = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def execute(self, *args):
|
||||
self.execute_args = args
|
||||
|
||||
def fetchone(self):
|
||||
return self.rows[0] if self.rows else None
|
||||
|
||||
def fetchall(self):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class _Connection:
|
||||
def __init__(self, rows):
|
||||
self.cursor_instance = _Cursor(rows)
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def _record() -> dict:
|
||||
return {
|
||||
"platform": "dy",
|
||||
"stat_date": datetime(2026, 8, 3).date(),
|
||||
"product_id": "product-1",
|
||||
"source_file": "report.xlsx",
|
||||
}
|
||||
|
||||
|
||||
def _patch_valid_import(monkeypatch, *, upserted: int = 1) -> None:
|
||||
target_dir = Path("2026-08-03")
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
|
||||
monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: [Path("report.xlsx")])
|
||||
monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: [_record()])
|
||||
monkeypatch.setattr(product_import, "get_conn", lambda: _Context(object()))
|
||||
monkeypatch.setattr(
|
||||
product_import,
|
||||
"upsert_product_daily_metrics",
|
||||
lambda _conn, _records: upserted,
|
||||
)
|
||||
|
||||
|
||||
def test_selected_platform_collector_failure_stops_daily_aggregation(monkeypatch) -> None:
|
||||
results = iter(
|
||||
[
|
||||
{"name": "jd", "code": 0},
|
||||
{"name": "dy", "code": 1, "error": "collector failed"},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(daily, "run_step_with_retry", lambda *_args, **_kwargs: next(results))
|
||||
monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1)
|
||||
|
||||
with pytest.raises(RuntimeError, match="required platform collector.*dy"):
|
||||
daily.run_platforms(
|
||||
parallel=False,
|
||||
platforms=["jd", "dy"],
|
||||
target_date="2026-08-03",
|
||||
)
|
||||
|
||||
def test_platform_selection_keeps_omitted_collectors_optional(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_run(name, *_args, **_kwargs):
|
||||
calls.append(name)
|
||||
return {"name": name, "code": 0}
|
||||
|
||||
monkeypatch.setattr(daily, "run_step_with_retry", fake_run)
|
||||
monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1)
|
||||
|
||||
results = daily.run_platforms(
|
||||
parallel=False,
|
||||
platforms=["jd"],
|
||||
target_date="2026-08-03",
|
||||
)
|
||||
|
||||
assert calls == ["jd"]
|
||||
assert results == [{"name": "jd", "code": 0}]
|
||||
|
||||
|
||||
def test_successful_collector_exit_still_requires_fresh_artifact(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *_args, **_kwargs: {"name": "jd", "code": 0},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"verify_platform_artifacts",
|
||||
lambda *_args: (_ for _ in ()).throw(RuntimeError("missing-or-stale")),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing-or-stale"):
|
||||
daily.run_platforms(
|
||||
parallel=False,
|
||||
platforms=["jd"],
|
||||
target_date="2026-08-03",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "payload"),
|
||||
[
|
||||
(
|
||||
"jd",
|
||||
{
|
||||
"采集日期": "2026-08-03",
|
||||
"matched_row_count": 0,
|
||||
"download_file": "jd.xlsx",
|
||||
},
|
||||
),
|
||||
(
|
||||
"dy",
|
||||
{
|
||||
"date_range": "2026-08-03 ~ 2026-08-03",
|
||||
"matched_row_count": 0,
|
||||
"download_file": "dy.xlsx",
|
||||
},
|
||||
),
|
||||
(
|
||||
"tm",
|
||||
{
|
||||
"date": "2026-08-03",
|
||||
"source_file": "tm.xlsx",
|
||||
"style": {"匹配商品ID数": 0},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_zero_matched_product_is_not_a_valid_artifact(platform, payload) -> None:
|
||||
assert daily._artifact_has_valid_product(platform, payload, "2026-08-03") is False
|
||||
|
||||
|
||||
def test_platform_artifact_verifier_rejects_missing_configured_style(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {"jd": {"collectable": ["style-a", "style-b"], "skipped": []}}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
path = daily._platform_artifact_path("jd", "style-a", "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"采集日期": "2026-08-03",
|
||||
"matched_row_count": 1,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="style-b:missing-or-stale"):
|
||||
daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns)
|
||||
|
||||
|
||||
def test_platform_artifact_verifier_accepts_fresh_valid_product(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
class Loader:
|
||||
def get_daily_styles(self):
|
||||
return {"jd": {"collectable": ["style-a"], "skipped": []}}
|
||||
|
||||
monkeypatch.setattr(daily, "StyleConfigLoader", Loader)
|
||||
monkeypatch.setattr(daily, "DATA_ROOT", tmp_path)
|
||||
marker_ns = time.time_ns()
|
||||
path = daily._platform_artifact_path("jd", "style-a", "2026-08-03")
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(
|
||||
json.dumps({
|
||||
"采集日期": "2026-08-03",
|
||||
"matched_row_count": 2,
|
||||
"download_file": "_downloads/2026-08-03/report.xlsx",
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) == 1
|
||||
|
||||
|
||||
def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None:
|
||||
args = SimpleNamespace(
|
||||
skip_erp=True,
|
||||
skip_platforms=False,
|
||||
serial_platforms=False,
|
||||
dry_run=False,
|
||||
skip_reviews_import=False,
|
||||
)
|
||||
monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: [])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1},
|
||||
)
|
||||
|
||||
daily.run_collect(args, "2026-08-03", ["jd"], failed=None)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0][0] == "import_product_reviews"
|
||||
assert calls[0][1]["critical"] is False
|
||||
|
||||
|
||||
def test_formal_aggregation_fails_when_postgres_write_is_not_fresh(monkeypatch) -> None:
|
||||
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
|
||||
args = SimpleNamespace(dry_run=False)
|
||||
monkeypatch.setattr(daily, "database_write_marker", lambda: marker)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *_args, **_kwargs: {"name": "aggregate_daily_final", "code": 0},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"verify_daily_metrics_write",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
RuntimeError("daily aggregation produced no fresh PostgreSQL rows")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="no fresh PostgreSQL rows"):
|
||||
daily.run_analyze(args, "2026-08-03", failed=None)
|
||||
|
||||
|
||||
def test_dry_run_aggregation_does_not_require_postgres(monkeypatch) -> None:
|
||||
args = SimpleNamespace(dry_run=True)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"database_write_marker",
|
||||
lambda: pytest.fail("dry-run must not connect to PostgreSQL"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"verify_daily_metrics_write",
|
||||
lambda *_args: pytest.fail("dry-run must not verify PostgreSQL"),
|
||||
)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
daily,
|
||||
"run_step_with_retry",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0},
|
||||
)
|
||||
|
||||
daily.run_analyze(args, "2026-08-03", failed=None)
|
||||
|
||||
assert calls[0][1]["dry_run"] is True
|
||||
|
||||
|
||||
def test_aggregate_command_rejects_zero_style_outputs(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setattr(aggregate, "collect_style_names", lambda *_args: [])
|
||||
monkeypatch.setattr(aggregate, "managed_data_path", lambda _path: tmp_path)
|
||||
|
||||
assert aggregate.main([
|
||||
"--date",
|
||||
"2026-08-03",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
]) == 1
|
||||
|
||||
|
||||
def test_aggregate_command_rejects_partial_postgres_write(monkeypatch, tmp_path) -> None:
|
||||
payload = {
|
||||
"date": "2026-08-03",
|
||||
"style_name": "style-a",
|
||||
"erp": {"style_codes": ["erp-a"]},
|
||||
"jd": {
|
||||
"sales": 1,
|
||||
"refund_orders": 0,
|
||||
"visitors": 2,
|
||||
"cart_users": 1,
|
||||
"source": "jd.json",
|
||||
},
|
||||
"dy": {},
|
||||
"tm": {},
|
||||
}
|
||||
monkeypatch.setattr(aggregate, "collect_style_names", lambda *_args: ["style-a"])
|
||||
monkeypatch.setattr(aggregate, "managed_data_path", lambda _path: tmp_path)
|
||||
monkeypatch.setattr(aggregate, "build_payload", lambda *_args: payload)
|
||||
monkeypatch.setattr(aggregate, "write_outputs", lambda *_args: (tmp_path / "a", tmp_path / "b"))
|
||||
monkeypatch.setattr(product_db, "get_conn", lambda: _Context(object()))
|
||||
monkeypatch.setattr(product_db, "upsert_daily_metrics", lambda *_args: 0)
|
||||
|
||||
assert aggregate.main([
|
||||
"--date",
|
||||
"2026-08-03",
|
||||
"--data-root",
|
||||
str(tmp_path),
|
||||
]) == 1
|
||||
|
||||
|
||||
def test_postgres_verifier_rejects_stale_or_missing_rows(monkeypatch) -> None:
|
||||
connection = _Connection([])
|
||||
monkeypatch.setattr(daily, "get_conn", lambda: _Context(connection))
|
||||
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing fresh PostgreSQL rows"):
|
||||
daily.verify_daily_metrics_write(
|
||||
"2026-08-03",
|
||||
marker,
|
||||
required_platforms=["jd"],
|
||||
expected_pairs={("jd", "style-a")},
|
||||
)
|
||||
|
||||
assert connection.cursor_instance.execute_args[1] == (
|
||||
"2026-08-03",
|
||||
marker,
|
||||
["jd"],
|
||||
)
|
||||
|
||||
|
||||
def test_postgres_verifier_requires_every_expected_platform_style(monkeypatch) -> None:
|
||||
connection = _Connection([("jd", "style-a"), ("dy", "style-a")])
|
||||
monkeypatch.setattr(daily, "get_conn", lambda: _Context(connection))
|
||||
marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc)
|
||||
|
||||
with pytest.raises(RuntimeError, match="tm/style-a"):
|
||||
daily.verify_daily_metrics_write(
|
||||
"2026-08-03",
|
||||
marker,
|
||||
required_platforms=["jd", "dy", "tm"],
|
||||
expected_pairs={
|
||||
("jd", "style-a"),
|
||||
("dy", "style-a"),
|
||||
("tm", "style-a"),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_import_fails_when_selected_platform_has_no_target_directory(monkeypatch, capsys) -> None:
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [])
|
||||
|
||||
code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"])
|
||||
|
||||
assert code == 2
|
||||
assert "dy: 没有匹配所选日期范围的报表目录" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_import_fails_when_target_directory_has_no_input_file(monkeypatch, capsys) -> None:
|
||||
target_dir = Path("2026-08-03")
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
|
||||
monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: [])
|
||||
monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: [])
|
||||
|
||||
code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"])
|
||||
|
||||
assert code == 2
|
||||
assert "目标目录中没有可导入的报表文件" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_import_fails_when_input_has_no_valid_rows(monkeypatch, capsys) -> None:
|
||||
target_dir = Path("2026-08-03")
|
||||
monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir])
|
||||
monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: [Path("report.xlsx")])
|
||||
monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: [])
|
||||
|
||||
code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"])
|
||||
|
||||
assert code == 2
|
||||
assert "报表中没有有效商品明细行" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_import_fails_when_postgres_reports_zero_written_rows(monkeypatch, capsys) -> None:
|
||||
_patch_valid_import(monkeypatch, upserted=0)
|
||||
|
||||
code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"])
|
||||
|
||||
assert code == 2
|
||||
assert "PostgreSQL 未写入任何商品明细行" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_valid_selected_platform_input_still_imports_successfully(monkeypatch) -> None:
|
||||
_patch_valid_import(monkeypatch, upserted=1)
|
||||
|
||||
code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"])
|
||||
|
||||
assert code == 0
|
||||
|
||||
|
||||
def test_import_command_passes_target_date_and_propagates_failure(monkeypatch) -> None:
|
||||
monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-04")
|
||||
invoked = []
|
||||
monkeypatch.setattr(
|
||||
import_daily_command,
|
||||
"import_main",
|
||||
lambda argv: invoked.append(argv) or 2,
|
||||
)
|
||||
|
||||
assert import_daily_command.main() == 2
|
||||
assert invoked == [["--date", "2026-08-03"]]
|
||||
@@ -0,0 +1,42 @@
|
||||
import unittest
|
||||
|
||||
from db import build_pg_config
|
||||
|
||||
|
||||
class DatabaseConfigTests(unittest.TestCase):
|
||||
def test_complete_config_is_parsed(self):
|
||||
config = build_pg_config(
|
||||
{
|
||||
"PG_HOST": "db.example.test",
|
||||
"PG_PORT": "5544",
|
||||
"PG_DB": "warehouse",
|
||||
"PG_USER": "collector",
|
||||
"PG_PASSWORD": "secret",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
config,
|
||||
{
|
||||
"host": "db.example.test",
|
||||
"port": 5544,
|
||||
"dbname": "warehouse",
|
||||
"user": "collector",
|
||||
"password": "secret",
|
||||
},
|
||||
)
|
||||
|
||||
def test_missing_final_config_is_rejected(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "PG_HOST"):
|
||||
build_pg_config(
|
||||
{
|
||||
"PG_PORT": "5432",
|
||||
"PG_DB": "warehouse",
|
||||
"PG_USER": "collector",
|
||||
"PG_PASSWORD": "secret",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,286 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from collect_dy_market_rank import (
|
||||
TARGET_DY_MARKET_CATEGORIES,
|
||||
append_feishu_category_table,
|
||||
build_dy_category_table_xml,
|
||||
build_dy_feishu_batch_xml,
|
||||
build_dy_feishu_intro_xml,
|
||||
is_excluded_product_title,
|
||||
navigate_to_product_rank,
|
||||
parse_market_product_rank_html,
|
||||
parse_money_range_upper,
|
||||
payment_metric_header_index,
|
||||
require_payment_metric_header,
|
||||
)
|
||||
from collect_sycm_market_rank import build_feishu_batch_xml
|
||||
|
||||
SAMPLE_HTML = """
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>排名</th><th>商品</th><th>店铺名称</th>
|
||||
<th>用户支付金额</th><th>点击次数</th><th>成交件数</th><th>操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr aria-hidden="true"><td>排名</td><td>商品</td><td></td><td></td></tr>
|
||||
<tr class="aurora-table-row" data-row-key="3826298175627592082_1">
|
||||
<td><div class="tagItem">首次上榜</div></td>
|
||||
<td>
|
||||
<img src="https://p9-aio.ecombdimg.com/product.jpg"/>
|
||||
<div class="name-Gh2R04">测试旅行拉杆箱</div>
|
||||
<div class="price-HH6TP1">价格带 <span>¥1889-¥2289</span></div>
|
||||
</td>
|
||||
<td>测试旗舰店</td>
|
||||
<td><div>¥2,500万</div>-<div>¥5,000万</div></td>
|
||||
<td>250万-500万</td><td>10万-25万</td><td>查看详情</td>
|
||||
</tr>
|
||||
<tr class="aurora-table-row" data-row-key="3826298175627592083_3">
|
||||
<td><div>↑4</div></td>
|
||||
<td>
|
||||
<img src="https://p9-aio.ecombdimg.com/product-3.jpg"/>
|
||||
<div elementtiming="pccp_element">第三名旅行箱</div>
|
||||
<div>价格带 <span>¥999</span></div>
|
||||
</td>
|
||||
<td>第三名旗舰店</td>
|
||||
<td><div>¥750万</div>-<div>¥1,000万</div></td>
|
||||
<td>10万</td><td>1万</td><td>查看详情</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
class DyMarketRankTests(unittest.TestCase):
|
||||
def test_title_blacklist_filters_children_primary_and_middle_school(self):
|
||||
self.assertTrue(is_excluded_product_title("儿童轻便双肩包"))
|
||||
self.assertTrue(is_excluded_product_title("小学生护脊书包"))
|
||||
self.assertTrue(is_excluded_product_title("初中生大容量背包"))
|
||||
self.assertTrue(
|
||||
is_excluded_product_title(
|
||||
"TigerFamily【老爸抽检】学生生日礼物1-3年级男女护脊减负折叠书包"
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
is_excluded_product_title(
|
||||
"【官方旗舰】AIRJORDAN耐克青少年运动休闲男女大号双肩书包9300"
|
||||
)
|
||||
)
|
||||
self.assertTrue(is_excluded_product_title("老人轻便防水随身包"))
|
||||
self.assertTrue(is_excluded_product_title("高中生大容量双肩包"))
|
||||
self.assertFalse(is_excluded_product_title("成人通勤双肩包"))
|
||||
|
||||
def test_default_categories_match_requested_eight_filters(self):
|
||||
specs = {item["name"]: item for item in TARGET_DY_MARKET_CATEGORIES}
|
||||
self.assertEqual(
|
||||
set(specs),
|
||||
{"运动包", "旅行箱", "托特包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"},
|
||||
)
|
||||
self.assertEqual((specs["运动包"]["min_price"], specs["运动包"]["max_price"]), (200, 1000))
|
||||
self.assertEqual((specs["旅行箱"]["min_price"], specs["旅行箱"]["max_price"]), (400, 2000))
|
||||
self.assertEqual((specs["托特包"]["min_price"], specs["托特包"]["max_price"]), (200, 3000))
|
||||
self.assertEqual(specs["托特包"]["search_keyword"], "托特包")
|
||||
self.assertEqual((specs["手机包"]["min_price"], specs["手机包"]["max_price"]), (100, 1000))
|
||||
self.assertEqual(specs["手机包"]["paths"][0][-1], "耳机包")
|
||||
self.assertEqual((specs["电脑包"]["min_price"], specs["电脑包"]["max_price"]), (200, 1000))
|
||||
self.assertEqual(specs["电脑包"]["search_keyword"], "电脑包")
|
||||
self.assertEqual(specs["电脑包"]["paths"][0][-2:], ("功能箱包", "全部"))
|
||||
|
||||
def test_parse_money_range_upper(self):
|
||||
self.assertEqual(parse_money_range_upper("¥2,500万-¥5,000万"), 50_000_000)
|
||||
self.assertEqual(parse_money_range_upper("¥750万"), 7_500_000)
|
||||
|
||||
def test_payment_metric_header_uses_exact_supported_aliases(self):
|
||||
self.assertEqual(
|
||||
payment_metric_header_index(["排名", "支付金额（元）", "成交人数"]),
|
||||
1,
|
||||
)
|
||||
self.assertEqual(
|
||||
payment_metric_header_index(["排名", "成交金额", "成交人数"]),
|
||||
1,
|
||||
)
|
||||
self.assertIsNone(
|
||||
payment_metric_header_index(["排名", "用户支付金额同比", "成交人数"])
|
||||
)
|
||||
|
||||
def test_missing_payment_metric_reports_current_headers(self):
|
||||
page = MagicMock()
|
||||
page.locator.return_value.all_inner_texts.return_value = [
|
||||
"排名",
|
||||
"商品",
|
||||
"成交人数",
|
||||
]
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
RuntimeError,
|
||||
"允许的精确候选.*当前表头.*成交人数",
|
||||
):
|
||||
require_payment_metric_header(page)
|
||||
|
||||
def test_login_probe_can_open_rank_page_without_waiting_for_metrics(self):
|
||||
page = MagicMock()
|
||||
|
||||
with patch("collect_dy_market_rank.wait_for_product_rank_ready") as ready:
|
||||
navigate_to_product_rank(page, wait_until_ready=False)
|
||||
|
||||
page.goto.assert_called_once()
|
||||
ready.assert_not_called()
|
||||
|
||||
def test_parser_accepts_supported_payment_metric_alias(self):
|
||||
rows = parse_market_product_rank_html(
|
||||
SAMPLE_HTML.replace("用户支付金额", "支付金额（元）"),
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)
|
||||
|
||||
self.assertEqual(len(rows), 2)
|
||||
|
||||
def test_parse_rank_html_extracts_requested_fields(self):
|
||||
rows = parse_market_product_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)
|
||||
|
||||
self.assertEqual(len(rows), 2)
|
||||
row = rows[0]
|
||||
self.assertEqual(row["rank"], 1)
|
||||
self.assertEqual(row["product_id"], "3826298175627592082")
|
||||
self.assertEqual(row["name"], "测试旅行拉杆箱")
|
||||
self.assertEqual(row["image_url"], "https://p9-aio.ecombdimg.com/product.jpg")
|
||||
self.assertEqual(row["price_range"], "¥1889-¥2289")
|
||||
self.assertEqual(row["sales_amount_range"], "¥2,500万-¥5,000万")
|
||||
self.assertEqual(row["sales_amount_upper"], 50_000_000)
|
||||
self.assertEqual(
|
||||
row["product_url"],
|
||||
"https://haohuo.jinritemai.com/views/product/item2?id=3826298175627592082",
|
||||
)
|
||||
self.assertEqual(rows[1]["rank"], 3)
|
||||
|
||||
def test_parse_rank_html_drops_blacklisted_title(self):
|
||||
rows = parse_market_product_rank_html(
|
||||
SAMPLE_HTML.replace("测试旅行拉杆箱", "小学生测试旅行拉杆箱"),
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)
|
||||
|
||||
self.assertEqual([row["name"] for row in rows], ["第三名旅行箱"])
|
||||
|
||||
def test_dy_feishu_xml_contains_sales_image_and_link(self):
|
||||
product = parse_market_product_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)[0]
|
||||
|
||||
xml = build_dy_feishu_batch_xml([product], start_rank=1)
|
||||
|
||||
self.assertIn(">销售额</th>", xml)
|
||||
self.assertIn("¥2,500万-¥5,000万", xml)
|
||||
self.assertIn('<img href="https://p9-aio.ecombdimg.com/product.jpg"', xml)
|
||||
self.assertIn("https://haohuo.jinritemai.com/views/product/item2?id=3826298175627592082", xml)
|
||||
|
||||
def test_category_output_is_one_table_per_category(self):
|
||||
product = parse_market_product_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)[0]
|
||||
|
||||
xml = build_dy_category_table_xml("旅行箱", [product, dict(product, rank=2)])
|
||||
|
||||
self.assertEqual(xml.count("<table>"), 1)
|
||||
self.assertEqual(xml.count("<th "), 7)
|
||||
self.assertIn("<h2>旅行箱（2条）</h2>", xml)
|
||||
self.assertIn("<td>1</td>", xml)
|
||||
self.assertIn("<td>2</td>", xml)
|
||||
|
||||
def test_category_upload_timeout_is_verified_before_continuing(self):
|
||||
product = parse_market_product_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)[0]
|
||||
with (
|
||||
patch(
|
||||
"collect_dy_market_rank.append_feishu_doc",
|
||||
side_effect=RuntimeError("server time out error"),
|
||||
),
|
||||
patch(
|
||||
"collect_dy_market_rank._feishu_doc_contains_keyword",
|
||||
side_effect=[False, True, True],
|
||||
) as verify,
|
||||
):
|
||||
append_feishu_category_table(
|
||||
"https://example.feishu.cn/docx/test",
|
||||
"旅行箱",
|
||||
[product],
|
||||
2,
|
||||
)
|
||||
|
||||
self.assertEqual(verify.call_count, 3)
|
||||
|
||||
def test_category_upload_skips_complete_existing_table(self):
|
||||
product = parse_market_product_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱",
|
||||
)[0]
|
||||
with (
|
||||
patch("collect_dy_market_rank.append_feishu_doc") as append,
|
||||
patch(
|
||||
"collect_dy_market_rank._feishu_doc_contains_keyword",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
append_feishu_category_table(
|
||||
"https://example.feishu.cn/docx/test",
|
||||
"旅行箱",
|
||||
[product],
|
||||
2,
|
||||
)
|
||||
|
||||
append.assert_not_called()
|
||||
|
||||
def test_dy_feishu_intro_records_price_and_search_filters(self):
|
||||
xml = build_dy_feishu_intro_xml(
|
||||
title="抖音市场榜单",
|
||||
period="近30天",
|
||||
products=[
|
||||
{
|
||||
"category_name": "托特包",
|
||||
"filter_min_price": 200,
|
||||
"filter_max_price": 3000,
|
||||
"search_keyword": "托特包",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
self.assertIn("托特包 ¥200-¥3000（搜索：托特包）", xml)
|
||||
self.assertIn("标题过滤", xml)
|
||||
for keyword in ("儿童", "学生", "青少年", "老人", "中学", "书包"):
|
||||
self.assertIn(keyword, xml)
|
||||
|
||||
def test_tmall_feishu_table_also_has_sales_column(self):
|
||||
xml = build_feishu_batch_xml(
|
||||
[{
|
||||
"category_name": "旅行箱",
|
||||
"rank": 1,
|
||||
"name": "测试商品",
|
||||
"buyer_range": "2500~5000",
|
||||
"buyer_max": 5000,
|
||||
"price": "¥418",
|
||||
"product_url": "https://item.taobao.com/item.htm?id=1",
|
||||
"image_url": "https://img.alicdn.com/test.jpg",
|
||||
}],
|
||||
start_rank=1,
|
||||
)
|
||||
|
||||
self.assertIn(">销售额</th>", xml)
|
||||
self.assertIn(">未获取</td>", xml)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,89 @@
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
VENDOR_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
/ "vendors"
|
||||
/ "dy-data-flow"
|
||||
)
|
||||
sys.path.insert(0, str(VENDOR_ROOT))
|
||||
|
||||
from dy_store_competitor_store_scraping import ( # noqa: E402
|
||||
_check_login_status,
|
||||
_choose_session_root,
|
||||
_looks_like_login_url,
|
||||
)
|
||||
|
||||
|
||||
def _write_cookies(root: Path, domains: list[str]) -> None:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
cookies = [{"name": f"c{index}", "value": "x", "domain": domain, "path": "/"}
|
||||
for index, domain in enumerate(domains)]
|
||||
(root / "compass_cookies.json").write_text(
|
||||
json.dumps(cookies), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_choose_session_root_prefers_complete_sso_bundle(tmp_path):
|
||||
project = tmp_path / "project"
|
||||
legacy = tmp_path / "legacy"
|
||||
_write_cookies(project, [".compass.jinritemai.com"])
|
||||
_write_cookies(
|
||||
legacy,
|
||||
[
|
||||
".compass.jinritemai.com",
|
||||
".doudian-sso.jinritemai.com",
|
||||
".fxg.jinritemai.com",
|
||||
],
|
||||
)
|
||||
|
||||
assert _choose_session_root([project, legacy]) == legacy
|
||||
|
||||
|
||||
def test_choose_session_root_honors_explicit_root(tmp_path):
|
||||
project = tmp_path / "project"
|
||||
explicit = tmp_path / "explicit"
|
||||
_write_cookies(project, [".compass.jinritemai.com", ".fxg.jinritemai.com"])
|
||||
|
||||
assert _choose_session_root([project], explicit=explicit) == explicit
|
||||
|
||||
|
||||
def test_login_redirect_url_detection():
|
||||
assert _looks_like_login_url("https://compass.jinritemai.com/login?redirect=x")
|
||||
assert _looks_like_login_url("https://doudian-sso.jinritemai.com/login")
|
||||
assert _looks_like_login_url("https://fxg.jinritemai.com/passport/sso")
|
||||
assert not _looks_like_login_url(
|
||||
"https://compass.jinritemai.com/shop/commodity/product-list"
|
||||
)
|
||||
|
||||
|
||||
class _EmptyLocator:
|
||||
def count(self):
|
||||
return 0
|
||||
|
||||
|
||||
class _FakeContext:
|
||||
pages = []
|
||||
|
||||
|
||||
class _FakeShopPage:
|
||||
url = "https://compass.jinritemai.com/shop/commodity/product-list"
|
||||
context = _FakeContext()
|
||||
|
||||
def __init__(self):
|
||||
self.context.pages = [self]
|
||||
|
||||
def is_closed(self):
|
||||
return False
|
||||
|
||||
def locator(self, _selector):
|
||||
return _EmptyLocator()
|
||||
|
||||
|
||||
def test_shop_url_without_authenticated_dom_is_not_logged_in():
|
||||
assert not _check_login_status(_FakeShopPage())
|
||||
@@ -0,0 +1,92 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import erp_metric_overrides as overrides
|
||||
|
||||
|
||||
class ErpMetricOverridesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.TemporaryDirectory()
|
||||
self.data_root = Path(self.temp_dir.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.temp_dir.cleanup()
|
||||
|
||||
def _write_erp(self, code_results):
|
||||
data_dir = self.data_root / "dy" / "apollo_2026-07-22"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "data.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"style_name": "阿波罗x1",
|
||||
"erp_style_codes": ["10388"],
|
||||
"date_start": "2026-07-22",
|
||||
"yesterday_sales": 0,
|
||||
"yesterday_returns": 0,
|
||||
"code_results": code_results,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _apply(self, platform_sales, platform_refunds):
|
||||
with patch.object(overrides, "DATA_ROOT", self.data_root), patch.dict(
|
||||
overrides.PLATFORM_DIRS, {"dy": "dy"}
|
||||
):
|
||||
return overrides.apply_erp_override(
|
||||
"dy", "阿波罗x1", 394, 46,
|
||||
target_date="2026-07-22",
|
||||
platform_sales=platform_sales,
|
||||
platform_refunds=platform_refunds,
|
||||
)
|
||||
|
||||
def test_error_makes_ostensibly_ok_result_unavailable(self):
|
||||
self._write_erp([{
|
||||
"erp_style_code": "10388",
|
||||
"ok": True,
|
||||
"reason": "no_data",
|
||||
"error": "timeout waiting for report result: 10388 | allsku iframe not found",
|
||||
"yesterday_sales": 0,
|
||||
"yesterday_returns": 0,
|
||||
}])
|
||||
|
||||
result = self._apply(platform_sales=8, platform_refunds=2)
|
||||
|
||||
self.assertTrue(result["erp_unavailable"])
|
||||
self.assertEqual(result["销量"], 8)
|
||||
self.assertEqual(result["退款订单数"], 2)
|
||||
|
||||
def test_any_failed_code_disables_partial_erp_override(self):
|
||||
self._write_erp([
|
||||
{"erp_style_code": "1", "ok": True, "yesterday_sales": 3, "yesterday_returns": 1},
|
||||
{"erp_style_code": "2", "ok": False, "error": "timeout", "yesterday_sales": 0, "yesterday_returns": 0},
|
||||
])
|
||||
|
||||
result = self._apply(platform_sales=7, platform_refunds=2)
|
||||
|
||||
self.assertTrue(result["erp_unavailable"])
|
||||
self.assertEqual(result["销量"], 7)
|
||||
self.assertEqual(result["退款订单数"], 2)
|
||||
|
||||
def test_clean_no_data_is_a_reliable_zero(self):
|
||||
self._write_erp([{
|
||||
"erp_style_code": "10388",
|
||||
"ok": True,
|
||||
"reason": "no_data",
|
||||
"yesterday_sales": 0,
|
||||
"yesterday_returns": 0,
|
||||
}])
|
||||
|
||||
result = self._apply(platform_sales=8, platform_refunds=2)
|
||||
|
||||
self.assertFalse(result["erp_unavailable"])
|
||||
self.assertEqual(result["销量"], 0)
|
||||
self.assertEqual(result["退款订单数"], 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
import unittest
|
||||
|
||||
from collect_erp_yesterday_metrics import _no_data_report_is_fresh
|
||||
|
||||
|
||||
class ErpNoDataFreshnessTest(unittest.TestCase):
|
||||
def test_rejects_unchanged_no_data_page_from_previous_query(self):
|
||||
self.assertFalse(
|
||||
_no_data_report_is_fresh(
|
||||
previous_url="https://bi.erp321.com/allsku.aspx?r=old",
|
||||
current_url="https://bi.erp321.com/allsku.aspx?r=old",
|
||||
previous_body="NO_DATA",
|
||||
current_body="NO_DATA",
|
||||
reloaded=False,
|
||||
)
|
||||
)
|
||||
|
||||
def test_accepts_no_data_after_forced_reload(self):
|
||||
self.assertTrue(
|
||||
_no_data_report_is_fresh(
|
||||
previous_url="https://bi.erp321.com/allsku.aspx?r=old",
|
||||
current_url="https://bi.erp321.com/allsku.aspx?r=new",
|
||||
previous_body="NO_DATA",
|
||||
current_body="NO_DATA",
|
||||
reloaded=True,
|
||||
)
|
||||
)
|
||||
|
||||
def test_accepts_no_data_when_report_body_changed(self):
|
||||
self.assertTrue(
|
||||
_no_data_report_is_fresh(
|
||||
previous_url="https://bi.erp321.com/allsku.aspx?r=same",
|
||||
current_url="https://bi.erp321.com/allsku.aspx?r=same",
|
||||
previous_body="OLD REPORT",
|
||||
current_body="NO_DATA",
|
||||
reloaded=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,38 @@
|
||||
import ast
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
ERP_COLLECTOR = PROJECT_ROOT / "collect_erp_yesterday_metrics.py"
|
||||
|
||||
|
||||
def load_slow_skip_codes() -> set[str]:
|
||||
tree = ast.parse(ERP_COLLECTOR.read_text(encoding="utf-8"))
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
||||
if node.target.id == "ERP_SLOW_SKIP_CODES":
|
||||
if (
|
||||
isinstance(node.value, ast.Call)
|
||||
and isinstance(node.value.func, ast.Name)
|
||||
and node.value.func.id == "set"
|
||||
and not node.value.args
|
||||
and not node.value.keywords
|
||||
):
|
||||
return set()
|
||||
return set(ast.literal_eval(node.value))
|
||||
raise AssertionError("ERP_SLOW_SKIP_CODES not found")
|
||||
|
||||
|
||||
class ErpSlowSkipCodesTest(unittest.TestCase):
|
||||
def test_no_styles_are_permanently_skipped(self):
|
||||
self.assertEqual(load_slow_skip_codes(), set())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,52 @@
|
||||
import pytest
|
||||
from feishu_doc_native import (
|
||||
FeishuNativeDocClient,
|
||||
FeishuNativeDocError,
|
||||
append_native_category_table,
|
||||
parse_category_table_xml,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_category_table_xml_keeps_text_link_and_image():
|
||||
table = parse_category_table_xml(
|
||||
'<h2>旅行箱（1条）</h2><table><colgroup><col width="80"/>'
|
||||
'<col width="120"/></colgroup><thead><tr><th>商品图</th><th>商品</th></tr>'
|
||||
'<tbody><tr><td><img href="https://img.test/a.jpg"/></td>'
|
||||
'<td><a href="https://item.test/1">打开商品</a></td></tr></tbody></table>'
|
||||
)
|
||||
|
||||
assert table.heading == "旅行箱（1条）"
|
||||
assert table.widths == [80, 120]
|
||||
assert table.rows[1][0].image_url == "https://img.test/a.jpg"
|
||||
assert table.rows[1][1].link_url == "https://item.test/1"
|
||||
|
||||
|
||||
def test_batch_update_splits_image_resources_into_twenty(monkeypatch):
|
||||
client = FeishuNativeDocClient("test-token")
|
||||
calls = []
|
||||
|
||||
def fake_request(method, path, **kwargs):
|
||||
calls.append(kwargs["json_body"]["requests"])
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(client, "request", fake_request)
|
||||
requests_ = [
|
||||
{"block_id": f"img-{index}", "replace_image": {"token": f"t-{index}"}}
|
||||
for index in range(45)
|
||||
]
|
||||
|
||||
client.batch_update("doc", requests_)
|
||||
|
||||
assert [len(batch) for batch in calls] == [20, 20, 5]
|
||||
|
||||
|
||||
def test_rejects_table_over_feishu_2000_cell_limit_before_api_call():
|
||||
rows = "".join("<tr>" + "<td>x</td>" * 8 + "</tr>" for _ in range(251))
|
||||
xml = "<h2>超大类目</h2><table><tbody>" + rows + "</tbody></table>"
|
||||
|
||||
with pytest.raises(FeishuNativeDocError, match="超过飞书单表 2000"):
|
||||
append_native_category_table(
|
||||
"https://example.feishu.cn/docx/test",
|
||||
xml,
|
||||
client=object(),
|
||||
)
|
||||
@@ -0,0 +1,469 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from upload_video_to_guanghe import (
|
||||
GuangHeUploader,
|
||||
_match_tm_item_ids_from_product_titles,
|
||||
download_videos,
|
||||
)
|
||||
|
||||
|
||||
class GuangHeMetadataTests(unittest.TestCase):
|
||||
def test_downloads_with_same_attachment_name_are_isolated_by_record(self):
|
||||
attachment = {"file_token": "file-token", "name": "same-name.mp4"}
|
||||
|
||||
with TemporaryDirectory() as tmp_dir, patch(
|
||||
"upload_video_to_guanghe.DOWNLOAD_DIR", Path(tmp_dir)
|
||||
), patch("upload_video_to_guanghe.run_lark") as run_lark_mock:
|
||||
def fake_download(args):
|
||||
output_dir = Path(args[args.index("--output") + 1])
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / attachment["name"]).write_bytes(b"video")
|
||||
return 0, "", ""
|
||||
|
||||
run_lark_mock.side_effect = fake_download
|
||||
|
||||
first = download_videos("record-one", [attachment])
|
||||
second = download_videos("record-two", [attachment])
|
||||
|
||||
self.assertEqual(first[0].parent.name, "record-one")
|
||||
self.assertEqual(second[0].parent.name, "record-two")
|
||||
self.assertNotEqual(first[0], second[0])
|
||||
|
||||
def test_unresolved_style_uses_exact_database_product_title_match(self):
|
||||
result = _match_tm_item_ids_from_product_titles(
|
||||
["宙斯3", "未建档款"],
|
||||
[
|
||||
(
|
||||
"986141740958",
|
||||
"GYXX/光影行星宙斯3双肩包男士背包机能大容量",
|
||||
),
|
||||
("655191144133", "GYXX光影行星宙斯双肩包男士电脑包"),
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(result, {"宙斯3": ["986141740958"]})
|
||||
|
||||
def test_product_search_term_removes_series_suffix(self):
|
||||
self.assertEqual(
|
||||
GuangHeUploader._normalize_product_search_term("宙斯系列"),
|
||||
"宙斯",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._normalize_product_search_term(" 宙斯 系列款 "),
|
||||
"宙斯",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._normalize_product_search_term("星迹2"),
|
||||
"星迹2",
|
||||
)
|
||||
|
||||
def test_cover_generation_pending_marker_is_not_upload_done(self):
|
||||
self.assertTrue(GuangHeUploader._cover_generation_is_pending("封面生成中"))
|
||||
self.assertFalse(GuangHeUploader._cover_generation_is_pending("智能封面图"))
|
||||
|
||||
def test_content_label_option_allows_browse_count_but_rejects_longer_label(self):
|
||||
self.assertTrue(
|
||||
GuangHeUploader._content_label_text_matches("# 斜挎包 41673031次浏览", "斜挎包")
|
||||
)
|
||||
self.assertTrue(
|
||||
GuangHeUploader._content_label_text_matches("商品展示", "商品展示")
|
||||
)
|
||||
self.assertTrue(
|
||||
GuangHeUploader._content_label_text_matches("#斜挎包", "斜挎包")
|
||||
)
|
||||
self.assertFalse(
|
||||
GuangHeUploader._content_label_text_matches("# 单肩斜挎包 32871826次浏览", "斜挎包")
|
||||
)
|
||||
|
||||
def test_topic_result_card_requires_topic_and_activity_statistics(self):
|
||||
self.assertTrue(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"# 在淘宝种草一夏 小二推荐 作品数469.5万 参与数16.2万",
|
||||
"在淘宝种草一夏",
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
GuangHeUploader._topic_result_card_text_matches(
|
||||
"在淘宝种草一夏",
|
||||
"在淘宝种草一夏",
|
||||
)
|
||||
)
|
||||
self.assertTrue(GuangHeUploader._topic_card_box_is_clickable(620, 120))
|
||||
self.assertFalse(GuangHeUploader._topic_card_box_is_clickable(620, 45))
|
||||
self.assertFalse(GuangHeUploader._topic_card_box_is_clickable(720, 600))
|
||||
|
||||
def test_poseidon_edge_uses_authoritative_tote_category(self):
|
||||
self.assertEqual(
|
||||
GuangHeUploader._resolve_title_category(
|
||||
"波塞冬edge",
|
||||
"双肩包、电脑包、托特包、大容量",
|
||||
),
|
||||
"托特包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._resolve_title_category("星迹2", ""),
|
||||
"斜挎包",
|
||||
)
|
||||
|
||||
def test_invalid_online_info_is_rejected_when_main_category_conflicts(self):
|
||||
result = GuangHeUploader._validate_online_product_info(
|
||||
"波塞冬edge",
|
||||
"主包型：双肩包；尺寸：39cm；容量/适配：16寸；材质：尼龙；卖点：分区",
|
||||
)
|
||||
self.assertEqual(result, "")
|
||||
|
||||
def test_verified_fallback_info_contains_all_product_fields(self):
|
||||
info = GuangHeUploader._verified_product_info("星迹2")
|
||||
for field in ["主包型：", "尺寸：", "容量/适配：", "材质：", "卖点："]:
|
||||
self.assertIn(field, info)
|
||||
self.assertIn("斜挎包", info)
|
||||
|
||||
def test_title_category_uses_name_then_search_features(self):
|
||||
self.assertEqual(
|
||||
GuangHeUploader._resolve_title_category("盖亚斜挎", "轻量、大容量"),
|
||||
"斜挎包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._resolve_title_category("宙斯", "双肩包、大容量、笔电仓"),
|
||||
"双肩包",
|
||||
)
|
||||
|
||||
def test_title_normalization_keeps_search_structure_and_uses_computer_bag_term(self):
|
||||
title = GuangHeUploader._normalize_generated_title(
|
||||
"宙斯笔电大容量通勤真能装",
|
||||
["宙斯"],
|
||||
"双肩包",
|
||||
)
|
||||
|
||||
self.assertTrue(title.startswith("宙斯双肩包,"))
|
||||
self.assertTrue(title.endswith("!"))
|
||||
self.assertIn("电脑包", title)
|
||||
self.assertNotIn("笔电", title)
|
||||
self.assertNotIn("笔记本电脑", title)
|
||||
self.assertNotIn("电脑包仓", title)
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_title_normalization_removes_compact_duplicate_model_and_broken_tail(self):
|
||||
title = GuangHeUploader._normalize_generated_title(
|
||||
"光影行星宙斯zair双肩包,宙斯zair,大容量分区收纳,从容应",
|
||||
["宙斯z air"],
|
||||
"双肩包",
|
||||
)
|
||||
|
||||
self.assertTrue(title.startswith("宙斯z air双肩包,"))
|
||||
self.assertEqual(title.replace(" ", "").count("宙斯zair"), 1)
|
||||
self.assertNotIn("从容应!", title)
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_same_style_multiple_videos_get_distinct_titles_when_hermes_repeats(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包科学分区收纳轻量通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1, video_total=2)
|
||||
second = uploader._generate_title(["宙斯"], video_index=2, video_total=2)
|
||||
|
||||
self.assertNotEqual(first, second)
|
||||
self.assertTrue(first.startswith("宙斯双肩包,"))
|
||||
self.assertTrue(second.startswith("宙斯双肩包,"))
|
||||
|
||||
def test_same_style_repeated_hermes_copy_gets_low_similarity_bodies(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "光影行星宙斯双肩包,科学分区收纳,轻量通勤出行!"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
):
|
||||
titles = [
|
||||
uploader._generate_title(["宙斯"], video_index=index, video_total=4)
|
||||
for index in range(1, 5)
|
||||
]
|
||||
|
||||
bodies = [GuangHeUploader._title_body(title) for title in titles]
|
||||
self.assertEqual(len(set(bodies)), 4)
|
||||
for index, body in enumerate(bodies):
|
||||
for other in bodies[index + 1:]:
|
||||
self.assertLess(
|
||||
GuangHeUploader._title_similarity(body, other),
|
||||
GuangHeUploader.TITLE_SIMILARITY_LIMIT,
|
||||
)
|
||||
|
||||
def test_different_styles_cannot_reuse_same_title_skeleton(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
repeated = "科学分区收纳,轻装通勤出行"
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch("upload_video_to_guanghe._call_hermes", return_value=(repeated, {})),
|
||||
):
|
||||
first = uploader._generate_title(["宙斯"], video_index=1)
|
||||
second = uploader._generate_title(["星云2"], video_index=1)
|
||||
|
||||
first_body = GuangHeUploader._title_body(first)
|
||||
second_body = GuangHeUploader._title_body(second)
|
||||
self.assertLess(
|
||||
GuangHeUploader._title_similarity(first_body, second_body),
|
||||
GuangHeUploader.TITLE_SIMILARITY_LIMIT,
|
||||
)
|
||||
|
||||
def test_title_similarity_ignores_product_prefix(self):
|
||||
first = "宙斯双肩包,科学分区收纳,轻装通勤出行!"
|
||||
second = "星云2斜挎包,科学分区收纳,轻装通勤出行!"
|
||||
|
||||
self.assertEqual(GuangHeUploader._title_similarity(first, second), 1.0)
|
||||
|
||||
def test_full_batch_can_claim_48_globally_distinct_title_bodies(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
titles = []
|
||||
for index in range(48):
|
||||
product_names = [f"款式{index % 12 + 1}"]
|
||||
video_index = index // 12 + 1
|
||||
candidate = uploader._fallback_title(
|
||||
product_names,
|
||||
"双肩包",
|
||||
variant_index=video_index,
|
||||
)
|
||||
titles.append(
|
||||
uploader._claim_unique_title(
|
||||
candidate,
|
||||
product_names,
|
||||
"双肩包",
|
||||
variant_index=video_index,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(set(titles)), 48)
|
||||
self.assertTrue(all(26 <= len(title) <= 30 for title in titles))
|
||||
for index, title in enumerate(titles):
|
||||
for other in titles[index + 1:]:
|
||||
self.assertLess(
|
||||
GuangHeUploader._title_similarity(title, other),
|
||||
GuangHeUploader.TITLE_SIMILARITY_LIMIT,
|
||||
)
|
||||
|
||||
def test_two_store_uploaders_can_share_one_global_title_registry(self):
|
||||
shared_global = []
|
||||
shared_styles = {}
|
||||
shared_usage = {}
|
||||
first_uploader = object.__new__(GuangHeUploader)
|
||||
second_uploader = object.__new__(GuangHeUploader)
|
||||
for uploader in (first_uploader, second_uploader):
|
||||
uploader._used_titles_global = shared_global
|
||||
uploader._used_titles_by_products = shared_styles
|
||||
uploader._title_variant_usage = shared_usage
|
||||
|
||||
first = first_uploader._claim_unique_title(
|
||||
first_uploader._fallback_title(["宙斯"], "双肩包", 1),
|
||||
["宙斯"],
|
||||
"双肩包",
|
||||
1,
|
||||
)
|
||||
second = second_uploader._claim_unique_title(
|
||||
second_uploader._fallback_title(["星云2"], "斜挎包", 1),
|
||||
["星云2"],
|
||||
"斜挎包",
|
||||
1,
|
||||
)
|
||||
|
||||
self.assertLess(
|
||||
GuangHeUploader._title_similarity(first, second),
|
||||
GuangHeUploader.TITLE_SIMILARITY_LIMIT,
|
||||
)
|
||||
|
||||
def test_product_search_specs_prefer_tm_item_ids_and_fallback_to_keyword(self):
|
||||
specs = GuangHeUploader._build_product_search_specs(
|
||||
["星云2", "未建档系列款"],
|
||||
{"星云2": ["900176117674", "865283623976"]},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
specs,
|
||||
[
|
||||
{
|
||||
"product_name": "星云2",
|
||||
"search_term": "900176117674",
|
||||
"exact_id": True,
|
||||
},
|
||||
{
|
||||
"product_name": "未建档系列款",
|
||||
"search_term": "未建档",
|
||||
"exact_id": False,
|
||||
},
|
||||
{
|
||||
"product_name": "星云2",
|
||||
"search_term": "865283623976",
|
||||
"exact_id": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
def test_exact_id_searches_get_one_store_keyword_fallback(self):
|
||||
specs = GuangHeUploader._build_product_search_specs(
|
||||
["极星双肩", "未建档款"],
|
||||
{"极星双肩": ["972285625931", "966753757238"]},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
GuangHeUploader._build_exact_search_fallback_specs(specs),
|
||||
[
|
||||
{
|
||||
"product_name": "极星双肩",
|
||||
"search_term": "极星双肩",
|
||||
"exact_id": False,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_title_fallback_contains_product_category_solution_and_result(self):
|
||||
title = GuangHeUploader._fallback_title(["盖亚斜挎"], "斜挎包")
|
||||
|
||||
self.assertTrue(title.startswith("盖亚斜挎包,"))
|
||||
self.assertIn(",", title)
|
||||
self.assertTrue(title.endswith("!"))
|
||||
self.assertIn("钥匙卡包各归其位", title)
|
||||
self.assertIn("早高峰取物", title)
|
||||
self.assertGreaterEqual(title.count(","), 2)
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_generated_title_replaces_generic_category_with_model_category(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(uploader, "_search_product_features", return_value=""),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
return_value=("光影行星宙斯双肩包大容量笔记本电脑轻量通勤", {}),
|
||||
),
|
||||
):
|
||||
title = uploader._generate_title(["宙斯"])
|
||||
|
||||
self.assertTrue(title.startswith("宙斯双肩包,"))
|
||||
self.assertNotIn("包袋双肩包", title)
|
||||
|
||||
def test_generated_poseidon_title_cannot_be_changed_to_backpack_by_hermes(self):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
with (
|
||||
patch.object(
|
||||
uploader,
|
||||
"_search_product_features",
|
||||
return_value="双肩包、电脑包、托特包、大容量",
|
||||
),
|
||||
patch("upload_video_to_guanghe.HAS_HERMES", True),
|
||||
patch(
|
||||
"upload_video_to_guanghe._call_hermes",
|
||||
return_value=(
|
||||
"光影行星波塞冬edge双肩包暴雨通勤防泼水电脑仓从容出行",
|
||||
{},
|
||||
),
|
||||
),
|
||||
):
|
||||
title = uploader._generate_title(["波塞冬edge"])
|
||||
|
||||
self.assertTrue(title.startswith("波塞冬edge托特包,"))
|
||||
self.assertNotIn("双肩包", title)
|
||||
self.assertTrue(title.endswith("!"))
|
||||
self.assertGreaterEqual(len(title), 26)
|
||||
self.assertLessEqual(len(title), 30)
|
||||
|
||||
def test_primary_content_label_uses_first_product_bag_type(self):
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["盖亚斜挎", "极星双肩"]),
|
||||
"斜挎包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["盖亚微单"]),
|
||||
"相机包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["波塞冬edge"]),
|
||||
"托特包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["波塞冬手提电脑包"]),
|
||||
"电脑包",
|
||||
)
|
||||
|
||||
def test_catalog_product_titles_supply_style_category_and_verified_info(self):
|
||||
try:
|
||||
GuangHeUploader.configure_catalog_product_titles(
|
||||
{
|
||||
"星云2": [
|
||||
"GYXX光影行星星云2斜挎包男士单肩包平板胸包",
|
||||
"光影行星星云斜挎包男款邮差单肩包",
|
||||
],
|
||||
"宙斯": ["GYXX光影行星宙斯双肩包男大容量通勤背包"],
|
||||
"宙斯z air": ["光影行星宙斯zair双肩包电脑包通勤背包"],
|
||||
"黑曜石托特": ["光影行星黑曜石双肩托特多用包"],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["星云2"]),
|
||||
"斜挎包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._resolve_title_category("宙斯"),
|
||||
"双肩包",
|
||||
)
|
||||
self.assertIn(
|
||||
"主包型：斜挎包",
|
||||
GuangHeUploader._verified_product_info("星云2"),
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["宙斯z air"]),
|
||||
"双肩包",
|
||||
)
|
||||
self.assertEqual(
|
||||
GuangHeUploader._content_label_for_products(["黑曜石托特"]),
|
||||
"托特包",
|
||||
)
|
||||
finally:
|
||||
GuangHeUploader.configure_catalog_product_titles({})
|
||||
|
||||
def test_metadata_selects_style_category_and_product_showcase_before_topic(self):
|
||||
cases = [
|
||||
(["盖亚斜挎"], ["斜挎包", "商品展示"]),
|
||||
(["极星双肩"], ["双肩包", "商品展示"]),
|
||||
]
|
||||
for product_names, expected_labels in cases:
|
||||
with self.subTest(product_names=product_names):
|
||||
uploader = object.__new__(GuangHeUploader)
|
||||
calls = []
|
||||
uploader._pick_content_labels = Mock(
|
||||
side_effect=lambda labels: calls.append(("labels", list(labels)))
|
||||
)
|
||||
uploader._pick_topic = Mock(
|
||||
side_effect=lambda topic: calls.append(("topic", topic))
|
||||
)
|
||||
uploader._snapshot = Mock()
|
||||
|
||||
uploader._prepare_publish_metadata(product_names)
|
||||
|
||||
self.assertEqual(
|
||||
calls,
|
||||
[
|
||||
("labels", expected_labels),
|
||||
("topic", "在淘宝种草一夏"),
|
||||
],
|
||||
)
|
||||
|
||||
def test_fixed_content_labels_only_keep_product_showcase(self):
|
||||
self.assertEqual(GuangHeUploader.FIXED_CONTENT_LABELS, ("商品展示",))
|
||||
self.assertNotIn("户外", GuangHeUploader.FIXED_CONTENT_LABELS)
|
||||
|
||||
def test_fixed_topic_is_not_a_recommended_first_result(self):
|
||||
self.assertEqual(GuangHeUploader.FIXED_TOPIC, "在淘宝种草一夏")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,118 @@
|
||||
import unittest
|
||||
|
||||
from upload_video_to_guanghe import (
|
||||
STORE_FLAGSHIP,
|
||||
STORE_LUGGAGE,
|
||||
TM_UPLOAD_FIELD,
|
||||
select_pending_records,
|
||||
)
|
||||
|
||||
|
||||
def record(
|
||||
*,
|
||||
brand="光影行星",
|
||||
gender="女",
|
||||
uploaded=False,
|
||||
attachments=True,
|
||||
date=None,
|
||||
):
|
||||
return {
|
||||
"品牌": [brand] if brand is not None else None,
|
||||
"性别": [gender] if gender is not None else None,
|
||||
TM_UPLOAD_FIELD: uploaded,
|
||||
"代拍附件": [{"name": "video.mp4"}] if attachments else [],
|
||||
"日期": date,
|
||||
}
|
||||
|
||||
|
||||
class GuangHeStoreRoutingTests(unittest.TestCase):
|
||||
def test_tm_upload_field_matches_current_feishu_schema(self):
|
||||
self.assertEqual(TM_UPLOAD_FIELD, "天猫上传")
|
||||
|
||||
def test_flagship_requires_brand_gender_attachment_and_unchecked_status(self):
|
||||
records = [
|
||||
record(gender="男"),
|
||||
record(gender="女"),
|
||||
record(gender="女", uploaded=True),
|
||||
record(brand="OZKO"),
|
||||
record(brand=None),
|
||||
record(gender=None),
|
||||
record(attachments=False),
|
||||
]
|
||||
|
||||
selected = select_pending_records(records, STORE_FLAGSHIP)
|
||||
|
||||
self.assertEqual(selected, records[:2])
|
||||
|
||||
def test_luggage_uploads_female_brand_records_regardless_of_tm_status(self):
|
||||
unchecked = record(gender="女", uploaded=False)
|
||||
checked = record(gender="女", uploaded=True)
|
||||
unchecked["代拍产品"] = "极星双肩"
|
||||
checked["代拍产品"] = "极星双肩"
|
||||
records = [
|
||||
unchecked,
|
||||
checked,
|
||||
record(gender="男"),
|
||||
record(brand="OZKO", gender="女"),
|
||||
record(gender=None),
|
||||
record(gender="女", attachments=False),
|
||||
]
|
||||
|
||||
selected = select_pending_records(
|
||||
records,
|
||||
STORE_LUGGAGE,
|
||||
luggage_style_whitelist={"极星双肩", "觅光"},
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [unchecked, checked])
|
||||
|
||||
def test_luggage_rejects_styles_outside_whitelist_and_trims_names(self):
|
||||
allowed = record()
|
||||
allowed["代拍产品"] = " 黑曜石电脑包 "
|
||||
blocked = record()
|
||||
blocked["代拍产品"] = "宙斯z air"
|
||||
multi_with_blocked_style = record()
|
||||
multi_with_blocked_style["代拍产品"] = "极星双肩 / 宙斯z air"
|
||||
|
||||
selected = select_pending_records(
|
||||
[allowed, blocked, multi_with_blocked_style],
|
||||
STORE_LUGGAGE,
|
||||
luggage_style_whitelist={"黑曜石电脑包", "极星双肩"},
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [allowed])
|
||||
|
||||
def test_luggage_requires_a_loaded_whitelist(self):
|
||||
with self.assertRaisesRegex(ValueError, "款式白名单"):
|
||||
select_pending_records([record()], STORE_LUGGAGE)
|
||||
|
||||
def test_flagship_republish_from_date_includes_uploaded_records_in_range(self):
|
||||
before = record(uploaded=True, date="2026-06-02 00:00:00")
|
||||
first = record(uploaded=True, date="2026-06-03 00:00:00")
|
||||
later = record(uploaded=True, date="2026-07-26 00:00:00")
|
||||
missing_date = record(uploaded=True, date=None)
|
||||
|
||||
selected = select_pending_records(
|
||||
[before, first, later, missing_date],
|
||||
STORE_FLAGSHIP,
|
||||
republish_from_date="2026-06-03",
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [first, later])
|
||||
|
||||
def test_republish_from_date_is_rejected_for_luggage_store(self):
|
||||
with self.assertRaisesRegex(ValueError, "旗舰店"):
|
||||
select_pending_records(
|
||||
[record(date="2026-06-03 00:00:00")],
|
||||
STORE_LUGGAGE,
|
||||
luggage_style_whitelist={"极星双肩"},
|
||||
republish_from_date="2026-06-03",
|
||||
)
|
||||
|
||||
def test_unknown_store_is_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "未知光合店铺"):
|
||||
select_pending_records([record()], "unknown")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,110 @@
|
||||
import unittest
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import import_product_daily
|
||||
|
||||
TM_31_HEADERS = [
|
||||
"统计日期", "商品ID", "商品名称", "货号", "商品状态", "商品访客数",
|
||||
"商品浏览量", "平均停留时长", "商品详情页跳出率", "商品收藏人数",
|
||||
"商品加购件数", "商品加购人数", "下单买家数", "下单件数", "下单金额",
|
||||
"下单转化率", "支付买家数", "支付件数", "支付金额", "商品支付转化率",
|
||||
"支付新买家数", "支付老买家数", "老买家支付金额", "聚划算支付金额",
|
||||
"访客平均价值", "成功退款金额", "搜索引导支付转化率", "搜索引导访客数",
|
||||
"搜索引导支付买家数", "结构化详情引导转化率", "结构化详情引导成交占比",
|
||||
]
|
||||
|
||||
TM_38_HEADERS = [
|
||||
"统计日期", "商品ID", "商品名称", "主商品ID", "商品类型", "货号", "商品状态",
|
||||
"商品标签", "商品访客数", "商品浏览量", "平均停留时长",
|
||||
"商品详情页跳出率", "商品收藏人数", "商品加购件数", "商品加购人数",
|
||||
"下单买家数", "下单件数", "下单金额", "下单转化率", "支付买家数",
|
||||
"支付件数", "支付金额", "商品支付转化率", "支付新买家数", "支付老买家数",
|
||||
"老买家支付金额", "聚划算支付金额", "访客平均价值", "成功退款金额",
|
||||
"竞争力评分", "年累计支付金额", "月累计支付金额", "月累计支付件数",
|
||||
"搜索引导支付转化率", "搜索引导访客数", "搜索引导支付买家数",
|
||||
"结构化详情引导转化率", "结构化详情引导成交占比",
|
||||
]
|
||||
|
||||
|
||||
class _FakeSheet:
|
||||
def __init__(self, headers, row):
|
||||
self.nrows = 6
|
||||
self._headers = headers
|
||||
self._row = row
|
||||
|
||||
def row_values(self, row_index):
|
||||
if row_index == 4:
|
||||
return self._headers
|
||||
if row_index == 5:
|
||||
return self._row
|
||||
return []
|
||||
|
||||
|
||||
class _FakeWorkbook:
|
||||
def __init__(self, headers, row):
|
||||
self._sheet = _FakeSheet(headers, row)
|
||||
|
||||
def sheet_by_index(self, _sheet_index):
|
||||
return self._sheet
|
||||
|
||||
|
||||
def _tm_row(headers):
|
||||
values = {
|
||||
"统计日期": "2026-07-03",
|
||||
"商品ID": "123456",
|
||||
"商品名称": "测试商品",
|
||||
"商品类型": "普通商品",
|
||||
"货号": "SKU-001",
|
||||
"商品访客数": "100",
|
||||
"商品浏览量": "200",
|
||||
"支付买家数": "8",
|
||||
"支付件数": "10",
|
||||
"支付金额": "1234.56",
|
||||
"商品支付转化率": "8%",
|
||||
"访客平均价值": "12.35",
|
||||
"成功退款金额": "23.45",
|
||||
"竞争力评分": "95",
|
||||
"年累计支付金额": "9999.99",
|
||||
"月累计支付金额": "5678.90",
|
||||
"结构化详情引导成交占比": "45%",
|
||||
}
|
||||
row = [values.get(header, "") for header in headers]
|
||||
return row
|
||||
|
||||
|
||||
class TmallProductDailyImportTests(unittest.TestCase):
|
||||
def test_parse_31_column_export(self):
|
||||
row = _tm_row(TM_31_HEADERS)
|
||||
with patch.object(
|
||||
import_product_daily.xlrd,
|
||||
"open_workbook",
|
||||
return_value=_FakeWorkbook(TM_31_HEADERS, row),
|
||||
):
|
||||
records = list(import_product_daily.parse_tm_xls(Path("31-columns.xls")))
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
self.assertEqual(records[0]["product_no"], "SKU-001")
|
||||
self.assertEqual(records[0]["visitors"], 100)
|
||||
self.assertEqual(records[0]["paid_qty"], 10)
|
||||
self.assertEqual(records[0]["paid_amount"], Decimal("1234.56"))
|
||||
self.assertEqual(records[0]["refund_amount"], Decimal("23.45"))
|
||||
self.assertNotIn("月累计支付金额", records[0]["raw_data"])
|
||||
|
||||
def test_parse_38_column_export_keeps_extended_fields(self):
|
||||
row = _tm_row(TM_38_HEADERS)
|
||||
with patch.object(
|
||||
import_product_daily.xlrd,
|
||||
"open_workbook",
|
||||
return_value=_FakeWorkbook(TM_38_HEADERS, row),
|
||||
):
|
||||
records = list(import_product_daily.parse_tm_xls(Path("38-columns.xls")))
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
self.assertEqual(records[0]["raw_data"]["月累计支付金额"], "5678.90")
|
||||
self.assertEqual(records[0]["raw_data"]["结构化详情引导成交占比"], "45%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,319 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
PROJECT_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
JD_VENDOR_DIR = PROJECT_ROOT / "vendors" / "jd-data-flow"
|
||||
MODULE_PATH = JD_VENDOR_DIR / "jd_data_collector.py"
|
||||
|
||||
sys.path.insert(0, str(JD_VENDOR_DIR))
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("jd_data_collector_under_test", MODULE_PATH)
|
||||
jd = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(jd)
|
||||
finally:
|
||||
sys.path.remove(str(JD_VENDOR_DIR))
|
||||
|
||||
|
||||
class FakeKeyboard:
|
||||
def __init__(self, page):
|
||||
self.page = page
|
||||
self.presses = []
|
||||
|
||||
def press(self, key):
|
||||
self.presses.append(key)
|
||||
|
||||
|
||||
class FakeLocator:
|
||||
def __init__(self, visible=False, on_click=None):
|
||||
self.visible = visible
|
||||
self.on_click = on_click
|
||||
self.clicks = 0
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def is_visible(self, timeout=None):
|
||||
return self.visible
|
||||
|
||||
def click(self, timeout=None):
|
||||
self.clicks += 1
|
||||
if self.on_click:
|
||||
self.on_click()
|
||||
|
||||
|
||||
class FakeOverlayPage:
|
||||
def __init__(self, close_button_works, overlay_visible=True, blocks_entry=True):
|
||||
self.overlay_visible = overlay_visible
|
||||
self.close_button_works = close_button_works
|
||||
self.blocks_entry = blocks_entry
|
||||
self.keyboard = FakeKeyboard(self)
|
||||
self.evaluate_calls = []
|
||||
self.waits = []
|
||||
|
||||
def locator(self, selector, **kwargs):
|
||||
if selector == "div.all-dialog:visible":
|
||||
return FakeLocator(visible=self.overlay_visible)
|
||||
if self.close_button_works and "close-btn" in selector:
|
||||
return FakeLocator(visible=True, on_click=self._close_overlay)
|
||||
return FakeLocator(visible=False)
|
||||
|
||||
def _close_overlay(self):
|
||||
self.overlay_visible = False
|
||||
|
||||
def evaluate(self, script, *args):
|
||||
self.evaluate_calls.append(script)
|
||||
if "elementFromPoint" in script and "setProperty('display'" in script:
|
||||
if self.overlay_visible and self.blocks_entry:
|
||||
self.overlay_visible = False
|
||||
return 1
|
||||
return 0
|
||||
if "elementFromPoint" in script:
|
||||
return self.overlay_visible and self.blocks_entry
|
||||
if "all-dialog" in script and self.blocks_entry:
|
||||
self.overlay_visible = False
|
||||
return 1
|
||||
if "all-dialog" in script:
|
||||
# The old broad fallback would hide this non-blocking business dialog.
|
||||
if "elementFromPoint" not in script:
|
||||
self.overlay_visible = False
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
self.waits.append(milliseconds)
|
||||
|
||||
|
||||
class FakePage:
|
||||
def __init__(
|
||||
self,
|
||||
url="https://shop.jd.com/jdm/home",
|
||||
click_actions=None,
|
||||
wait_actions=None,
|
||||
):
|
||||
self.url = url
|
||||
self.closed = False
|
||||
self.click_actions = list(click_actions or [])
|
||||
self.entry_clicks = 0
|
||||
self.keyboard = FakeKeyboard(self)
|
||||
self.locator_calls = []
|
||||
self.wait_actions = list(wait_actions or [])
|
||||
|
||||
def is_closed(self):
|
||||
return self.closed
|
||||
|
||||
def locator(self, selector, **kwargs):
|
||||
self.locator_calls.append((selector, kwargs))
|
||||
|
||||
def click_entry():
|
||||
self.entry_clicks += 1
|
||||
if self.click_actions:
|
||||
action = self.click_actions.pop(0)
|
||||
action()
|
||||
|
||||
is_precise_shangzhi_entry = (
|
||||
selector == '.shop-menu-navigate__menu-name:has-text("商智"):visible'
|
||||
)
|
||||
return FakeLocator(visible=is_precise_shangzhi_entry, on_click=click_entry)
|
||||
|
||||
def wait_for_timeout(self, milliseconds):
|
||||
if self.wait_actions:
|
||||
self.wait_actions.pop(0)()
|
||||
return None
|
||||
|
||||
def wait_for_load_state(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
def goto(self, url, **kwargs):
|
||||
self.url = url
|
||||
|
||||
|
||||
class FakeContext:
|
||||
def __init__(self, page):
|
||||
self.pages = [page]
|
||||
|
||||
def new_page(self):
|
||||
page = FakePage()
|
||||
self.pages.append(page)
|
||||
return page
|
||||
|
||||
|
||||
class JdEnterShangzhiTests(unittest.TestCase):
|
||||
def test_dismiss_clicks_real_close_button_before_dom_fallback(self):
|
||||
page = FakeOverlayPage(close_button_works=True)
|
||||
|
||||
self.assertTrue(jd.dismiss_shop_home_overlays(page))
|
||||
|
||||
self.assertFalse(page.overlay_visible)
|
||||
self.assertFalse(
|
||||
any("setProperty('display'" in script for script in page.evaluate_calls)
|
||||
)
|
||||
|
||||
def test_dismiss_hides_stuck_pointer_blocking_overlay(self):
|
||||
page = FakeOverlayPage(close_button_works=False)
|
||||
|
||||
self.assertTrue(jd.dismiss_shop_home_overlays(page))
|
||||
|
||||
self.assertFalse(page.overlay_visible)
|
||||
self.assertTrue(page.keyboard.presses)
|
||||
self.assertTrue(any("all-dialog" in script for script in page.evaluate_calls))
|
||||
|
||||
def test_dismiss_does_not_touch_dom_when_no_overlay_is_visible(self):
|
||||
page = FakeOverlayPage(close_button_works=False, overlay_visible=False)
|
||||
|
||||
self.assertTrue(jd.dismiss_shop_home_overlays(page))
|
||||
|
||||
self.assertEqual(page.keyboard.presses, [])
|
||||
self.assertEqual(page.evaluate_calls, [])
|
||||
|
||||
def test_dismiss_does_not_hide_nonblocking_business_dialog(self):
|
||||
page = FakeOverlayPage(
|
||||
close_button_works=False,
|
||||
overlay_visible=True,
|
||||
blocks_entry=False,
|
||||
)
|
||||
|
||||
self.assertTrue(jd.dismiss_shop_home_overlays(page))
|
||||
|
||||
self.assertTrue(page.overlay_visible)
|
||||
self.assertFalse(any("setProperty('display'" in script for script in page.evaluate_calls))
|
||||
|
||||
def test_enter_retries_after_first_click_is_intercepted(self):
|
||||
source = FakePage()
|
||||
context = FakeContext(source)
|
||||
destination = FakePage(url="https://sz.jd.com/szweb/index.html")
|
||||
|
||||
def fail_first_click():
|
||||
raise RuntimeError("all-dialog intercepts pointer events")
|
||||
|
||||
def open_destination():
|
||||
context.pages.append(destination)
|
||||
|
||||
source.click_actions = [fail_first_click, open_destination]
|
||||
dismiss_calls = []
|
||||
with (
|
||||
patch.object(
|
||||
jd,
|
||||
"dismiss_shop_home_overlays",
|
||||
side_effect=lambda page: dismiss_calls.append(page) or True,
|
||||
),
|
||||
patch.object(jd, "close_popup_if_present", return_value=None),
|
||||
):
|
||||
result = jd.step_enter_shangzhi(context, source)
|
||||
|
||||
self.assertIs(result, destination)
|
||||
self.assertEqual(source.entry_clicks, 2)
|
||||
self.assertGreaterEqual(len(dismiss_calls), 2)
|
||||
self.assertFalse(any(selector == "text=商智" for selector, _ in source.locator_calls))
|
||||
|
||||
def test_enter_accepts_current_page_navigation(self):
|
||||
source = FakePage()
|
||||
context = FakeContext(source)
|
||||
source.click_actions = [
|
||||
lambda: setattr(source, "url", "https://sz.jd.com/szweb/index.html")
|
||||
]
|
||||
with (
|
||||
patch.object(jd, "dismiss_shop_home_overlays", return_value=True),
|
||||
patch.object(jd, "close_popup_if_present", return_value=None),
|
||||
):
|
||||
result = jd.step_enter_shangzhi(context, source)
|
||||
|
||||
self.assertIs(result, source)
|
||||
self.assertEqual(source.entry_clicks, 1)
|
||||
|
||||
def test_enter_rejects_unrelated_new_page_and_retries(self):
|
||||
source = FakePage()
|
||||
context = FakeContext(source)
|
||||
advertisement = FakePage(url="https://pro.jd.com/promotion.html")
|
||||
destination = FakePage(url="https://sz.jd.com/szweb/index.html")
|
||||
source.click_actions = [
|
||||
lambda: context.pages.append(advertisement),
|
||||
lambda: context.pages.append(destination),
|
||||
]
|
||||
with (
|
||||
patch.object(jd, "dismiss_shop_home_overlays", return_value=True),
|
||||
patch.object(jd, "close_popup_if_present", return_value=None),
|
||||
):
|
||||
result = jd.step_enter_shangzhi(context, source)
|
||||
|
||||
self.assertIs(result, destination)
|
||||
self.assertEqual(source.entry_clicks, 2)
|
||||
|
||||
def test_enter_rejects_login_page_whose_return_url_mentions_shangzhi(self):
|
||||
source = FakePage()
|
||||
context = FakeContext(source)
|
||||
login_page = FakePage(
|
||||
url=(
|
||||
"https://passport.jd.com/login?returnUrl="
|
||||
"https%3A%2F%2Fsz.jd.com%2Fszweb%2Findex.html"
|
||||
)
|
||||
)
|
||||
destination = FakePage(url="https://sz.jd.com/szweb/index.html")
|
||||
source.click_actions = [
|
||||
lambda: context.pages.append(login_page),
|
||||
lambda: context.pages.append(destination),
|
||||
]
|
||||
with (
|
||||
patch.object(jd, "dismiss_shop_home_overlays", return_value=True),
|
||||
patch.object(jd, "close_popup_if_present", return_value=None),
|
||||
):
|
||||
result = jd.step_enter_shangzhi(context, source)
|
||||
|
||||
self.assertIs(result, destination)
|
||||
self.assertEqual(source.entry_clicks, 2)
|
||||
|
||||
def test_shangzhi_url_match_uses_hostname_not_query_or_similar_domain(self):
|
||||
self.assertTrue(jd._is_shangzhi_url("https://sz.jd.com/szweb/index.html"))
|
||||
self.assertTrue(jd._is_shangzhi_url("https://sub.sz.jd.com/path"))
|
||||
self.assertTrue(
|
||||
jd._is_shangzhi_url(
|
||||
"https://jdsz.jd.com/szweb/view/index/home.html"
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
jd._is_shangzhi_url(
|
||||
"https://passport.jd.com/login?returnUrl=https://sz.jd.com/szweb/"
|
||||
)
|
||||
)
|
||||
self.assertFalse(jd._is_shangzhi_url("https://evil-sz.jd.com/szweb/"))
|
||||
self.assertFalse(jd._is_shangzhi_url("https://evil-jdsz.jd.com/szweb/"))
|
||||
|
||||
def test_enter_waits_for_about_blank_popup_to_navigate_to_shangzhi(self):
|
||||
source = FakePage()
|
||||
context = FakeContext(source)
|
||||
destination = FakePage(url="about:blank")
|
||||
source.click_actions = [lambda: context.pages.append(destination)]
|
||||
source.wait_actions = [
|
||||
lambda: None,
|
||||
lambda: setattr(
|
||||
destination,
|
||||
"url",
|
||||
"https://jdsz.jd.com/szweb/view/index/home.html",
|
||||
),
|
||||
]
|
||||
with (
|
||||
patch.object(jd, "dismiss_shop_home_overlays", return_value=True),
|
||||
patch.object(jd, "close_popup_if_present", return_value=None),
|
||||
):
|
||||
result = jd.step_enter_shangzhi(context, source)
|
||||
|
||||
self.assertIs(result, destination)
|
||||
self.assertEqual(
|
||||
result.url,
|
||||
"https://jdsz.jd.com/szweb/view/index/home.html",
|
||||
)
|
||||
self.assertEqual(source.entry_clicks, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,295 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from collect_jd_market_rank import (
|
||||
NEXT_PAGE_SELECTORS,
|
||||
TARGET_JD_MARKET_CATEGORIES,
|
||||
build_jd_feishu_batch_xml,
|
||||
build_jd_feishu_category_xml,
|
||||
build_jd_feishu_intro_xml,
|
||||
category_selection_matches,
|
||||
exact_category_label_pattern,
|
||||
group_products_by_category,
|
||||
high_resolution_jd_image_url,
|
||||
inspect_feishu_table_images,
|
||||
parse_jd_market_rank_html,
|
||||
parse_money_range_upper,
|
||||
resize_feishu_table_xml,
|
||||
select_category_paths,
|
||||
)
|
||||
|
||||
SAMPLE_HTML = """
|
||||
<table class="jmtd-table-main">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>排名</th>
|
||||
<th>商品信息</th>
|
||||
<th>成交金额</th>
|
||||
<th>成交单量</th>
|
||||
<th>访客数</th>
|
||||
<th>搜索点击次数</th>
|
||||
<th>关注人数</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><span class="jmtd-table-formatter-rank-icon-1"></span><span>↑3名</span></td>
|
||||
<td>
|
||||
<div class="prod-info">
|
||||
<a class="prod-name" href="//item.jd.com/10012345678901.html?pcdk=tracking"
|
||||
title="测试高颜值拉杆箱">测试高颜值拉杆箱</a>
|
||||
<a class="shop-name">测试旗舰店</a>
|
||||
<img data-src="//img10.360buyimg.com/n1/jfs/test.jpg"/>
|
||||
</div>
|
||||
</td>
|
||||
<td>¥ 50万 ～ ¥ 75万</td>
|
||||
<td>4,000~6,000</td>
|
||||
<td>4万~6万</td>
|
||||
<td>4万~6万</td>
|
||||
<td>357</td>
|
||||
<td>详情</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="jmtd-table-formatter-rank-number">4</span></td>
|
||||
<td>
|
||||
<div class="prod-info">
|
||||
<a class="prod-name" href="https://item.jd.com/10012345678902.html">
|
||||
测试电脑包
|
||||
</a>
|
||||
<a class="shop-name">电脑包旗舰店</a>
|
||||
<img src="https://img11.360buyimg.com/n1/jfs/test2.jpg"/>
|
||||
</div>
|
||||
</td>
|
||||
<td>¥10万~¥25万</td>
|
||||
<td>50~100</td>
|
||||
<td>4,000~6,000</td>
|
||||
<td>1,000~2,000</td>
|
||||
<td>20</td>
|
||||
<td>详情</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
class JdMarketRankTests(unittest.TestCase):
|
||||
def test_default_categories_match_requested_seven_filters(self):
|
||||
specs = {item["name"]: item for item in TARGET_JD_MARKET_CATEGORIES}
|
||||
self.assertEqual(
|
||||
set(specs),
|
||||
{"旅行箱", "运动包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"},
|
||||
)
|
||||
self.assertEqual(specs["旅行箱"]["paths"], (("功能箱包", "行李箱"),))
|
||||
self.assertEqual(specs["运动包"]["paths"], (("功能箱包", "休闲运动包"),))
|
||||
self.assertEqual(
|
||||
specs["双肩包"]["paths"],
|
||||
(("男包", "双肩包"), ("男包", "男士双肩包")),
|
||||
)
|
||||
self.assertEqual(specs["手机包"]["paths"], (("男包", "男士手机包"),))
|
||||
self.assertEqual(specs["电脑包"]["paths"], (("功能箱包", "电脑包"),))
|
||||
|
||||
def test_category_label_matching_is_exact_not_contains(self):
|
||||
pattern = exact_category_label_pattern("双肩包")
|
||||
|
||||
self.assertIsNotNone(pattern.fullmatch(" 双肩包 "))
|
||||
self.assertIsNone(pattern.fullmatch("男士双肩包"))
|
||||
self.assertIsNone(pattern.fullmatch("双肩包配件"))
|
||||
|
||||
def test_selected_category_verification_accepts_only_exact_display_forms(self):
|
||||
path = ("男包", "双肩包")
|
||||
|
||||
self.assertTrue(category_selection_matches(("双肩包",), path))
|
||||
self.assertTrue(category_selection_matches(("男包 > 双肩包",), path))
|
||||
self.assertFalse(category_selection_matches(("男士双肩包",), path))
|
||||
self.assertFalse(category_selection_matches(("双肩包配件",), path))
|
||||
|
||||
def test_category_candidates_fall_back_only_after_explicit_failure(self):
|
||||
page = MagicMock()
|
||||
paths = (("男包", "双肩包"), ("男包", "男士双肩包"))
|
||||
with patch(
|
||||
"collect_jd_market_rank.select_category_path",
|
||||
side_effect=[RuntimeError("当前页面无此精确类目"), paths[1]],
|
||||
) as select_one:
|
||||
selected = select_category_paths(page, paths)
|
||||
|
||||
self.assertEqual(selected, paths[1])
|
||||
self.assertEqual(
|
||||
[call.args[1] for call in select_one.call_args_list],
|
||||
list(paths),
|
||||
)
|
||||
|
||||
def test_all_missing_category_candidates_fail_with_diagnostics(self):
|
||||
page = MagicMock()
|
||||
paths = (("男包", "双肩包"), ("男包", "男士双肩包"))
|
||||
with (
|
||||
patch(
|
||||
"collect_jd_market_rank.select_category_path",
|
||||
side_effect=[RuntimeError("无双肩包"), RuntimeError("无男士双肩包")],
|
||||
),
|
||||
self.assertRaisesRegex(RuntimeError, "候选类目路径.*无双肩包.*无男士双肩包"),
|
||||
):
|
||||
select_category_paths(page, paths)
|
||||
|
||||
def test_parse_money_range_upper(self):
|
||||
self.assertEqual(parse_money_range_upper("¥50万~¥75万"), 750_000)
|
||||
self.assertEqual(parse_money_range_upper("¥ 1亿以上"), 100_000_000)
|
||||
|
||||
def test_jd_thumbnail_is_upgraded_to_n0_original(self):
|
||||
self.assertEqual(
|
||||
high_resolution_jd_image_url(
|
||||
"https://img10.360buyimg.com/n5/jfs/t1/test/image.jpg"
|
||||
),
|
||||
"https://img10.360buyimg.com/n0/jfs/t1/test/image.jpg",
|
||||
)
|
||||
|
||||
def test_real_jd_next_page_selector_is_supported(self):
|
||||
self.assertIn(".jmtd-pagination-item-next", NEXT_PAGE_SELECTORS)
|
||||
|
||||
def test_inspect_feishu_table_images_finds_incomplete_table(self):
|
||||
content = """
|
||||
<title>测试</title>
|
||||
<table id="table-1">
|
||||
<thead><tr><th>商品图</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><img src="token-1"/></td></tr>
|
||||
<tr><td></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<table id="table-2">
|
||||
<thead><tr><th>商品图</th></tr></thead>
|
||||
<tbody><tr><td><img src="token-2"/></td></tr></tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
stats = inspect_feishu_table_images(content)
|
||||
|
||||
self.assertEqual(
|
||||
stats,
|
||||
[
|
||||
{"block_id": "table-1", "rows": 2, "images": 1},
|
||||
{"block_id": "table-2", "rows": 1, "images": 1},
|
||||
],
|
||||
)
|
||||
|
||||
def test_resize_feishu_table_xml_reuses_media_token_at_compact_size(self):
|
||||
table_xml = """
|
||||
<table id="table-1">
|
||||
<tbody>
|
||||
<tr id="row-1">
|
||||
<td>
|
||||
<img id="image-1" src="image-token" href="https://internal/image"
|
||||
width="220" height="800"/>
|
||||
</td>
|
||||
<td><a href="https://item.jd.com/1.html">打开商品</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
resized = resize_feishu_table_xml(table_xml, image_size=160)
|
||||
|
||||
self.assertNotIn('id="table-1"', resized)
|
||||
self.assertNotIn('id="image-1"', resized)
|
||||
self.assertIn(
|
||||
'<img src="image-token" width="160" height="160"/>',
|
||||
resized,
|
||||
)
|
||||
self.assertIn(
|
||||
'<a href="https://item.jd.com/1.html">打开商品</a>',
|
||||
resized,
|
||||
)
|
||||
|
||||
def test_parse_html_extracts_image_name_link_and_sales(self):
|
||||
rows = parse_jd_market_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="功能箱包 > 行李箱",
|
||||
)
|
||||
|
||||
self.assertEqual(len(rows), 2)
|
||||
first = rows[0]
|
||||
self.assertEqual(first["platform"], "jd")
|
||||
self.assertEqual(first["rank"], 1)
|
||||
self.assertEqual(first["product_id"], "10012345678901")
|
||||
self.assertEqual(first["name"], "测试高颜值拉杆箱")
|
||||
self.assertEqual(
|
||||
first["image_url"],
|
||||
"https://img10.360buyimg.com/n0/jfs/test.jpg",
|
||||
)
|
||||
self.assertEqual(
|
||||
first["product_url"],
|
||||
"https://item.jd.com/10012345678901.html",
|
||||
)
|
||||
self.assertEqual(first["sales_amount_range"], "¥50万~¥75万")
|
||||
self.assertEqual(first["sales_amount_upper"], 750_000)
|
||||
self.assertEqual(rows[1]["rank"], 4)
|
||||
|
||||
def test_feishu_xml_adds_sales_column_and_product_assets(self):
|
||||
product = parse_jd_market_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="功能箱包 > 行李箱",
|
||||
)[0]
|
||||
xml = build_jd_feishu_batch_xml([product], start_rank=1)
|
||||
|
||||
self.assertIn(">销售额</th>", xml)
|
||||
self.assertIn("<b>¥50万~¥75万</b>", xml)
|
||||
self.assertIn(
|
||||
'<img href="https://img10.360buyimg.com/n0/jfs/test.jpg" '
|
||||
'width="160" height="160"',
|
||||
xml,
|
||||
)
|
||||
self.assertIn(
|
||||
'<a href="https://item.jd.com/10012345678901.html">打开商品</a>',
|
||||
xml,
|
||||
)
|
||||
|
||||
def test_category_xml_uses_one_named_table_for_the_category(self):
|
||||
products = parse_jd_market_rank_html(
|
||||
SAMPLE_HTML,
|
||||
category_name="旅行箱",
|
||||
category_path="功能箱包 > 行李箱",
|
||||
)
|
||||
|
||||
xml = build_jd_feishu_category_xml("旅行箱", products)
|
||||
|
||||
self.assertIn("<h2>旅行箱（2条）</h2>", xml)
|
||||
self.assertEqual(xml.count("<table>"), 1)
|
||||
self.assertEqual(xml.count("<tbody>"), 1)
|
||||
self.assertEqual(xml.count("<tr>"), 3)
|
||||
|
||||
def test_group_products_by_category_preserves_category_order(self):
|
||||
grouped = group_products_by_category(
|
||||
[
|
||||
{"category_name": "旅行箱", "rank": 1},
|
||||
{"category_name": "运动包", "rank": 1},
|
||||
{"category_name": "旅行箱", "rank": 2},
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[(name, len(items)) for name, items in grouped],
|
||||
[("旅行箱", 2), ("运动包", 1)],
|
||||
)
|
||||
|
||||
def test_intro_records_period_and_category_path(self):
|
||||
xml = build_jd_feishu_intro_xml(
|
||||
title="京东市场榜单",
|
||||
period="近30天",
|
||||
products=[
|
||||
{
|
||||
"category_name": "运动包",
|
||||
"category_path": "功能箱包 > 休闲运动包",
|
||||
"search_keyword": "",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
self.assertIn("<h1>京东</h1>", xml)
|
||||
self.assertIn("近30天", xml)
|
||||
self.assertIn("运动包（功能箱包 > 休闲运动包）", xml)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,449 @@
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import db as product_db
|
||||
import run_weekly_jd_main_image as jd_runner
|
||||
import run_weekly_main_image as tmall_runner
|
||||
from main_image_db import persist_main_image_records, validate_main_image_schema
|
||||
from main_image_paths import collect_images_dir, resolve_images_dir
|
||||
from scripts import insert_jd_main_image_records as jd_insert
|
||||
from scripts import insert_main_image_records as tmall_insert
|
||||
from taobao_wanxiang_ai_creative_report import cleanup_images_after_visual_merge
|
||||
|
||||
|
||||
class MainImagePathTests(unittest.TestCase):
|
||||
def test_collectors_use_platform_owned_directories(self):
|
||||
style_dir = Path("style")
|
||||
self.assertEqual(collect_images_dir(style_dir, "tm"), style_dir / "tm_images")
|
||||
self.assertEqual(collect_images_dir(style_dir, "jd"), style_dir / "jd_images")
|
||||
|
||||
def test_insert_prefers_platform_directory_and_supports_legacy_data(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
style_dir = Path(tmp)
|
||||
legacy = style_dir / "images"
|
||||
legacy.mkdir()
|
||||
self.assertEqual(resolve_images_dir(style_dir, "jd"), legacy)
|
||||
|
||||
current = style_dir / "jd_images"
|
||||
current.mkdir()
|
||||
self.assertEqual(resolve_images_dir(style_dir, "jd"), current)
|
||||
|
||||
|
||||
class MainImageWorkflowPreflightTests(unittest.TestCase):
|
||||
def test_zero_valid_styles_is_a_formal_failure(self):
|
||||
for runner in (tmall_runner, jd_runner):
|
||||
with self.subTest(runner=runner.__name__):
|
||||
with self.assertRaisesRegex(RuntimeError, "拒绝按成功退出"):
|
||||
runner.require_valid_output(0, "2026-08-04")
|
||||
|
||||
def test_dynamic_style_read_failure_is_not_silently_ignored(self):
|
||||
with patch.object(
|
||||
tmall_runner,
|
||||
"get_main_image_styles",
|
||||
side_effect=RuntimeError("Feishu unavailable"),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "款式读取失败"):
|
||||
tmall_runner.load_dynamic_styles()
|
||||
|
||||
with patch.object(
|
||||
jd_runner,
|
||||
"get_jd_spu_groups",
|
||||
side_effect=RuntimeError("Feishu unavailable"),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "款式读取失败"):
|
||||
jd_runner.load_dynamic_styles()
|
||||
|
||||
def test_empty_dynamic_style_set_is_a_formal_failure(self):
|
||||
with patch.object(tmall_runner, "get_main_image_styles", return_value=[]):
|
||||
with self.assertRaisesRegex(RuntimeError, "未返回任何天猫"):
|
||||
tmall_runner.load_dynamic_styles()
|
||||
|
||||
with patch.object(jd_runner, "get_jd_spu_groups", return_value=[]):
|
||||
with self.assertRaisesRegex(RuntimeError, "未返回任何京东"):
|
||||
jd_runner.load_dynamic_styles()
|
||||
|
||||
|
||||
class TmallCleanupTests(unittest.TestCase):
|
||||
def test_locked_duplicate_is_retried_without_console_encoding_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
images_dir = Path(tmp)
|
||||
keep = images_dir / "keep.png"
|
||||
locked = images_dir / "locked.png"
|
||||
keep.write_bytes(b"keep")
|
||||
locked.write_bytes(b"duplicate")
|
||||
creatives = [{"local_image_path": str(keep), "local_image_paths": [str(keep), str(locked)]}]
|
||||
|
||||
original_unlink = Path.unlink
|
||||
attempts = 0
|
||||
|
||||
def flaky_unlink(path, *args, **kwargs):
|
||||
nonlocal attempts
|
||||
if path == locked:
|
||||
attempts += 1
|
||||
if attempts < 3:
|
||||
raise PermissionError("temporarily locked")
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
|
||||
raw = io.BytesIO()
|
||||
strict_gbk_stdout = io.TextIOWrapper(raw, encoding="gbk", errors="strict")
|
||||
with patch.object(Path, "unlink", new=flaky_unlink):
|
||||
with redirect_stdout(strict_gbk_stdout):
|
||||
cleanup_images_after_visual_merge(
|
||||
images_dir,
|
||||
creatives,
|
||||
retry_attempts=3,
|
||||
retry_delay=0,
|
||||
)
|
||||
strict_gbk_stdout.flush()
|
||||
|
||||
self.assertEqual(attempts, 3)
|
||||
self.assertFalse(locked.exists())
|
||||
self.assertTrue(keep.exists())
|
||||
|
||||
|
||||
class MainImageDatabaseTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _schema_connection(columns):
|
||||
class Cursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, _sql):
|
||||
return None
|
||||
|
||||
def fetchone(self):
|
||||
return (list(columns),)
|
||||
|
||||
class Connection:
|
||||
@staticmethod
|
||||
def cursor():
|
||||
return Cursor()
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
return Connection()
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
return ConnectionContext
|
||||
|
||||
def test_schema_preflight_accepts_platform_aware_primary_key(self):
|
||||
columns = ("collect_date", "style_name", "platform", "image_key")
|
||||
self.assertEqual(
|
||||
validate_main_image_schema(get_conn_fn=self._schema_connection(columns)),
|
||||
columns,
|
||||
)
|
||||
|
||||
def test_schema_preflight_rejects_legacy_primary_key_before_external_writes(self):
|
||||
legacy = ("collect_date", "style_name", "image_key")
|
||||
with self.assertRaisesRegex(RuntimeError, "primary key mismatch"):
|
||||
validate_main_image_schema(get_conn_fn=self._schema_connection(legacy))
|
||||
|
||||
def test_empty_batch_does_not_open_database_connection(self):
|
||||
def unexpected_connection():
|
||||
raise AssertionError("database should not be opened")
|
||||
|
||||
self.assertEqual(
|
||||
persist_main_image_records([], get_conn_fn=unexpected_connection),
|
||||
0,
|
||||
)
|
||||
|
||||
def test_database_connection_is_opened_only_when_records_are_persisted(self):
|
||||
events = []
|
||||
connection = object()
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
events.append("connect")
|
||||
return connection
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
events.append("close")
|
||||
|
||||
def upsert(conn, records):
|
||||
self.assertIs(conn, connection)
|
||||
events.append(("upsert", len(records)))
|
||||
return len(records)
|
||||
|
||||
self.assertEqual(
|
||||
persist_main_image_records(
|
||||
[{"image_key": "a"}, {"image_key": "b"}],
|
||||
get_conn_fn=ConnectionContext,
|
||||
upsert_fn=upsert,
|
||||
),
|
||||
2,
|
||||
)
|
||||
self.assertEqual(events, ["connect", ("upsert", 2), "close"])
|
||||
|
||||
def test_upsert_identity_contains_platform_and_preserves_both_platform_rows(self):
|
||||
class CursorContext:
|
||||
def __enter__(self):
|
||||
return object()
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
class Connection:
|
||||
@staticmethod
|
||||
def cursor():
|
||||
return CursorContext()
|
||||
|
||||
records = [
|
||||
{
|
||||
"collect_date": "2026-08-02",
|
||||
"style_name": "同款",
|
||||
"platform": platform,
|
||||
"image_key": "main.jpg",
|
||||
}
|
||||
for platform in ("jd", "tm")
|
||||
]
|
||||
with patch.object(product_db, "execute_values") as execute_values:
|
||||
count = product_db.upsert_main_image_creatives(Connection(), records)
|
||||
|
||||
self.assertEqual(count, 2)
|
||||
sql = execute_values.call_args.args[1]
|
||||
rows = execute_values.call_args.args[2]
|
||||
self.assertIn(
|
||||
"ON CONFLICT (collect_date, style_name, platform, image_key)",
|
||||
sql,
|
||||
)
|
||||
self.assertEqual([row[2] for row in rows], ["jd", "tm"])
|
||||
|
||||
def test_schema_migrates_existing_primary_key_to_platform_identity(self):
|
||||
schema = (Path(product_db.__file__).with_name("schema.sql")).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
expected = "PRIMARY KEY (collect_date, style_name, platform, image_key)"
|
||||
self.assertGreaterEqual(schema.count(expected), 2)
|
||||
self.assertIn("current_pk_columns IS DISTINCT FROM", schema)
|
||||
self.assertIn("DROP CONSTRAINT", schema)
|
||||
|
||||
|
||||
class MainImageSinkIsolationTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _write_source(
|
||||
root: Path,
|
||||
*,
|
||||
platform: str,
|
||||
date_str: str = "2026-08-02",
|
||||
style: str = "同款",
|
||||
) -> tuple[Path, Path]:
|
||||
style_dir = root / date_str / style
|
||||
images_dir = style_dir / f"{platform}_images"
|
||||
images_dir.mkdir(parents=True)
|
||||
image_path = images_dir / "main.jpg"
|
||||
image_path.write_bytes(b"image")
|
||||
if platform == "tm":
|
||||
payload = {
|
||||
"creatives": [
|
||||
{
|
||||
"image_key": image_path.name,
|
||||
"impressions": 10,
|
||||
"clicks": 2,
|
||||
}
|
||||
]
|
||||
}
|
||||
filename = "wanxiang_creative_main_images.json"
|
||||
else:
|
||||
payload = {
|
||||
"records": [
|
||||
{
|
||||
"image_key": image_path.name,
|
||||
"impressions": 20,
|
||||
"clicks": 3,
|
||||
}
|
||||
]
|
||||
}
|
||||
filename = "jd_main_images.json"
|
||||
(style_dir / filename).write_text(
|
||||
json.dumps(payload, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
return style_dir, image_path
|
||||
|
||||
def test_tmall_pg_payload_survives_feishu_discovery_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_source(root, platform="tm")
|
||||
pg_records = []
|
||||
with (
|
||||
patch.object(
|
||||
tmall_insert,
|
||||
"artifact_relative_path",
|
||||
return_value="raw/product_commerce/main.jpg",
|
||||
),
|
||||
patch.object(
|
||||
tmall_insert,
|
||||
"discover_fields",
|
||||
side_effect=RuntimeError("Feishu unavailable"),
|
||||
),
|
||||
):
|
||||
result = tmall_insert.process_product(
|
||||
"同款",
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-02",
|
||||
root,
|
||||
False,
|
||||
pg_records,
|
||||
)
|
||||
|
||||
self.assertTrue(result["errors"])
|
||||
self.assertEqual(len(pg_records), 1)
|
||||
self.assertEqual(pg_records[0]["platform"], "tm")
|
||||
self.assertFalse(pg_records[0]["uploaded_to_bitable"])
|
||||
|
||||
def test_jd_pg_payload_survives_feishu_discovery_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_source(root, platform="jd")
|
||||
pg_records = []
|
||||
with (
|
||||
patch.object(
|
||||
jd_insert,
|
||||
"artifact_relative_path",
|
||||
return_value="raw/product_commerce/main.jpg",
|
||||
),
|
||||
patch.object(
|
||||
jd_insert,
|
||||
"discover_fields",
|
||||
side_effect=RuntimeError("Feishu unavailable"),
|
||||
),
|
||||
):
|
||||
result = jd_insert.process_style(
|
||||
"同款",
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-02",
|
||||
root,
|
||||
False,
|
||||
pg_records,
|
||||
)
|
||||
|
||||
self.assertTrue(result["errors"])
|
||||
self.assertEqual(len(pg_records), 1)
|
||||
self.assertEqual(pg_records[0]["platform"], "jd")
|
||||
self.assertFalse(pg_records[0]["uploaded_to_bitable"])
|
||||
|
||||
def test_tmall_pg_payload_survives_feishu_record_lookup_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_source(root, platform="tm")
|
||||
pg_records = []
|
||||
fields = {
|
||||
"时间": "time",
|
||||
"平台": "platform",
|
||||
"曝光量": "impressions",
|
||||
"点击量": "clicks",
|
||||
"主图": "image",
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
tmall_insert,
|
||||
"artifact_relative_path",
|
||||
return_value="raw/product_commerce/main.jpg",
|
||||
),
|
||||
patch.object(tmall_insert, "discover_fields", return_value=fields),
|
||||
patch.object(
|
||||
tmall_insert,
|
||||
"list_existing_records",
|
||||
side_effect=RuntimeError("Feishu record-list unavailable"),
|
||||
),
|
||||
):
|
||||
result = tmall_insert.process_product(
|
||||
"同款",
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-02",
|
||||
root,
|
||||
False,
|
||||
pg_records,
|
||||
)
|
||||
|
||||
self.assertIn("record-list 失败", result["errors"][0])
|
||||
self.assertEqual(len(pg_records), 1)
|
||||
self.assertEqual(pg_records[0]["platform"], "tm")
|
||||
|
||||
def test_jd_pg_payload_survives_feishu_record_lookup_failure(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self._write_source(root, platform="jd")
|
||||
pg_records = []
|
||||
fields = {
|
||||
"时间": "time",
|
||||
"平台": "platform",
|
||||
"曝光量": "impressions",
|
||||
"点击量": "clicks",
|
||||
"主图": "image",
|
||||
}
|
||||
with (
|
||||
patch.object(
|
||||
jd_insert,
|
||||
"artifact_relative_path",
|
||||
return_value="raw/product_commerce/main.jpg",
|
||||
),
|
||||
patch.object(jd_insert, "discover_fields", return_value=fields),
|
||||
patch.object(
|
||||
jd_insert,
|
||||
"list_existing_records",
|
||||
side_effect=RuntimeError("Feishu record-list unavailable"),
|
||||
),
|
||||
):
|
||||
result = jd_insert.process_style(
|
||||
"同款",
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-08-02",
|
||||
root,
|
||||
False,
|
||||
pg_records,
|
||||
)
|
||||
|
||||
self.assertIn("record-list 失败", result["errors"][0])
|
||||
self.assertEqual(len(pg_records), 1)
|
||||
self.assertEqual(pg_records[0]["platform"], "jd")
|
||||
|
||||
def test_feishu_existing_record_lookup_is_platform_scoped(self):
|
||||
fields = {
|
||||
"时间": "time",
|
||||
"平台": "platform",
|
||||
"曝光量": "impressions",
|
||||
"点击量": "clicks",
|
||||
"主图": "image",
|
||||
}
|
||||
payload = {
|
||||
"data": {
|
||||
"field_id_list": list(fields.values()),
|
||||
"record_id_list": ["tm-record", "jd-record"],
|
||||
"data": [
|
||||
["本周", ["天猫"], 10, 2, [{"name": "main.jpg"}]],
|
||||
["本周", ["京东"], 20, 3, [{"name": "main.jpg"}]],
|
||||
],
|
||||
}
|
||||
}
|
||||
with patch.object(
|
||||
tmall_insert,
|
||||
"run_lark",
|
||||
return_value=(0, json.dumps(payload, ensure_ascii=False), ""),
|
||||
) as run_lark:
|
||||
by_image, metric_tuples, duplicates = (
|
||||
tmall_insert.list_existing_records(
|
||||
"base", "table", fields, "本周", "京东"
|
||||
)
|
||||
)
|
||||
|
||||
command = run_lark.call_args.args[0]
|
||||
filter_json = json.loads(command[command.index("--filter-json") + 1])
|
||||
self.assertIn(["平台", "==", "京东"], filter_json["conditions"])
|
||||
self.assertEqual(by_image["main.jpg"]["record_id"], "jd-record")
|
||||
self.assertEqual(metric_tuples, {(20, 3)})
|
||||
self.assertEqual(duplicates, {})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date, datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import market_rank_hermes_notification as notification
|
||||
|
||||
REPORTS = [
|
||||
{
|
||||
"platform": "tm",
|
||||
"feishu_doc_url": "https://example.com/tm",
|
||||
"product_count": 1178,
|
||||
"report_title": "天猫九品类市场排行",
|
||||
},
|
||||
{
|
||||
"platform": "jd",
|
||||
"feishu_doc_url": "https://example.com/jd",
|
||||
"product_count": 1050,
|
||||
"report_title": "京东市场排行",
|
||||
},
|
||||
{
|
||||
"platform": "dy",
|
||||
"feishu_doc_url": "https://example.com/dy",
|
||||
"product_count": 1285,
|
||||
"report_title": "抖音市场排行",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_build_notification_message_contains_three_plain_links():
|
||||
message = notification.build_notification_message(date(2026, 7, 26), REPORTS)
|
||||
|
||||
assert "三平台市场排行" in message
|
||||
assert "天猫：https://example.com/tm" in message
|
||||
assert "京东：https://example.com/jd" in message
|
||||
assert "抖音：https://example.com/dy" in message
|
||||
assert "1178" in message
|
||||
|
||||
|
||||
def test_build_notification_message_supports_partial_platform_success():
|
||||
message = notification.build_notification_message(
|
||||
date(2026, 7, 26),
|
||||
[REPORTS[1]],
|
||||
)
|
||||
|
||||
assert "京东：https://example.com/jd" in message
|
||||
assert "天猫：https://" not in message
|
||||
assert "抖音：https://" not in message
|
||||
assert "当前成功平台 1 个" in message
|
||||
|
||||
|
||||
def test_build_hermes_payload_requires_direct_message_to_recipient():
|
||||
payload = notification.build_hermes_payload(
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
message="三个链接",
|
||||
)
|
||||
|
||||
encoded = json.dumps(payload, ensure_ascii=False)
|
||||
assert "ou_wangyunlong" in encoded
|
||||
assert "飞书私聊" in encoded
|
||||
assert "三个链接" in encoded
|
||||
|
||||
|
||||
def test_extracts_real_feishu_message_id_from_hermes_completion():
|
||||
response = {
|
||||
"id": "api-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功，消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
assert (
|
||||
notification.extract_feishu_message_id(response)
|
||||
== "om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
)
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_hermes_success_text_without_real_message_id_is_not_success():
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功，但没有返回消息回执"}}]
|
||||
}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) is None
|
||||
assert notification.hermes_send_succeeded(response) is False
|
||||
|
||||
|
||||
def test_extracts_nested_feishu_message_id_and_accepts_safe_suffixes():
|
||||
response = {"data": {"message_id": "om_real-id_123"}}
|
||||
|
||||
assert notification.extract_feishu_message_id(response) == "om_real-id_123"
|
||||
assert notification.hermes_send_succeeded(response) is True
|
||||
|
||||
|
||||
def test_notify_skips_when_same_day_recipient_was_already_sent():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=True),
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "already_sent"
|
||||
post_hermes.assert_not_called()
|
||||
|
||||
|
||||
def test_notify_sends_only_after_all_three_links_exist_and_persists_success():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"id": "hermes-response-id",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "发送成功，消息ID om_x100b696f1169f0a0b1f9c311e8dc323"
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(
|
||||
notification,
|
||||
"get_market_rank_reports_for_date",
|
||||
return_value=REPORTS,
|
||||
) as get_reports,
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
cutoff = datetime(2026, 7, 26, 10, 0, 0)
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
generated_after=cutoff,
|
||||
)
|
||||
|
||||
assert result["status"] == "sent"
|
||||
get_reports.assert_called_once_with(
|
||||
conn,
|
||||
date(2026, 7, 26),
|
||||
notification.PLATFORM_ORDER,
|
||||
updated_after=cutoff,
|
||||
)
|
||||
post_hermes.assert_called_once()
|
||||
saved = upsert.call_args.args[1]
|
||||
assert saved["status"] == "sent"
|
||||
assert saved["platform_links"] == {
|
||||
"tm": "https://example.com/tm",
|
||||
"jd": "https://example.com/jd",
|
||||
"dy": "https://example.com/dy",
|
||||
}
|
||||
|
||||
|
||||
def test_notify_sends_available_links_when_other_platforms_are_missing():
|
||||
conn = MagicMock()
|
||||
response = {
|
||||
"choices": [{"message": {"content": "发送成功，消息ID om_partial123"}}]
|
||||
}
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "market_rank_notification_was_sent", return_value=False),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=REPORTS[:2]),
|
||||
patch.object(notification, "post_hermes", return_value=response) as post_hermes,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
)
|
||||
|
||||
assert result["status"] == "sent"
|
||||
assert result["missing_platforms"] == ["dy"]
|
||||
post_hermes.assert_called_once()
|
||||
assert upsert.call_args.args[1]["status"] == "sent"
|
||||
|
||||
|
||||
def test_notify_skips_only_when_current_run_has_no_links():
|
||||
conn = MagicMock()
|
||||
with (
|
||||
patch.object(notification, "get_conn") as get_conn,
|
||||
patch.object(notification, "ensure_market_rank_notification_table"),
|
||||
patch.object(notification, "get_market_rank_reports_for_date", return_value=[]),
|
||||
patch.object(notification, "market_rank_notification_was_sent") as was_sent,
|
||||
patch.object(notification, "post_hermes") as post_hermes,
|
||||
patch.object(notification, "upsert_market_rank_notification") as upsert,
|
||||
):
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
result = notification.notify_market_rank_reports(
|
||||
report_date=date(2026, 7, 26),
|
||||
recipient_open_id="ou_wangyunlong",
|
||||
)
|
||||
|
||||
assert result["status"] == "skipped"
|
||||
assert result["reason"] == "no_platform_reports"
|
||||
was_sent.assert_not_called()
|
||||
post_hermes.assert_not_called()
|
||||
assert upsert.call_args.args[1]["status"] == "incomplete"
|
||||
@@ -0,0 +1,61 @@
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce.market_rank_limits import (
|
||||
category_rank_limit,
|
||||
remaining_category_rank_limit,
|
||||
require_valid_market_rank_products,
|
||||
)
|
||||
|
||||
|
||||
def test_tmall_and_douyin_collect_top_200_per_category():
|
||||
assert category_rank_limit("tm", "旅行箱") == 200
|
||||
assert category_rank_limit("tm", "双肩包") == 200
|
||||
assert category_rank_limit("dy", "旅行箱") == 200
|
||||
assert category_rank_limit("dy", "电脑包") == 200
|
||||
|
||||
|
||||
def test_jd_collects_top_150_for_luggage_and_top_100_for_other_categories():
|
||||
assert category_rank_limit("jd", "旅行箱") == 150
|
||||
assert category_rank_limit("jd", "运动包") == 100
|
||||
assert category_rank_limit("jd", "双肩包") == 100
|
||||
assert category_rank_limit("jd", "电脑包") == 100
|
||||
|
||||
|
||||
def test_remaining_limit_is_per_category_and_respects_optional_global_limit():
|
||||
assert remaining_category_rank_limit("tm", "双肩包", category_count=80) == 120
|
||||
assert remaining_category_rank_limit("jd", "旅行箱", category_count=120) == 30
|
||||
assert remaining_category_rank_limit("jd", "电脑包", category_count=100) == 0
|
||||
assert remaining_category_rank_limit(
|
||||
"dy",
|
||||
"双肩包",
|
||||
category_count=20,
|
||||
total_count=490,
|
||||
global_limit=500,
|
||||
) == 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "rows"),
|
||||
[
|
||||
("tm", ["not-a-product-row"]),
|
||||
("jd", [{"name": "", "rank": 1, "sales_amount_upper": 10}]),
|
||||
("dy", [{"name": "商品", "rank": 1, "sales_amount_upper": 0}]),
|
||||
],
|
||||
)
|
||||
def test_zero_valid_platform_results_are_never_success(platform, rows):
|
||||
with pytest.raises(RuntimeError, match="0 条有效商品.*拒绝标记成功"):
|
||||
require_valid_market_rank_products(platform, rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "metric_name"),
|
||||
[
|
||||
("tm", "buyer_max"),
|
||||
("jd", "sales_amount_upper"),
|
||||
("dy", "sales_amount_upper"),
|
||||
],
|
||||
)
|
||||
def test_valid_platform_result_passes_gate(platform, metric_name):
|
||||
row = {"name": "有效商品", "rank": 1, metric_name: 100}
|
||||
|
||||
assert require_valid_market_rank_products(platform, [row]) == 1
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from gyxx_flow.modules.product_commerce.market_rank_product_import import (
|
||||
build_product_snapshot,
|
||||
extract_document_labels,
|
||||
normalize_scene_name,
|
||||
normalize_style_name,
|
||||
)
|
||||
|
||||
RUNTIME_ROOT = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "product_commerce"
|
||||
)
|
||||
|
||||
|
||||
def test_known_label_aliases_are_normalized_without_guessing_unknown_styles():
|
||||
assert normalize_style_name("都是新旅") == "都市新旅"
|
||||
assert normalize_style_name("智性通勤") == "智性通勤"
|
||||
assert normalize_style_name("都市运动") == "都市运动"
|
||||
assert normalize_style_name("都市通勤") is None
|
||||
assert normalize_scene_name("上班通勤") == "通勤上班"
|
||||
assert normalize_scene_name("摄影爱好") == "摄影爱好"
|
||||
|
||||
|
||||
def test_tmall_snapshot_uses_buyer_upper_times_cached_unit_price():
|
||||
snapshot = build_product_snapshot(
|
||||
"tm",
|
||||
{
|
||||
"rank": 1,
|
||||
"name": "旅行箱",
|
||||
"buyer_range": "100 ~ 250",
|
||||
"buyer_max": 250,
|
||||
"product_url": "https://detail.tmall.com/item.htm?id=123",
|
||||
"category_name": "旅行箱",
|
||||
"category_path": "箱包 > 旅行箱",
|
||||
},
|
||||
price_cache={"123": "¥418"},
|
||||
)
|
||||
|
||||
assert snapshot["platform_product_id"] == "123"
|
||||
assert snapshot["unit_price_min"] == Decimal("418")
|
||||
assert snapshot["unit_price_max"] == Decimal("418")
|
||||
assert snapshot["estimated_gmv"] == Decimal("104500")
|
||||
assert snapshot["estimate_method"] == "buyer_upper_x_unit_price"
|
||||
assert snapshot["quality_status"] == "complete"
|
||||
|
||||
|
||||
def test_douyin_and_jd_snapshots_use_reported_sales_upper():
|
||||
for platform in ("dy", "jd"):
|
||||
snapshot = build_product_snapshot(
|
||||
platform,
|
||||
{
|
||||
"product_id": "9988",
|
||||
"name": "商品",
|
||||
"sales_amount_range": "¥100万-¥250万",
|
||||
"sales_amount_upper": 2_500_000,
|
||||
"price_range": "¥398-¥617",
|
||||
"product_url": "https://example.com/9988",
|
||||
"category_name": "双肩包",
|
||||
},
|
||||
)
|
||||
|
||||
assert snapshot["estimated_gmv"] == Decimal("2500000")
|
||||
assert snapshot["estimate_method"] == "reported_sales_upper"
|
||||
assert snapshot["quality_status"] == "complete"
|
||||
|
||||
|
||||
def test_missing_tmall_price_keeps_product_but_excludes_it_from_amount():
|
||||
snapshot = build_product_snapshot(
|
||||
"tm",
|
||||
{
|
||||
"buyer_max": 100,
|
||||
"name": "没有价格的商品",
|
||||
"product_url": "https://detail.tmall.com/item.htm?id=456",
|
||||
"category_name": "相机包",
|
||||
},
|
||||
price_cache={},
|
||||
)
|
||||
|
||||
assert snapshot["estimated_gmv"] is None
|
||||
assert snapshot["quality_status"] == "missing_price"
|
||||
|
||||
|
||||
def test_feishu_document_labels_keep_raw_values_and_normalized_values():
|
||||
content = """
|
||||
<title>京东榜单</title>
|
||||
<h2>旅行箱（1条）</h2>
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>风格（运营手动区分）</th><th>场景用途（运营手动区分）</th>
|
||||
<th>品类</th><th>排名</th><th>商品图</th><th>商品名称</th>
|
||||
<th>销售额</th><th>商品价格</th><th>商品链接</th>
|
||||
</tr></thead>
|
||||
<tbody><tr>
|
||||
<td><b>都是新旅</b></td><td>上班通勤</td><td>旅行箱</td><td>1</td>
|
||||
<td></td><td>测试商品</td><td>￥50万~￥75万</td><td>￥399</td>
|
||||
<td><a href="https://item.jd.com/123456.html">打开商品</a></td>
|
||||
</tr></tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
labels = extract_document_labels(content, "jd")
|
||||
|
||||
assert labels["123456"]["raw_style_name"] == "都是新旅"
|
||||
assert labels["123456"]["style_name"] == "都市新旅"
|
||||
assert labels["123456"]["raw_scene_name"] == "上班通勤"
|
||||
assert labels["123456"]["scene_name"] == "通勤上班"
|
||||
assert labels["123456"]["unit_price_text"] == "￥399"
|
||||
|
||||
|
||||
def test_collector_schema_contains_idempotent_market_snapshot_tables():
|
||||
schema = (RUNTIME_ROOT / "db" / "schema.sql").read_text(encoding="utf-8")
|
||||
|
||||
assert "CREATE TABLE IF NOT EXISTS market_product" in schema
|
||||
assert "CREATE TABLE IF NOT EXISTS fact_market_product_snapshot" in schema
|
||||
assert "CREATE TABLE IF NOT EXISTS market_product_classification" in schema
|
||||
assert "UNIQUE (report_id, market_product_id, category_name)" in schema
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from unittest.mock import patch
|
||||
|
||||
from db import upsert_market_rank_report
|
||||
from market_rank_report_archive import (
|
||||
archive_market_rank_report,
|
||||
extract_feishu_doc_id,
|
||||
normalize_market_rank_platform,
|
||||
)
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self):
|
||||
self.sql = ""
|
||||
self.params = ()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
def fetchone(self):
|
||||
return (37,)
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.cursor_instance = FakeCursor()
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def test_extract_feishu_doc_id_from_docx_url():
|
||||
assert (
|
||||
extract_feishu_doc_id(
|
||||
"https://bu0zgpibak.feishu.cn/docx/ZPCYdLtvfoksGmxy5zcc8E7anzg"
|
||||
)
|
||||
== "ZPCYdLtvfoksGmxy5zcc8E7anzg"
|
||||
)
|
||||
|
||||
|
||||
def test_platform_names_are_normalized_to_database_codes():
|
||||
assert normalize_market_rank_platform("天猫") == "tm"
|
||||
assert normalize_market_rank_platform("jd") == "jd"
|
||||
assert normalize_market_rank_platform("抖音") == "dy"
|
||||
|
||||
|
||||
def test_upsert_market_rank_report_is_idempotent_per_platform_and_day():
|
||||
conn = FakeConnection()
|
||||
|
||||
report_id = upsert_market_rank_report(
|
||||
conn,
|
||||
{
|
||||
"platform": "tm",
|
||||
"report_date": date(2026, 7, 24),
|
||||
"period": "30天",
|
||||
"category_scope": "九品类",
|
||||
"category_names": ["旅行箱", "运动包"],
|
||||
"report_title": "天猫九品类市场排行 30天 2026-07-24",
|
||||
"feishu_doc_id": "doc-token",
|
||||
"feishu_doc_url": "https://example.feishu.cn/docx/doc-token",
|
||||
"product_count": 146,
|
||||
"source_json_path": "data/sycm_market_rank/result.json",
|
||||
"run_metadata": {"min_price": 400, "max_price": 2000},
|
||||
},
|
||||
)
|
||||
|
||||
assert report_id == 37
|
||||
assert "ON CONFLICT (platform, report_date)" in conn.cursor_instance.sql
|
||||
assert "ON CONFLICT (platform, report_date, period, category_scope)" not in conn.cursor_instance.sql
|
||||
assert conn.cursor_instance.params[0] == "tm"
|
||||
assert conn.cursor_instance.params[4] == ["旅行箱", "运动包"]
|
||||
assert conn.cursor_instance.params[7] == "https://example.feishu.cn/docx/doc-token"
|
||||
assert conn.cursor_instance.params[8] == 146
|
||||
|
||||
|
||||
def test_archive_market_rank_report_builds_and_saves_normalized_record():
|
||||
with (
|
||||
patch("market_rank_report_archive.get_conn") as get_conn,
|
||||
patch(
|
||||
"market_rank_report_archive.upsert_market_rank_report",
|
||||
return_value=88,
|
||||
) as upsert,
|
||||
):
|
||||
conn = object()
|
||||
get_conn.return_value.__enter__.return_value = conn
|
||||
|
||||
report_id = archive_market_rank_report(
|
||||
platform="京东",
|
||||
report_date=date(2026, 7, 24),
|
||||
period="近30天",
|
||||
category_scope="七品类",
|
||||
category_names=["旅行箱", "双肩包"],
|
||||
report_title="京东七品类市场商品榜单 近30天 2026-07-24",
|
||||
feishu_doc_url="https://example.feishu.cn/docx/jd-doc-token",
|
||||
product_count=70,
|
||||
source_json_path="data/jd_market_rank/result.json",
|
||||
run_metadata={"max_pages": 20},
|
||||
)
|
||||
|
||||
assert report_id == 88
|
||||
record = upsert.call_args.args[1]
|
||||
assert upsert.call_args.args[0] is conn
|
||||
assert record["platform"] == "jd"
|
||||
assert record["feishu_doc_id"] == "jd-doc-token"
|
||||
assert record["category_names"] == ["旅行箱", "双肩包"]
|
||||
assert record["run_metadata"] == {"max_pages": 20}
|
||||
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
from threading import Barrier
|
||||
|
||||
import orchestrate_market_rank_collection as workflow
|
||||
import pytest
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
PROJECT_ROOT = (
|
||||
REPOSITORY_ROOT / "src" / "gyxx_flow" / "modules" / "product_commerce"
|
||||
)
|
||||
HISTORY_ROOT = REPOSITORY_ROOT / "docs" / "history" / "product-commerce"
|
||||
|
||||
|
||||
def test_default_platform_order_and_commands():
|
||||
assert workflow.DEFAULT_PLATFORMS == ("tm", "jd", "dy")
|
||||
report_date = date(2026, 8, 4)
|
||||
|
||||
tm_command = workflow.build_platform_command(
|
||||
"tm", report_date=report_date, python_executable=sys.executable
|
||||
)
|
||||
jd_command = workflow.build_platform_command(
|
||||
"jd", report_date=report_date, python_executable=sys.executable
|
||||
)
|
||||
dy_command = workflow.build_platform_command(
|
||||
"dy", report_date=report_date, python_executable=sys.executable
|
||||
)
|
||||
|
||||
assert tm_command[0] == sys.executable
|
||||
assert Path(tm_command[1]).name == "collect_sycm_market_rank.py"
|
||||
assert tm_command[-2:] == ["--risk-mode", "manual"]
|
||||
assert jd_command[0] == sys.executable
|
||||
assert Path(jd_command[1]).name == "collect_jd_market_rank.py"
|
||||
assert dy_command[0] == sys.executable
|
||||
assert Path(dy_command[1]).name == "collect_dy_market_rank.py"
|
||||
for command in (tm_command, jd_command, dy_command):
|
||||
assert command[command.index("--report-date") + 1] == "2026-08-04"
|
||||
|
||||
|
||||
def test_historical_live_snapshot_date_is_rejected():
|
||||
today = date(2026, 8, 4)
|
||||
|
||||
with pytest.raises(ValueError, match="拒绝错标"):
|
||||
workflow.validate_live_snapshot_date(
|
||||
today - timedelta(days=1),
|
||||
current_date=today,
|
||||
)
|
||||
|
||||
|
||||
def test_notification_recipient_comes_only_from_runtime_environment():
|
||||
assert workflow.notification_recipient_from_env({}) == ("", "")
|
||||
assert workflow.notification_recipient_from_env(
|
||||
{
|
||||
"GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_runtime",
|
||||
"GYXX_NOTIFICATION_RECIPIENT_NAME": "runtime-owner",
|
||||
}
|
||||
) == ("ou_runtime", "runtime-owner")
|
||||
assert workflow.notification_recipient_from_env(
|
||||
{
|
||||
"MARKET_RANK_NOTIFY_OPEN_ID": "ou_market",
|
||||
"MARKET_RANK_NOTIFY_NAME": "market-owner",
|
||||
"GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_generic",
|
||||
}
|
||||
) == ("ou_market", "market-owner")
|
||||
|
||||
|
||||
def test_child_environment_forces_utf8_unbuffered_output():
|
||||
env = workflow.build_subprocess_env({"EXISTING": "kept"})
|
||||
|
||||
assert env["EXISTING"] == "kept"
|
||||
assert env["PYTHONIOENCODING"] == "utf-8"
|
||||
assert env["PYTHONUNBUFFERED"] == "1"
|
||||
|
||||
|
||||
def test_console_text_replaces_characters_not_supported_by_gbk():
|
||||
assert workflow.console_safe_text("prefix \ufffd suffix", "gbk") == "prefix ? suffix"
|
||||
|
||||
|
||||
def test_workflow_continues_after_one_platform_fails(tmp_path):
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_executor(*, platform, command, log_path, dry_run):
|
||||
calls.append(platform)
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-07-26T10:00:00",
|
||||
"finished_at": "2026-07-26T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 1 if platform == "tm" else 0,
|
||||
"status": "failed" if platform == "tm" else "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="test-run",
|
||||
log_dir=tmp_path,
|
||||
executor=fake_executor,
|
||||
)
|
||||
|
||||
assert set(calls) == {"tm", "jd", "dy"}
|
||||
assert [item["platform"] for item in summary["steps"]] == ["tm", "jd", "dy"]
|
||||
assert summary["exit_code"] == 1
|
||||
assert [item["status"] for item in summary["steps"]] == [
|
||||
"failed", "success", "success"
|
||||
]
|
||||
saved = json.loads((tmp_path / "test-run_summary.json").read_text(encoding="utf-8"))
|
||||
assert saved["run_id"] == "test-run"
|
||||
assert saved["exit_code"] == 1
|
||||
|
||||
|
||||
def test_workflow_continues_when_executor_raises(tmp_path):
|
||||
calls: list[str] = []
|
||||
|
||||
def raising_executor(*, platform, command, log_path, dry_run):
|
||||
calls.append(platform)
|
||||
if platform == "tm":
|
||||
raise RuntimeError("browser startup failed")
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-07-26T10:00:00",
|
||||
"finished_at": "2026-07-26T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="raised-run",
|
||||
log_dir=tmp_path,
|
||||
executor=raising_executor,
|
||||
)
|
||||
|
||||
assert set(calls) == {"tm", "jd", "dy"}
|
||||
assert summary["exit_code"] == 1
|
||||
assert summary["steps"][0]["status"] == "failed"
|
||||
assert "browser startup failed" in summary["steps"][0]["error"]
|
||||
|
||||
|
||||
def test_three_platforms_start_in_parallel(tmp_path):
|
||||
all_started = Barrier(3)
|
||||
|
||||
def synchronized_executor(*, platform, command, log_path, dry_run):
|
||||
all_started.wait(timeout=1)
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-07-26T10:00:00",
|
||||
"finished_at": "2026-07-26T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="parallel-run",
|
||||
log_dir=tmp_path,
|
||||
executor=synchronized_executor,
|
||||
)
|
||||
|
||||
assert summary["exit_code"] == 0
|
||||
assert [item["platform"] for item in summary["steps"]] == ["tm", "jd", "dy"]
|
||||
|
||||
|
||||
def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path):
|
||||
all_started = Barrier(3)
|
||||
notifications: list[dict] = []
|
||||
report_date = date.today()
|
||||
|
||||
def synchronized_executor(*, platform, command, log_path, dry_run):
|
||||
all_started.wait(timeout=1)
|
||||
return {
|
||||
"platform": platform,
|
||||
"label": workflow.PLATFORM_LABELS[platform],
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-07-26T10:00:00",
|
||||
"finished_at": "2026-07-26T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
def fake_notifier(**kwargs):
|
||||
notifications.append(kwargs)
|
||||
return {"status": "sent", "recipient_open_id": kwargs["recipient_open_id"]}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="notify-run",
|
||||
log_dir=tmp_path,
|
||||
executor=synchronized_executor,
|
||||
report_date=report_date,
|
||||
notify_open_id="ou_wangyunlong",
|
||||
notify_name="runtime-owner",
|
||||
notifier=fake_notifier,
|
||||
)
|
||||
|
||||
assert len(notifications) == 1
|
||||
assert notifications[0]["report_date"] == report_date
|
||||
assert notifications[0]["recipient_open_id"] == "ou_wangyunlong"
|
||||
assert notifications[0]["recipient_name"] == "runtime-owner"
|
||||
assert notifications[0]["generated_after"] is not None
|
||||
for step in summary["steps"]:
|
||||
command = step["command"]
|
||||
assert command[command.index("--report-date") + 1] == report_date.isoformat()
|
||||
assert summary["notification"]["status"] == "sent"
|
||||
saved = json.loads((tmp_path / "notify-run_summary.json").read_text(encoding="utf-8"))
|
||||
assert saved["notification"]["status"] == "sent"
|
||||
|
||||
|
||||
def test_partial_current_run_links_can_still_send_successfully(tmp_path):
|
||||
def successful_executor(*, platform, command, log_path, dry_run):
|
||||
return {
|
||||
"platform": platform,
|
||||
"command": command,
|
||||
"log_path": str(log_path),
|
||||
"started_at": "2026-07-26T10:00:00",
|
||||
"finished_at": "2026-07-26T10:01:00",
|
||||
"duration_seconds": 60.0,
|
||||
"exit_code": 0,
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
summary = workflow.execute_workflow(
|
||||
workflow.DEFAULT_PLATFORMS,
|
||||
run_id="missing-link-run",
|
||||
log_dir=tmp_path,
|
||||
executor=successful_executor,
|
||||
notify_open_id="ou_wangyunlong",
|
||||
notifier=lambda **kwargs: {
|
||||
"status": "sent",
|
||||
"platform_links": {"jd": "https://example.com/jd"},
|
||||
"missing_platforms": ["tm", "dy"],
|
||||
},
|
||||
)
|
||||
|
||||
assert summary["exit_code"] == 0
|
||||
assert summary["status"] == "success"
|
||||
|
||||
|
||||
def test_platform_parser_deduplicates_while_preserving_order():
|
||||
assert workflow.parse_platforms("dy,tm,dy,jd") == ("dy", "tm", "jd")
|
||||
|
||||
|
||||
def test_main_requires_runtime_recipient_unless_notification_disabled(monkeypatch):
|
||||
monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "")
|
||||
monkeypatch.setattr(sys, "argv", ["orchestrate_market_rank_collection.py", "--dry-run"])
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
workflow.main()
|
||||
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
|
||||
def test_main_allows_explicit_no_notify_without_recipient(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "")
|
||||
observed = {}
|
||||
|
||||
def fake_execute(*args, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return {"exit_code": 0}
|
||||
|
||||
monkeypatch.setattr(workflow, "execute_workflow", fake_execute)
|
||||
monkeypatch.setattr(workflow, "managed_data_path", Path)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"orchestrate_market_rank_collection.py",
|
||||
"--dry-run",
|
||||
"--no-notify",
|
||||
"--log-dir",
|
||||
str(tmp_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert workflow.main() == 0
|
||||
assert observed["notify_open_id"] is None
|
||||
|
||||
|
||||
def test_weekly_launcher_uses_market_rank_orchestrator():
|
||||
launcher = (
|
||||
HISTORY_ROOT / "launchers_reference" / "run_weekly_market_rank.bat"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert ".venv\\Scripts\\python.exe" in launcher
|
||||
assert "orchestrate_market_rank_collection.py" in launcher
|
||||
@@ -0,0 +1,319 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import collect_dy_persona_to_bitable as dy_worker
|
||||
import collect_jd_persona_to_bitable as jd_worker
|
||||
import collect_persona_to_bitable as tm_worker
|
||||
import dy_audience_profile_collect as dy_collector
|
||||
import pytest
|
||||
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
||||
|
||||
from gyxx_flow.adapters import acceptance_policy
|
||||
|
||||
WORKERS = (tm_worker, dy_worker, jd_worker)
|
||||
BUSINESS_DATE = date(2026, 8, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_platform_collector_failure_forces_nonzero(worker) -> None:
|
||||
assert worker.combined_result_exit_code(
|
||||
[{"status": "ok"}],
|
||||
collection_ok=False,
|
||||
) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_upsert_requires_response_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"ok": True, "data": {}}),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert "record_id" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"data": {"record": {"record_id_list": ["rec12345678"]}},
|
||||
}
|
||||
),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is True
|
||||
assert message == "record_id=rec12345678"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: (_ for _ in ()).throw(
|
||||
AssertionError("lark subprocess must not run after acceptance skip")
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert "acceptance-skipped" in message
|
||||
|
||||
|
||||
def test_tm_payload_must_match_requested_business_date(monkeypatch, tmp_path: Path) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
(style_dir / f"{style}_123_chart_values.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"style_name": style,
|
||||
"item_id": "123",
|
||||
"business_date": "2026-08-03",
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(tm_worker, "DMP_OUTPUT_ROOT", tmp_path)
|
||||
|
||||
assert tm_worker.load_latest_chart_values(style, BUSINESS_DATE) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", (dy_worker, jd_worker))
|
||||
def test_profile_payload_must_match_requested_business_date(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
worker,
|
||||
) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
(style_dir / f"{style}_123_profile.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"style_name": style,
|
||||
"product_id": "123",
|
||||
"business_date": "2026-08-03",
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
||||
|
||||
assert worker.load_latest_profile(style, BUSINESS_DATE) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", (tm_worker, dy_worker))
|
||||
def test_current_run_does_not_reuse_old_artifact(monkeypatch, tmp_path: Path, worker) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
if worker is tm_worker:
|
||||
path = style_dir / f"{style}_123_chart_values.json"
|
||||
payload = {
|
||||
"style_name": style,
|
||||
"item_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
}
|
||||
monkeypatch.setattr(worker, "DMP_OUTPUT_ROOT", tmp_path)
|
||||
load = worker.load_latest_chart_values
|
||||
else:
|
||||
path = style_dir / f"{style}_123_profile.json"
|
||||
payload = {
|
||||
"style_name": style,
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
}
|
||||
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
||||
load = worker.load_latest_profile
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
assert load(
|
||||
style,
|
||||
BUSINESS_DATE,
|
||||
artifact_not_before=path.stat().st_mtime + 10,
|
||||
) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_database_failure_stops_before_base_write(monkeypatch, worker) -> None:
|
||||
if worker is tm_worker:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_chart_values",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"item_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"chart_values_to_fields",
|
||||
lambda *_a, **_k: {"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
normalize_name = "normalize_tm_payload"
|
||||
elif worker is dy_worker:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_profile",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(worker, "profile_has_positive_distribution", lambda _profile: True)
|
||||
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
||||
monkeypatch.setattr(worker, "profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
||||
normalize_name = "normalize_dy_payload"
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_profile",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"性别": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
||||
monkeypatch.setattr(worker, "map_profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
||||
normalize_name = "normalize_jd_payload"
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
get_conn=lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")),
|
||||
upsert_persona_metrics=lambda *_a, **_k: 1,
|
||||
**{normalize_name: lambda _payload: {"valid": True}},
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "db", fake_db)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"upsert_record",
|
||||
lambda *_a, **_k: (_ for _ in ()).throw(
|
||||
AssertionError("Base write must not run after DB failure")
|
||||
),
|
||||
)
|
||||
|
||||
result = worker.upsert_one_style(
|
||||
"款A",
|
||||
{"base_token": "bas_test", "table_id": "tbl_test"},
|
||||
"8.4",
|
||||
"user",
|
||||
False,
|
||||
BUSINESS_DATE,
|
||||
)
|
||||
|
||||
assert result["status"] == "db_failed"
|
||||
|
||||
|
||||
def test_zero_distributions_are_not_valid_persona() -> None:
|
||||
assert dy_worker.profile_has_positive_distribution(
|
||||
{
|
||||
"gender_distribution": [{"name": "男", "value": 0}],
|
||||
"age_distribution": [],
|
||||
"strategy_crowd_distribution": [{"name": "Z世代", "value": "0%"}],
|
||||
}
|
||||
) is False
|
||||
|
||||
|
||||
def test_dmp_skips_are_not_valid_style_results() -> None:
|
||||
styles = {"款A": ["123"]}
|
||||
assert dmp.records_exit_code(
|
||||
styles,
|
||||
[{"style_name": "款A", "status": "skipped", "skip_reason": "无图表"}],
|
||||
) == 2
|
||||
assert dmp.records_exit_code(
|
||||
styles,
|
||||
[
|
||||
{
|
||||
"style_name": "款A",
|
||||
"status": "ok",
|
||||
"chart_values": {
|
||||
"用户性别": [{"category": "男性用户", "分析人群占比": 50}]
|
||||
},
|
||||
}
|
||||
],
|
||||
) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("detector", "url", "body", "expected"),
|
||||
(
|
||||
(dmp.blocking_page_reason, "https://dmp.taobao.com/", "暂无权限", "权限"),
|
||||
(dy_collector.blocking_page_reason, "https://compass.test/", "请登录", "登录"),
|
||||
(jd_worker.blocking_page_reason, "https://passport.jd.com/login", "", "登录"),
|
||||
),
|
||||
)
|
||||
def test_login_and_permission_pages_have_explicit_diagnostics(
|
||||
detector,
|
||||
url: str,
|
||||
body: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert expected in detector(url, body)
|
||||
|
||||
|
||||
def test_persona_search_selectors_are_finite_and_evidence_based() -> None:
|
||||
assert 1 <= len(dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS) <= 6
|
||||
assert 1 <= len(jd_worker.JD_SEARCH_INPUT_SELECTORS) <= 6
|
||||
assert all("input" in selector for selector in dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS)
|
||||
assert all("input" in selector for selector in jd_worker.JD_SEARCH_INPUT_SELECTORS)
|
||||
|
||||
|
||||
def test_tm_collector_command_redacts_credentials() -> None:
|
||||
rendered = tm_worker.redacted_command(
|
||||
["python", "collector.py", "--account", "secret-user", "--password", "secret-pass"]
|
||||
)
|
||||
|
||||
assert "secret-user" not in rendered
|
||||
assert "secret-pass" not in rendered
|
||||
assert rendered.count("***") == 2
|
||||
@@ -0,0 +1,57 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import collect_dy_persona_to_bitable as dy_worker
|
||||
import collect_jd_persona_to_bitable as jd_worker
|
||||
import collect_persona_to_bitable as tm_worker
|
||||
import run_daily_persona
|
||||
|
||||
|
||||
class PersonaLauncherTests(unittest.TestCase):
|
||||
def test_any_platform_failure_makes_launcher_fail(self):
|
||||
self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 2, "jd": 0}), 2)
|
||||
|
||||
def test_all_platforms_success_makes_launcher_success(self):
|
||||
self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 0, "jd": 0}), 0)
|
||||
|
||||
def test_launcher_passes_business_date_to_each_worker(self):
|
||||
command = run_daily_persona.build_child_command(
|
||||
Path("collect_persona_to_bitable.py"),
|
||||
date(2026, 8, 4),
|
||||
)
|
||||
|
||||
self.assertEqual(command[-2:], ["--date", "2026-08-04"])
|
||||
|
||||
def test_workers_fail_when_permission_is_skipped(self):
|
||||
results = [{"status": "ok"}, {"status": "permission_skipped"}]
|
||||
|
||||
for worker in (tm_worker, dy_worker, jd_worker):
|
||||
with self.subTest(worker=worker.__name__):
|
||||
self.assertEqual(worker.combined_result_exit_code(results), 2)
|
||||
|
||||
def test_workers_fail_when_no_styles_were_processed(self):
|
||||
for worker in (tm_worker, dy_worker, jd_worker):
|
||||
with self.subTest(worker=worker.__name__):
|
||||
self.assertEqual(worker.combined_result_exit_code([]), 2)
|
||||
|
||||
def test_workers_succeed_only_when_all_styles_are_successful(self):
|
||||
results = [{"status": "ok"}, {"status": "dry-run"}]
|
||||
|
||||
for worker in (tm_worker, dy_worker, jd_worker):
|
||||
with self.subTest(worker=worker.__name__):
|
||||
self.assertEqual(worker.combined_result_exit_code(results), 0)
|
||||
|
||||
def test_workers_fail_when_collector_itself_failed(self):
|
||||
results = [{"status": "ok"}]
|
||||
|
||||
for worker in (tm_worker, dy_worker, jd_worker):
|
||||
with self.subTest(worker=worker.__name__):
|
||||
self.assertEqual(
|
||||
worker.combined_result_exit_code(results, collection_ok=False),
|
||||
2,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,204 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import check_nine_day_decline as decline
|
||||
import insert_bitable_records as insert_records
|
||||
import lark_cli_runtime
|
||||
import market_rank_hermes_notification as market_notification
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
evidence = tmp_path / "product-evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
return evidence
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
[
|
||||
["base", "+record-upsert"],
|
||||
["base", "+record-delete"],
|
||||
["sheets", "+values-batch-update"],
|
||||
],
|
||||
)
|
||||
def test_product_shared_lark_cli_skips_table_mutations(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
arguments: list[str],
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_spawned(*args, **kwargs):
|
||||
raise AssertionError("lark-cli subprocess must not start")
|
||||
|
||||
monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fail_if_spawned)
|
||||
result = lark_cli_runtime.run_lark_cli(arguments)
|
||||
|
||||
assert result["acceptance_skipped"] is True
|
||||
|
||||
|
||||
def test_legacy_product_upsert_bypass_is_physically_skipped(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = _enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_spawned(*args, **kwargs):
|
||||
raise AssertionError("legacy lark-cli subprocess must not start")
|
||||
|
||||
monkeypatch.setattr(insert_records.subprocess, "run", fail_if_spawned)
|
||||
insert_records.run_lark_upsert("base_secret", "tbl_test", {"field": 1})
|
||||
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["operation"].endswith("record-upsert")
|
||||
assert payload["details"]["base_token"] == "<redacted>"
|
||||
|
||||
|
||||
def test_product_shared_lark_cli_keeps_production_write_behavior(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
||||
|
||||
monkeypatch.setattr(lark_cli_runtime, "_node_cli_entry", lambda: ("node", "run.js"))
|
||||
monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fake_run)
|
||||
result = lark_cli_runtime.run_lark_cli(["base", "+record-upsert"])
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert len(calls) == 1
|
||||
assert calls[0][:4] == ["node", "run.js", "--profile", "hermes-analyzer"]
|
||||
|
||||
|
||||
def test_product_shared_lark_cli_allows_safe_profile_override(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
||||
|
||||
monkeypatch.setattr(lark_cli_runtime, "_node_cli_entry", lambda: ("node", "run.js"))
|
||||
monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fake_run)
|
||||
lark_cli_runtime.run_lark_cli(
|
||||
["auth", "status", "--json"],
|
||||
env={"GYXX_LARK_CLI_PROFILE": "product-analyzer.test"},
|
||||
)
|
||||
|
||||
assert calls[0][:4] == [
|
||||
"node",
|
||||
"run.js",
|
||||
"--profile",
|
||||
"product-analyzer.test",
|
||||
]
|
||||
|
||||
|
||||
def test_decline_hermes_prompt_forces_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(decline, "HERMES_API_KEY", "test-only")
|
||||
captured = {}
|
||||
|
||||
class Response:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
|
||||
|
||||
def fake_urlopen(request, timeout):
|
||||
captured["body"] = json.load(io.BytesIO(request.data))
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(decline.urllib.request, "urlopen", fake_urlopen)
|
||||
decline.notify_hermes(
|
||||
[{"style_name": "test", "platform": "tm"}],
|
||||
"2026-08-01",
|
||||
9,
|
||||
openid="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["body"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
|
||||
|
||||
def test_market_rank_notification_forces_wang_yunlong(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_enable_acceptance(monkeypatch, tmp_path)
|
||||
captured = {}
|
||||
|
||||
class ConnectionContext:
|
||||
def __enter__(self):
|
||||
return object()
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(market_notification, "get_conn", ConnectionContext)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"ensure_market_rank_notification_table",
|
||||
lambda conn: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"get_market_rank_reports_for_date",
|
||||
lambda *args, **kwargs: [
|
||||
{
|
||||
"platform": "tm",
|
||||
"feishu_doc_url": "https://example.test/tm",
|
||||
"product_count": 1,
|
||||
}
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"market_rank_notification_was_sent",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
|
||||
def fake_post(payload):
|
||||
captured["payload"] = payload
|
||||
return {"message_id": "om_test"}
|
||||
|
||||
def fake_archive(conn, record):
|
||||
captured["record"] = record
|
||||
|
||||
monkeypatch.setattr(market_notification, "post_hermes", fake_post)
|
||||
monkeypatch.setattr(
|
||||
market_notification,
|
||||
"upsert_market_rank_notification",
|
||||
fake_archive,
|
||||
)
|
||||
|
||||
result = market_notification.notify_market_rank_reports(
|
||||
report_date=decline.date(2026, 8, 1),
|
||||
recipient_open_id="ou_someone_else",
|
||||
)
|
||||
|
||||
encoded = json.dumps(captured["payload"], ensure_ascii=False)
|
||||
assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded
|
||||
assert "ou_someone_else" not in encoded
|
||||
assert result["recipient_open_id"] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
assert (
|
||||
captured["record"]["recipient_open_id"]
|
||||
== WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID
|
||||
)
|
||||
@@ -0,0 +1,232 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import backfill_collect
|
||||
import backfill_one_day
|
||||
import collect_dy_persona_to_bitable as dy_persona_worker
|
||||
import collect_persona_to_bitable as tm_persona_worker
|
||||
import orchestrate_daily_collection as daily
|
||||
import orchestrate_market_rank_collection as market_rank
|
||||
import orchestrate_review_collection as reviews
|
||||
import run_alerts_with_retry as alerts
|
||||
import run_daily_persona as persona
|
||||
import run_weekly_jd_main_image as jd_main_image
|
||||
import run_weekly_main_image as tm_main_image
|
||||
|
||||
|
||||
def _fake_rebind(observed: dict[str, object]):
|
||||
def rebind(target, base_environment):
|
||||
observed["target"] = Path(target)
|
||||
observed["base"] = dict(base_environment)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_SCRIPT_ID": f"product_commerce:{Path(target).name}",
|
||||
"GYXX_BROWSER_CDP_PORT": "child-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
|
||||
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
|
||||
}
|
||||
|
||||
return rebind
|
||||
|
||||
|
||||
def _assert_child_binding(environment: dict[str, str]) -> None:
|
||||
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
|
||||
|
||||
|
||||
def test_persona_builds_one_target_bound_environment_per_collector(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(persona, "environment_for_child_script", _fake_rebind(observed))
|
||||
target = persona.PROJECT_ROOT / "collect_persona_to_bitable.py"
|
||||
|
||||
environment = persona.build_child_environment(target, {"KEEP": "yes"})
|
||||
|
||||
assert observed["target"] == target
|
||||
assert environment["KEEP"] == "yes"
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_persona_workers_rebind_their_nested_browser_collectors(monkeypatch) -> None:
|
||||
cases = (
|
||||
(
|
||||
tm_persona_worker,
|
||||
tm_persona_worker.DMP_SCRIPT,
|
||||
lambda: tm_persona_worker.run_dmp_collect_all(
|
||||
"parent-profile",
|
||||
False,
|
||||
"",
|
||||
"",
|
||||
date(2026, 8, 4),
|
||||
),
|
||||
),
|
||||
(
|
||||
dy_persona_worker,
|
||||
dy_persona_worker.COLLECT_SCRIPT,
|
||||
lambda: dy_persona_worker.run_collect_all(
|
||||
"parent-profile",
|
||||
False,
|
||||
date(2026, 8, 4),
|
||||
),
|
||||
),
|
||||
)
|
||||
for worker, target, invoke in cases:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"environment_for_child_script",
|
||||
_fake_rebind(observed),
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["command"] = command
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(worker.subprocess, "run", fake_run)
|
||||
|
||||
assert invoke() is True
|
||||
assert observed["target"] == target
|
||||
command = observed["command"]
|
||||
assert command[command.index("--date") + 1] == "2026-08-04"
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_daily_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(daily, "environment_for_child_script", _fake_rebind(observed))
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(daily.subprocess, "run", fake_run)
|
||||
target = daily.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
result = daily.run_step("dy", ["python", str(target)], critical=True)
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
market_rank,
|
||||
"environment_for_child_script",
|
||||
_fake_rebind(observed),
|
||||
)
|
||||
target = market_rank.PROJECT_ROOT / "collect_jd_market_rank.py"
|
||||
|
||||
environment = market_rank.build_subprocess_env(
|
||||
{"KEEP": "yes"},
|
||||
target=target,
|
||||
)
|
||||
|
||||
assert observed["target"] == target
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["PYTHONIOENCODING"] == "utf-8"
|
||||
assert environment["PYTHONUNBUFFERED"] == "1"
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed))
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(reviews.subprocess, "run", fake_run)
|
||||
target = reviews.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
result = reviews.run_step("dy", ["python", str(target)], critical=True)
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None:
|
||||
for launcher in (tm_main_image, jd_main_image):
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
launcher,
|
||||
"environment_for_child_script",
|
||||
_fake_rebind(observed),
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(launcher.subprocess, "run", fake_run)
|
||||
target = launcher.PROJECT_ROOT / "collector.py"
|
||||
|
||||
result = launcher.run_step("collect", ["python", str(target)])
|
||||
|
||||
assert result["code"] == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
|
||||
|
||||
def test_alert_wrapper_exposes_target_bound_environment(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(alerts, "environment_for_child_script", _fake_rebind(observed))
|
||||
target = alerts.PROJECT_ROOT / "check_nine_day_decline.py"
|
||||
|
||||
environment = alerts.build_child_environment(target, {"KEEP": "yes"})
|
||||
|
||||
assert observed["target"] == target
|
||||
assert environment["KEEP"] == "yes"
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_backfill_range_rebinds_daily_orchestrator_after_overrides(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
backfill_collect,
|
||||
"environment_for_child_script",
|
||||
_fake_rebind(observed),
|
||||
)
|
||||
target = backfill_collect.PROJECT_ROOT / "orchestrate_daily_collection.py"
|
||||
|
||||
environment = backfill_collect._subprocess_env(target)
|
||||
|
||||
assert observed["target"] == target
|
||||
assert observed["base"]["KEEP_BROWSER_OPEN"] == "0"
|
||||
assert environment["KEEP_BROWSER_OPEN"] == "0"
|
||||
_assert_child_binding(environment)
|
||||
|
||||
|
||||
def test_backfill_day_rebinds_each_stage_target(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
monkeypatch.setattr(
|
||||
backfill_one_day,
|
||||
"environment_for_child_script",
|
||||
_fake_rebind(observed),
|
||||
)
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
observed["environment"] = kwargs["env"]
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(backfill_one_day.subprocess, "run", fake_run)
|
||||
target = backfill_one_day.PROJECT_ROOT / "dy_product_scraping.py"
|
||||
|
||||
code, _elapsed = backfill_one_day.run_subprocess(
|
||||
"dy",
|
||||
["python", str(target)],
|
||||
"2026-07-31",
|
||||
)
|
||||
|
||||
assert code == 0
|
||||
assert observed["target"] == target
|
||||
_assert_child_binding(observed["environment"])
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _workflow(workflow_id: str) -> dict:
|
||||
catalog = json.loads(
|
||||
(REPOSITORY_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
||||
)
|
||||
return next(item for item in catalog["workflows"] if item["id"] == workflow_id)
|
||||
|
||||
|
||||
def test_product_workflows_forward_the_selected_business_date() -> None:
|
||||
persona = _workflow("product.persona.daily")["execution"]["steps"][0]
|
||||
style = _workflow("product.style_analysis.interval")["execution"]["steps"][0]
|
||||
main_image = _workflow("product.main_image.weekly")["execution"]["steps"]
|
||||
market_rank = _workflow("product.market_rank")["execution"]["steps"][0]
|
||||
monthly_sales = _workflow("product.sales_sheet.daily")["execution"]["steps"][0]
|
||||
|
||||
assert persona["args"] == ["--date", "{business_date}"]
|
||||
assert style["args"][-2:] == ["--end-date", "{business_date}"]
|
||||
assert all(
|
||||
step["args"][:2] == ["--date", "{business_date}"]
|
||||
for step in main_image
|
||||
)
|
||||
assert market_rank["args"] == ["--report-date", "{business_date}"]
|
||||
assert monthly_sales["args"] == [
|
||||
"--month",
|
||||
"{business_date}",
|
||||
"--execute",
|
||||
"--allow-missing-erp-as-zero",
|
||||
]
|
||||
@@ -0,0 +1,695 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import analyze_style_with_hermes as analysis
|
||||
import orchestrate_daily_collection as orchestrator
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
HISTORY_ROOT = REPOSITORY_ROOT / "docs" / "history" / "product-commerce"
|
||||
|
||||
|
||||
class StyleAnalysisConfigTests(unittest.TestCase):
|
||||
def test_platform_analysis_url_is_aggregated_by_style(self):
|
||||
loader = StyleConfigLoader()
|
||||
payload = loader._aggregate([
|
||||
{
|
||||
"platform": "天猫",
|
||||
"style": "盖亚斜挎",
|
||||
"style_analysis_bitable_url": (
|
||||
"[分析表](https://example.feishu.cn/base/base_token"
|
||||
"?table=table_id&view=view_id)"
|
||||
),
|
||||
}
|
||||
])
|
||||
|
||||
self.assertEqual(
|
||||
payload["styles"]["盖亚斜挎"]["style_analysis_bitable"],
|
||||
{"base_token": "base_token", "table_id": "table_id", "view_id": "view_id"},
|
||||
)
|
||||
|
||||
|
||||
class StyleAnalysisWindowTests(unittest.TestCase):
|
||||
def test_main_image_analysis_remains_tmall_scoped_after_jd_merge(self):
|
||||
class Cursor:
|
||||
sql = ""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql, params):
|
||||
self.sql = sql
|
||||
self.params = params
|
||||
|
||||
@staticmethod
|
||||
def fetchall():
|
||||
return []
|
||||
|
||||
class Connection:
|
||||
def __init__(self, cursor):
|
||||
self._cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self._cursor
|
||||
|
||||
cursor = Cursor()
|
||||
with patch.object(analysis, "get_conn", return_value=Connection(cursor)):
|
||||
rows = analysis.pull_main_image(
|
||||
"同款", date(2026, 7, 1), date(2026, 7, 3)
|
||||
)
|
||||
|
||||
self.assertEqual(rows, [])
|
||||
self.assertIn("platform='tm'", cursor.sql)
|
||||
|
||||
def test_three_day_label_uses_compact_end_date(self):
|
||||
self.assertEqual(
|
||||
analysis.analysis_period_label(date(2026, 7, 13), date(2026, 7, 15)),
|
||||
"2026-07-13~07-15",
|
||||
)
|
||||
|
||||
def test_due_styles_wait_three_full_days(self):
|
||||
styles = ["A", "B", "C"]
|
||||
latest = {"A": date(2026, 7, 15), "B": date(2026, 7, 12)}
|
||||
|
||||
due = analysis.select_due_styles(styles, latest, date(2026, 7, 15), 3)
|
||||
|
||||
self.assertEqual(due, ["B", "C"])
|
||||
|
||||
def test_prompt_keeps_all_dimensions_inside_current_three_day_window(self):
|
||||
payload = {
|
||||
"total": {"sales": 30, "visitors": 1000, "cart": 80},
|
||||
"platforms": {"tm": {"sales": 20}, "jd": {"sales": 10}},
|
||||
"daily": [{"date": "2026-07-14", "plat": "tm", "sales": 10}],
|
||||
"persona": [{"date": "2026-07-14", "plat": "tm", "gender": []}],
|
||||
"product_reviews": {"tm|negative": {"n": 2, "samples": ["肩带偏硬"]}},
|
||||
"main_image": [{"date": "2026-07-14", "key": "hero", "ctr": "2.1"}],
|
||||
"marketing": {
|
||||
"notes_by_platform_all": [{"platform": "xiaohongshu", "views": 1000}],
|
||||
"note_comments_sample": [{"content": "求尺寸"}],
|
||||
},
|
||||
}
|
||||
|
||||
prompt = analysis.build_user_prompt(
|
||||
"盖亚斜挎", date(2026, 7, 14), date(2026, 7, 16), payload,
|
||||
)
|
||||
|
||||
self.assertIn("2026-07-14 ~ 2026-07-16", prompt)
|
||||
self.assertNotIn("对比基期", prompt)
|
||||
self.assertNotIn("紧邻上期", prompt)
|
||||
self.assertNotIn("最近 6 周", prompt)
|
||||
self.assertNotIn("近 60d", prompt)
|
||||
self.assertIn("逐日经营数据", prompt)
|
||||
self.assertIn("人物画像", prompt)
|
||||
self.assertIn("商品评价", prompt)
|
||||
self.assertIn("主图表现", prompt)
|
||||
self.assertIn("营销笔记、曝光互动与评论", prompt)
|
||||
self.assertIn("肩带偏硬", prompt)
|
||||
self.assertIn("求尺寸", prompt)
|
||||
self.assertIn('"sales": 30', prompt)
|
||||
|
||||
def test_system_prompt_requires_each_business_dimension(self):
|
||||
for dimension in ("访客", "加购", "人物画像", "营销笔记", "笔记评论", "商品评价", "主图"):
|
||||
self.assertIn(dimension, analysis.SYSTEM)
|
||||
|
||||
def test_system_prompt_requires_fixed_seven_section_report(self):
|
||||
headings = [
|
||||
"## 一、访客分析",
|
||||
"## 二、加购率分析",
|
||||
"## 三、点击率分析",
|
||||
"## 四、转化率分析",
|
||||
"## 五、退货率分析",
|
||||
"## 六、销量分析",
|
||||
"## 七、总评",
|
||||
]
|
||||
|
||||
for heading in headings:
|
||||
self.assertIn(heading, analysis.SYSTEM)
|
||||
|
||||
def test_plain_seven_section_titles_are_normalized_to_markdown(self):
|
||||
report = """盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)
|
||||
|
||||
一、访客分析
|
||||
内容
|
||||
二、加购率分析
|
||||
内容
|
||||
三、点击率分析
|
||||
内容
|
||||
四、转化率分析
|
||||
内容
|
||||
五、退货率分析
|
||||
内容
|
||||
六、销量分析
|
||||
内容
|
||||
七、总评
|
||||
内容
|
||||
"""
|
||||
|
||||
normalized = analysis.normalize_report_markdown(report)
|
||||
|
||||
self.assertTrue(normalized.startswith("# 盖亚斜挎"))
|
||||
self.assertTrue(analysis.has_exact_report_headings(normalized))
|
||||
|
||||
def test_report_gets_clear_subheadings_and_key_emphasis(self):
|
||||
blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"]
|
||||
for heading in analysis.REPORT_HEADINGS[:-1]:
|
||||
blocks.extend([
|
||||
heading.removeprefix("## "),
|
||||
"数据快照：三日访客2692，转化率0.71%。其余数据。",
|
||||
"问题判断：天猫访客高但转化偏低。需要重点处理。",
|
||||
"可执行建议：P0，24小时内完成核查，目标转化率1.00%。",
|
||||
])
|
||||
blocks.extend([
|
||||
"七、总评",
|
||||
"核心结论：当前核心问题是高访客、低转化。需要优先修复。",
|
||||
"链路表现为：曝光到访客再到成交。",
|
||||
"P0，48小时内完成详情页整改，目标转化率1.00%。",
|
||||
])
|
||||
|
||||
normalized = analysis.normalize_report_markdown("\n\n".join(blocks))
|
||||
|
||||
self.assertTrue(analysis.has_clear_report_hierarchy(normalized))
|
||||
self.assertTrue(analysis.has_key_emphasis(normalized))
|
||||
self.assertIn("### 数据快照", normalized)
|
||||
self.assertIn("### 核心判断", normalized)
|
||||
self.assertIn("### 优先级行动", normalized)
|
||||
self.assertIn("**P0**", normalized)
|
||||
self.assertIn("**24小时内**", normalized)
|
||||
|
||||
def test_extra_helpful_h3_does_not_reject_complete_required_hierarchy(self):
|
||||
required = "\n".join(analysis.REPORT_SUBHEADINGS)
|
||||
report = required.replace(
|
||||
"### 核心判断\n",
|
||||
"### 平台明细\n补充内容\n### 核心判断\n",
|
||||
1,
|
||||
)
|
||||
|
||||
self.assertTrue(analysis.has_clear_report_hierarchy(report))
|
||||
|
||||
def test_missing_calendar_day_is_reported(self):
|
||||
rows = [
|
||||
{"date": "2026-07-14", "plat": "tm"},
|
||||
{"date": "2026-07-16", "plat": "jd"},
|
||||
]
|
||||
|
||||
missing = analysis.missing_window_dates(
|
||||
rows, date(2026, 7, 14), date(2026, 7, 16),
|
||||
)
|
||||
|
||||
self.assertEqual(missing, [date(2026, 7, 15)])
|
||||
|
||||
|
||||
class LarkEnvironmentTests(unittest.TestCase):
|
||||
def test_legacy_lark_config_is_used_when_default_is_missing(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
home = Path(temp_dir)
|
||||
legacy = home / ".lark-cli" / "hermes"
|
||||
legacy.mkdir(parents=True)
|
||||
(legacy / "config.json").write_text("{}", encoding="utf-8")
|
||||
with patch.dict(os.environ, {}, clear=True), patch.object(Path, "home", return_value=home):
|
||||
env = analysis.lark_cli_env()
|
||||
|
||||
self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(legacy))
|
||||
|
||||
def test_root_config_with_analyzer_profile_wins_over_legacy_config(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
home = Path(temp_dir)
|
||||
config_root = home / ".lark-cli"
|
||||
legacy = config_root / "hermes"
|
||||
legacy.mkdir(parents=True)
|
||||
(config_root / "config.json").write_text(
|
||||
json.dumps({"apps": [{"name": "hermes-analyzer"}]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(legacy / "config.json").write_text("{}", encoding="utf-8")
|
||||
with patch.dict(os.environ, {}, clear=True), patch.object(Path, "home", return_value=home):
|
||||
env = analysis.lark_cli_env()
|
||||
|
||||
self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(config_root))
|
||||
|
||||
def test_empty_root_config_falls_back_to_legacy_config(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
home = Path(temp_dir)
|
||||
config_root = home / ".lark-cli"
|
||||
legacy = config_root / "hermes"
|
||||
legacy.mkdir(parents=True)
|
||||
(config_root / "config.json").write_text("{}", encoding="utf-8")
|
||||
(legacy / "config.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
env = analysis.lark_cli_env(base_env={}, home=home)
|
||||
|
||||
self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(legacy))
|
||||
|
||||
def test_explicit_config_directory_is_preserved(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
home = Path(temp_dir)
|
||||
explicit = home / "explicit-lark-config"
|
||||
config_root = home / ".lark-cli"
|
||||
config_root.mkdir(parents=True)
|
||||
(config_root / "config.json").write_text(
|
||||
json.dumps({"profiles": {"hermes-analyzer": {}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = analysis.lark_cli_env(
|
||||
base_env={"LARKSUITE_CLI_CONFIG_DIR": str(explicit)},
|
||||
home=home,
|
||||
)
|
||||
|
||||
self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(explicit))
|
||||
|
||||
|
||||
class FeishuDocumentWriteTests(unittest.TestCase):
|
||||
def test_preflight_refreshes_and_requires_both_identities(self):
|
||||
response = {
|
||||
"verified": True,
|
||||
"identities": {
|
||||
"bot": {"available": True, "verified": True},
|
||||
"user": {"available": True, "verified": True},
|
||||
},
|
||||
}
|
||||
|
||||
with patch.object(analysis, "run_lark_cli", return_value=response) as run:
|
||||
analysis.ensure_lark_cli_ready()
|
||||
|
||||
run.assert_called_once_with(["auth", "status", "--json", "--verify"])
|
||||
|
||||
def test_preflight_rejects_missing_user_identity(self):
|
||||
response = {
|
||||
"verified": True,
|
||||
"identities": {
|
||||
"bot": {"available": True, "verified": True},
|
||||
"user": {"available": False, "verified": False},
|
||||
},
|
||||
}
|
||||
|
||||
with patch.object(analysis, "run_lark_cli", return_value=response):
|
||||
with self.assertRaisesRegex(RuntimeError, "user"):
|
||||
analysis.ensure_lark_cli_ready()
|
||||
|
||||
def test_bot_created_document_is_granted_to_current_user(self):
|
||||
responses = [
|
||||
{"data": {"document": {
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
"document_id": "doc_token",
|
||||
}}},
|
||||
{"identities": {"user": {"openId": "ou_current_user"}}},
|
||||
{"ok": True, "data": {"perm": "full_access"}},
|
||||
]
|
||||
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
|
||||
url, doc_id = analysis.write_feishu_doc("测试分析", "# 内容")
|
||||
|
||||
self.assertEqual(url, "https://example.feishu.cn/docx/doc_token")
|
||||
self.assertEqual(doc_id, "doc_token")
|
||||
create_args = run.call_args_list[0].args[0]
|
||||
self.assertEqual(create_args[create_args.index("--as") + 1], "bot")
|
||||
content_arg = create_args[create_args.index("--content") + 1]
|
||||
self.assertFalse(Path(content_arg.removeprefix("@")).is_absolute())
|
||||
self.assertEqual(
|
||||
run.call_args_list[0].kwargs["cwd"],
|
||||
analysis.report_cache_path("测试分析").parent,
|
||||
)
|
||||
grant_args = run.call_args_list[2].args[0]
|
||||
self.assertIn("+member-add", grant_args)
|
||||
self.assertIn("ou_current_user", grant_args)
|
||||
self.assertIn("full_access", grant_args)
|
||||
self.assertIn("--yes", grant_args)
|
||||
|
||||
def test_confirmed_automatic_permission_grant_skips_duplicate_member_add(self):
|
||||
response = {
|
||||
"data": {"document": {
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
"document_id": "doc_token",
|
||||
}},
|
||||
"permission_grant": {"status": "granted", "perm": "full_access"},
|
||||
}
|
||||
|
||||
with patch.object(analysis, "run_lark_cli", return_value=response) as run:
|
||||
url, doc_id = analysis.write_feishu_doc("测试分析", "# 内容")
|
||||
|
||||
self.assertEqual(url, "https://example.feishu.cn/docx/doc_token")
|
||||
self.assertEqual(doc_id, "doc_token")
|
||||
run.assert_called_once()
|
||||
|
||||
def test_missing_current_user_identity_stops_before_base_write(self):
|
||||
responses = [
|
||||
{"data": {"document": {
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
"document_id": "doc_token",
|
||||
}}},
|
||||
{"identities": {"user": {"ready": False}}},
|
||||
]
|
||||
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses):
|
||||
with self.assertRaisesRegex(RuntimeError, "open_id"):
|
||||
analysis.write_feishu_doc("测试分析", "# 内容")
|
||||
|
||||
|
||||
class LocalReportRecoveryTests(unittest.TestCase):
|
||||
def test_only_complete_exact_window_report_is_reused(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)):
|
||||
title = "测试款 单款式多维度分析 2026-08-01~2026-08-03"
|
||||
report_lines: list[str] = []
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
report_lines.append(heading)
|
||||
for subheading in analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3]:
|
||||
report_lines.extend([subheading, "**重点**"])
|
||||
report = "\n".join(report_lines)
|
||||
cache = analysis.report_cache_path(title)
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache.write_text(report, encoding="utf-8")
|
||||
|
||||
self.assertEqual(
|
||||
analysis.load_validated_cached_report(title),
|
||||
analysis.normalize_report_markdown(report),
|
||||
)
|
||||
|
||||
cache.write_text("## 一、访客分析\n### 数据快照\n**重点**", encoding="utf-8")
|
||||
self.assertIsNone(analysis.load_validated_cached_report(title))
|
||||
|
||||
|
||||
class ExternalEffectCheckpointTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def checkpoint_spec(report: str = "# 内容"):
|
||||
return analysis.StyleAnalysisCheckpointSpec.for_report(
|
||||
"测试款",
|
||||
date(2026, 8, 1),
|
||||
date(2026, 8, 3),
|
||||
3,
|
||||
"测试款 单款式多维度分析 2026-08-01~2026-08-03",
|
||||
report,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def complete_report() -> str:
|
||||
lines: list[str] = []
|
||||
for index, heading in enumerate(analysis.REPORT_HEADINGS):
|
||||
lines.append(heading)
|
||||
for subheading in analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3]:
|
||||
lines.extend([subheading, "**重点**"])
|
||||
return analysis.normalize_report_markdown("\n".join(lines))
|
||||
|
||||
def test_permission_failure_resumes_same_created_document(self):
|
||||
created = {
|
||||
"data": {"document": {
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
"document_id": "doc_token",
|
||||
}},
|
||||
"permission_grant": {"status": "failed"},
|
||||
}
|
||||
auth = {"identities": {"user": {"openId": "ou_current_user"}}}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)):
|
||||
spec = self.checkpoint_spec()
|
||||
with patch.object(
|
||||
analysis,
|
||||
"run_lark_cli",
|
||||
side_effect=[created, auth, RuntimeError("grant failed")],
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "grant failed"):
|
||||
analysis.write_feishu_doc(spec.title, "# 内容", spec)
|
||||
|
||||
state = analysis.load_style_analysis_checkpoint(spec)
|
||||
self.assertEqual(state["phase"], "doc_created")
|
||||
self.assertEqual(state["document"]["id"], "doc_token")
|
||||
|
||||
with patch.object(
|
||||
analysis,
|
||||
"run_lark_cli",
|
||||
side_effect=[auth, {"ok": True}],
|
||||
) as resumed:
|
||||
url, doc_id = analysis.write_feishu_doc(spec.title, "# 内容", spec)
|
||||
|
||||
self.assertEqual(url, "https://example.feishu.cn/docx/doc_token")
|
||||
self.assertEqual(doc_id, "doc_token")
|
||||
self.assertFalse(any(
|
||||
call.args[0][:2] == ["docs", "+create"]
|
||||
for call in resumed.call_args_list
|
||||
))
|
||||
self.assertEqual(
|
||||
analysis.load_style_analysis_checkpoint(spec)["phase"],
|
||||
"permission_granted",
|
||||
)
|
||||
|
||||
def test_crash_after_automatic_grant_resumes_without_member_add(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)):
|
||||
spec = self.checkpoint_spec()
|
||||
analysis.claim_style_analysis_checkpoint(spec)
|
||||
analysis.advance_style_analysis_checkpoint(
|
||||
spec,
|
||||
"doc_created",
|
||||
document={
|
||||
"id": "doc_token",
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
},
|
||||
automatic_permission_status="granted",
|
||||
)
|
||||
|
||||
with patch.object(analysis, "run_lark_cli") as run:
|
||||
url, doc_id = analysis.write_feishu_doc(spec.title, "# 内容", spec)
|
||||
|
||||
self.assertEqual(url, "https://example.feishu.cn/docx/doc_token")
|
||||
self.assertEqual(doc_id, "doc_token")
|
||||
run.assert_not_called()
|
||||
self.assertEqual(
|
||||
analysis.load_style_analysis_checkpoint(spec)["phase"],
|
||||
"permission_granted",
|
||||
)
|
||||
|
||||
def test_ambiguous_and_corrupt_checkpoints_fail_closed(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)):
|
||||
ambiguous = self.checkpoint_spec()
|
||||
state, owns_creation = analysis.claim_style_analysis_checkpoint(ambiguous)
|
||||
self.assertTrue(owns_creation)
|
||||
self.assertEqual(state["phase"], "doc_creating")
|
||||
with patch.object(analysis, "run_lark_cli") as run:
|
||||
with self.assertRaisesRegex(
|
||||
analysis.StyleAnalysisCheckpointError,
|
||||
"doc_creating",
|
||||
):
|
||||
analysis.write_feishu_doc(ambiguous.title, "# 内容", ambiguous)
|
||||
run.assert_not_called()
|
||||
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp) / "corrupt"):
|
||||
corrupt = self.checkpoint_spec()
|
||||
corrupt.path().parent.mkdir(parents=True, exist_ok=True)
|
||||
corrupt.path().write_text("{", encoding="utf-8")
|
||||
with patch.object(analysis, "run_lark_cli") as run:
|
||||
with self.assertRaisesRegex(
|
||||
analysis.StyleAnalysisCheckpointError,
|
||||
"损坏",
|
||||
):
|
||||
analysis.write_feishu_doc(corrupt.title, "# 内容", corrupt)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_report_hash_mismatch_fails_closed(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)):
|
||||
original = self.checkpoint_spec("# 原报告")
|
||||
analysis.claim_style_analysis_checkpoint(original)
|
||||
changed = self.checkpoint_spec("# 新报告")
|
||||
|
||||
with patch.object(analysis, "run_lark_cli") as run:
|
||||
with self.assertRaisesRegex(
|
||||
analysis.StyleAnalysisCheckpointError,
|
||||
"哈希不匹配",
|
||||
):
|
||||
analysis.write_feishu_doc(changed.title, "# 新报告", changed)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_base_and_postgres_phases_resume_without_recreating_document(self):
|
||||
class Connection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
report = self.complete_report()
|
||||
daily = [
|
||||
{"date": "2026-08-01"},
|
||||
{"date": "2026-08-02"},
|
||||
{"date": "2026-08-03"},
|
||||
]
|
||||
created = {
|
||||
"data": {"document": {
|
||||
"url": "https://example.feishu.cn/docx/doc_token",
|
||||
"document_id": "doc_token",
|
||||
}},
|
||||
"permission_grant": {"status": "granted", "perm": "full_access"},
|
||||
}
|
||||
target = {"base_token": "base", "table_id": "table"}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with (
|
||||
patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)),
|
||||
patch.object(analysis, "pull_daily", return_value=daily),
|
||||
patch.object(analysis, "aggregate_period", return_value={"tm": {"sales": 1}}),
|
||||
patch.object(analysis, "aggregate_style_total", return_value={"sales": 1}),
|
||||
patch.object(analysis, "pull_persona", return_value=[]),
|
||||
patch.object(analysis, "pull_reviews", return_value={}),
|
||||
patch.object(analysis, "pull_main_image", return_value=[]),
|
||||
patch.object(analysis, "pull_seeding", return_value={}),
|
||||
patch.object(analysis, "build_user_prompt", return_value="prompt"),
|
||||
patch.object(analysis, "run_lark_cli", return_value=created) as run,
|
||||
patch.object(
|
||||
analysis,
|
||||
"write_analysis_link_to_base",
|
||||
return_value="rec_1",
|
||||
) as write_base,
|
||||
patch.object(analysis, "get_conn", return_value=Connection()),
|
||||
patch.object(
|
||||
analysis,
|
||||
"upsert_style_analysis_report",
|
||||
side_effect=[RuntimeError("pg unavailable"), 42],
|
||||
) as write_pg,
|
||||
):
|
||||
title = "测试款 单款式多维度分析 2026-08-01~2026-08-03"
|
||||
cache = analysis.report_cache_path(title)
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache.write_text(report, encoding="utf-8")
|
||||
spec = self.checkpoint_spec(report)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "pg unavailable"):
|
||||
analysis.analyze_one_style(
|
||||
"测试款",
|
||||
date(2026, 8, 1),
|
||||
date(2026, 8, 3),
|
||||
3,
|
||||
False,
|
||||
target,
|
||||
)
|
||||
base_state = analysis.load_style_analysis_checkpoint(spec)
|
||||
self.assertEqual(base_state["phase"], "base_written")
|
||||
self.assertEqual(base_state["base_record_id"], "rec_1")
|
||||
|
||||
ok, _ = analysis.analyze_one_style(
|
||||
"测试款",
|
||||
date(2026, 8, 1),
|
||||
date(2026, 8, 3),
|
||||
3,
|
||||
False,
|
||||
target,
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
write_base.assert_called_once()
|
||||
self.assertEqual(write_pg.call_count, 2)
|
||||
final_state = analysis.load_style_analysis_checkpoint(spec)
|
||||
self.assertEqual(final_state["phase"], "pg_written")
|
||||
self.assertEqual(final_state["pg_row_id"], 42)
|
||||
|
||||
|
||||
class TargetBaseWriteTests(unittest.TestCase):
|
||||
def test_existing_period_record_is_updated(self):
|
||||
responses = [
|
||||
{"data": {"fields": [
|
||||
{"name": "时间", "type": "select"},
|
||||
{"name": "笔记分析汇总", "type": "text"},
|
||||
]}},
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "2026-07-13~07-15"}],
|
||||
}}},
|
||||
{"data": {
|
||||
"data": [[['2026-07-13~07-15'], "old link"]],
|
||||
"fields": ["时间", "笔记分析汇总"],
|
||||
"record_id_list": ["rec_existing"],
|
||||
}},
|
||||
{"ok": True, "data": {"record_id": "rec_existing"}},
|
||||
]
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
|
||||
record_id = analysis.write_analysis_link_to_base(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"2026-07-13~07-15",
|
||||
"https://example.feishu.cn/docx/doc",
|
||||
"盖亚斜挎",
|
||||
)
|
||||
|
||||
self.assertEqual(record_id, "rec_existing")
|
||||
upsert_args = run.call_args_list[3].args[0]
|
||||
self.assertIn("--record-id", upsert_args)
|
||||
self.assertIn("rec_existing", upsert_args)
|
||||
payload = json.loads(upsert_args[upsert_args.index("--json") + 1])
|
||||
self.assertIn("笔记分析汇总", payload)
|
||||
|
||||
def test_missing_period_is_appended_without_dropping_existing_options(self):
|
||||
responses = [
|
||||
{"data": {"field": {
|
||||
"id": "fld_time", "name": "时间", "type": "select",
|
||||
"multiple": False, "options": [{"name": "7月第二周", "hue": "Blue"}],
|
||||
}}},
|
||||
{"ok": True},
|
||||
]
|
||||
with patch.object(analysis, "run_lark_cli", side_effect=responses) as run:
|
||||
analysis.ensure_select_option(
|
||||
{"base_token": "base", "table_id": "table"},
|
||||
"时间", "2026-07-13~07-15", "盖亚斜挎",
|
||||
)
|
||||
|
||||
update_args = run.call_args_list[1].args[0]
|
||||
definition = json.loads(update_args[update_args.index("--json") + 1])
|
||||
self.assertEqual(
|
||||
[option["name"] for option in definition["options"]],
|
||||
["7月第二周", "2026-07-13~07-15"],
|
||||
)
|
||||
self.assertIn("--yes", update_args)
|
||||
|
||||
|
||||
class OrchestratorTests(unittest.TestCase):
|
||||
def test_style_analysis_stage_runs_three_day_batch(self):
|
||||
args = SimpleNamespace(dry_run=False)
|
||||
with patch.object(orchestrator, "run_step_with_retry") as run:
|
||||
orchestrator.run_style_analysis(args, "2026-07-15", failed=None)
|
||||
|
||||
command = run.call_args.args[1]
|
||||
self.assertIn("analyze_style_with_hermes.py", [Path(item).name for item in command])
|
||||
self.assertIn("--all-styles", command)
|
||||
self.assertEqual(command[command.index("--days") + 1], "3")
|
||||
self.assertEqual(command[command.index("--min-interval-days") + 1], "3")
|
||||
self.assertFalse(run.call_args.kwargs["critical"])
|
||||
|
||||
|
||||
class LauncherScheduleTests(unittest.TestCase):
|
||||
def test_daily_launcher_does_not_run_style_analysis(self):
|
||||
launcher = (
|
||||
HISTORY_ROOT / "launchers_reference" / "run_daily_collect.bat"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
command_lines = [line for line in launcher.splitlines() if "orchestrate_daily_collection.py" in line]
|
||||
self.assertEqual(len(command_lines), 1)
|
||||
self.assertNotIn("style_analysis", command_lines[0])
|
||||
|
||||
def test_three_day_launcher_runs_only_style_analysis(self):
|
||||
launcher = (
|
||||
HISTORY_ROOT / "launchers_reference" / "run_style_analysis_3d.bat"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("analyze_style_with_hermes.py", launcher)
|
||||
self.assertIn("--all-styles", launcher)
|
||||
self.assertIn("--days 3", launcher)
|
||||
self.assertIn("--skip-existing", launcher)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from config.style_config_loader import StyleConfigLoader
|
||||
|
||||
|
||||
def test_erp_codes_use_tm_record_even_when_jd_record_comes_first() -> None:
|
||||
loader = StyleConfigLoader()
|
||||
output = io.StringIO()
|
||||
|
||||
with redirect_stdout(output):
|
||||
payload = loader._aggregate(
|
||||
[
|
||||
{"platform": "京东旗舰店", "style": "极星托特", "erp_codes": "10513001"},
|
||||
{"platform": "天猫", "style": "极星托特", "erp_codes": "10513"},
|
||||
]
|
||||
)
|
||||
|
||||
assert payload["styles"]["极星托特"]["erp_codes"] == ["10513"]
|
||||
assert "ERP 编码冲突" in output.getvalue()
|
||||
assert "10513001" in output.getvalue()
|
||||
|
||||
|
||||
def test_non_tm_erp_code_is_not_used_when_tm_code_is_missing() -> None:
|
||||
loader = StyleConfigLoader()
|
||||
output = io.StringIO()
|
||||
|
||||
with redirect_stdout(output):
|
||||
payload = loader._aggregate(
|
||||
[
|
||||
{"platform": "京东旗舰店", "style": "极星托特", "erp_codes": "10513001"},
|
||||
{"platform": "天猫", "style": "极星托特", "erp_codes": ""},
|
||||
]
|
||||
)
|
||||
|
||||
assert payload["styles"]["极星托特"]["erp_codes"] == []
|
||||
assert "天猫分组未填写" in output.getvalue()
|
||||
|
||||
|
||||
def test_exact_style_field_wins_over_style_content_prefix(monkeypatch) -> None:
|
||||
loader = StyleConfigLoader()
|
||||
monkeypatch.setattr(
|
||||
loader,
|
||||
"_run_lark",
|
||||
lambda _args: {
|
||||
"data": {
|
||||
"fields": ["款式内容", "款式", "ERP款式编码", "平台"],
|
||||
"data": [["营销文案", "盖世m1", "10416,10455", ["天猫"]]],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
records = loader._load_records_from_lark()
|
||||
|
||||
assert records == [
|
||||
{
|
||||
"style": "盖世m1",
|
||||
"erp_codes": "10416,10455",
|
||||
"platform": "天猫",
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,564 @@
|
||||
import inspect
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from collect_sycm_market_rank import (
|
||||
TARGET_MARKET_CATEGORIES,
|
||||
_feishu_append_xml_path,
|
||||
_original_taobao_image_url,
|
||||
_product_item_id,
|
||||
_risk_control_reason,
|
||||
build_feishu_category_xml,
|
||||
build_feishu_doc_xml,
|
||||
close_market_rank_browser,
|
||||
collect_all_rank_pages,
|
||||
create_feishu_doc,
|
||||
detect_risk_control_text,
|
||||
ensure_market_rank_login,
|
||||
group_products_by_category,
|
||||
inspect_feishu_batch_tables,
|
||||
inspect_feishu_category_tables,
|
||||
parse_market_rank_html,
|
||||
parse_range_upper,
|
||||
prepare_market_rank_filters,
|
||||
prepare_market_rank_period,
|
||||
)
|
||||
|
||||
SAMPLE_HTML = """
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>排名</th><th>商品</th><th>商品关键词</th>
|
||||
<th>店铺</th><th>支付买家数</th><th>访客数</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>1</td>
|
||||
<td>
|
||||
<a href="//detail.tmall.com/item.htm?id=123">
|
||||
<img data-src="//img.alicdn.com/bao/uploaded/demo.jpg"/>
|
||||
【新品】CECE大容量旅行箱
|
||||
</a>
|
||||
</td>
|
||||
<td>旅行</td><td>cece旗舰店</td><td>2500 ~ 5000</td><td>10万 ~ 25万</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
class SycmMarketRankTests(unittest.TestCase):
|
||||
def test_expired_login_without_credentials_fails_clearly(self):
|
||||
with (
|
||||
patch("collect_sycm_market_rank.sycm.ACCOUNT", ""),
|
||||
patch("collect_sycm_market_rank.sycm.PASSWORD", ""),
|
||||
patch(
|
||||
"collect_sycm_market_rank.sycm.ensure_logged_in",
|
||||
return_value=None,
|
||||
),
|
||||
self.assertRaisesRegex(
|
||||
RuntimeError,
|
||||
"登录态失效.*SYCM_ACCOUNT/SYCM_PASSWORD",
|
||||
),
|
||||
):
|
||||
ensure_market_rank_login(MagicMock(), MagicMock())
|
||||
|
||||
def test_expired_login_does_not_swallow_login_exception(self):
|
||||
with (
|
||||
patch("collect_sycm_market_rank.sycm.ACCOUNT", "configured"),
|
||||
patch("collect_sycm_market_rank.sycm.PASSWORD", "configured"),
|
||||
patch(
|
||||
"collect_sycm_market_rank.sycm.ensure_logged_in",
|
||||
side_effect=RuntimeError("login rejected"),
|
||||
),
|
||||
self.assertRaisesRegex(RuntimeError, "自动登录失败: RuntimeError"),
|
||||
):
|
||||
ensure_market_rank_login(MagicMock(), MagicMock())
|
||||
|
||||
def test_parallel_documents_use_isolated_append_xml_files(self):
|
||||
first = _feishu_append_xml_path("https://example.feishu.cn/docx/first", 1)
|
||||
second = _feishu_append_xml_path("https://example.feishu.cn/docx/second", 1)
|
||||
|
||||
self.assertNotEqual(first, second)
|
||||
self.assertEqual(first.parent, second.parent)
|
||||
|
||||
def test_title_with_question_mark_is_rejected_before_creating_document(self):
|
||||
with self.assertRaisesRegex(ValueError, "编码损坏"):
|
||||
create_feishu_doc("??九品类市场排行", "<p>test</p>")
|
||||
|
||||
def test_category_groups_are_one_table_each(self):
|
||||
products = [
|
||||
{"category_name": "旅行箱", "rank": 1, "name": "A"},
|
||||
{"category_name": "双肩包", "rank": 1, "name": "B"},
|
||||
{"category_name": "旅行箱", "rank": 2, "name": "C"},
|
||||
]
|
||||
groups = group_products_by_category(products)
|
||||
|
||||
self.assertEqual([name for name, _ in groups], ["旅行箱", "双肩包"])
|
||||
self.assertEqual([len(items) for _, items in groups], [2, 1])
|
||||
category_xml = build_feishu_category_xml(*groups[0])
|
||||
self.assertEqual(category_xml.count("<table>"), 1)
|
||||
self.assertEqual(category_xml.count("<th "), 7)
|
||||
self.assertNotIn(">品类</th>", category_xml)
|
||||
|
||||
def test_category_table_inspection_keeps_heading_and_row_count(self):
|
||||
content = """
|
||||
<h2>旅行箱（2条）</h2><table id="t1"><tbody>
|
||||
<tr><td><img src="1"/></td></tr><tr><td><img src="2"/></td></tr>
|
||||
</tbody></table>
|
||||
<h2>双肩包（1条）</h2><table id="t2"><tbody><tr><td>缺图</td></tr></tbody></table>
|
||||
"""
|
||||
|
||||
self.assertEqual(
|
||||
inspect_feishu_category_tables(content),
|
||||
[
|
||||
{"block_id": "t1", "category": "旅行箱", "declared_rows": 2, "rows": 2, "images": 2},
|
||||
{"block_id": "t2", "category": "双肩包", "declared_rows": 1, "rows": 1, "images": 0},
|
||||
],
|
||||
)
|
||||
|
||||
def test_detect_risk_control_text_matches_actual_sycm_popup(self):
|
||||
popup_text = """
|
||||
请稍等片刻再点击刷新哦
|
||||
操作太频繁了，请稍后再试
|
||||
反馈码：a777b03e7183a9564215ff465acad489
|
||||
"""
|
||||
|
||||
self.assertEqual(
|
||||
detect_risk_control_text(popup_text),
|
||||
"操作太频繁了，请稍后再试",
|
||||
)
|
||||
|
||||
def test_detect_risk_control_text_ignores_normal_rank_page(self):
|
||||
self.assertEqual(
|
||||
detect_risk_control_text("市场排行 支付买家数 访客数 下一页"),
|
||||
"",
|
||||
)
|
||||
|
||||
def test_detect_risk_control_text_matches_baxia_mask(self):
|
||||
self.assertEqual(
|
||||
detect_risk_control_text('<div class="baxia-dialog-mask"></div>'),
|
||||
"生意参谋 Baxia 风控遮罩",
|
||||
)
|
||||
|
||||
def test_hidden_baxia_dialog_is_not_treated_as_active_captcha(self):
|
||||
page = MagicMock()
|
||||
dialog = MagicMock()
|
||||
dialog.count.return_value = 1
|
||||
dialog.nth.return_value.is_visible.return_value = False
|
||||
body = MagicMock()
|
||||
body.inner_text.return_value = "市场排行 支付买家数"
|
||||
page.locator.side_effect = lambda selector: (
|
||||
dialog if selector == ".baxia-dialog" else body
|
||||
)
|
||||
page.content.return_value = (
|
||||
'<div style="display:none" class="baxia-dialog">X</div>'
|
||||
)
|
||||
|
||||
self.assertEqual(_risk_control_reason(page), "")
|
||||
|
||||
def test_owned_tmall_browser_has_profile_scoped_fallback_cleanup(self):
|
||||
session = MagicMock()
|
||||
with (
|
||||
patch("collect_sycm_market_rank.sys.platform", "win32"),
|
||||
patch("collect_sycm_market_rank.time.sleep"),
|
||||
patch("collect_sycm_market_rank.subprocess.run") as run,
|
||||
):
|
||||
close_market_rank_browser(
|
||||
session,
|
||||
user_data_dir="portable-tm-profile",
|
||||
cdp_url=None,
|
||||
)
|
||||
|
||||
session.close.assert_called_once_with()
|
||||
run.assert_called_once()
|
||||
self.assertEqual(
|
||||
run.call_args.kwargs["env"]["TM_PROFILE_TO_CLOSE"],
|
||||
"portable-tm-profile",
|
||||
)
|
||||
|
||||
def test_cdp_browser_is_not_force_closed(self):
|
||||
session = MagicMock()
|
||||
with patch("collect_sycm_market_rank.subprocess.run") as run:
|
||||
close_market_rank_browser(
|
||||
session,
|
||||
user_data_dir="portable-tm-profile",
|
||||
cdp_url="http://127.0.0.1:9222",
|
||||
)
|
||||
|
||||
session.close.assert_called_once_with()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_prepare_market_period_waits_in_same_session_before_clicking_period(self):
|
||||
context = MagicMock()
|
||||
original_page = object()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"collect_sycm_market_rank._risk_control_reason",
|
||||
side_effect=["生意参谋 Baxia 风控遮罩", ""],
|
||||
),
|
||||
patch(
|
||||
"collect_sycm_market_rank.wait_for_manual_risk_clearance"
|
||||
) as wait_for_clearance,
|
||||
patch("collect_sycm_market_rank.select_market_period") as select_period,
|
||||
):
|
||||
result = prepare_market_rank_period(
|
||||
context,
|
||||
original_page,
|
||||
period="30天",
|
||||
risk_mode="manual",
|
||||
cooldown_seconds=0,
|
||||
max_retries=2,
|
||||
)
|
||||
|
||||
self.assertIs(result, original_page)
|
||||
context.clear_cookies.assert_not_called()
|
||||
wait_for_clearance.assert_called_once_with(
|
||||
original_page,
|
||||
"生意参谋 Baxia 风控遮罩",
|
||||
)
|
||||
select_period.assert_called_once_with(original_page, "30天")
|
||||
|
||||
def test_prepare_market_filters_retries_same_step_after_manual_captcha(self):
|
||||
page = object()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"collect_sycm_market_rank.select_market_period",
|
||||
) as select_period,
|
||||
patch(
|
||||
"collect_sycm_market_rank.select_market_category_path",
|
||||
side_effect=[RuntimeError("mask intercepts pointer events"), None],
|
||||
) as select_category,
|
||||
patch(
|
||||
"collect_sycm_market_rank._risk_control_reason",
|
||||
side_effect=["", "生意参谋 Baxia 风控遮罩", ""],
|
||||
),
|
||||
patch(
|
||||
"collect_sycm_market_rank.wait_for_manual_risk_clearance"
|
||||
) as wait_for_clearance,
|
||||
patch("collect_sycm_market_rank.apply_custom_price") as apply_price,
|
||||
):
|
||||
result = prepare_market_rank_filters(
|
||||
page,
|
||||
period="30天",
|
||||
category_path=("箱包皮具/热销女包/男包", "旅行箱"),
|
||||
min_price=400,
|
||||
max_price=2000,
|
||||
risk_mode="manual",
|
||||
cooldown_seconds=0,
|
||||
max_retries=2,
|
||||
)
|
||||
|
||||
self.assertIs(result, page)
|
||||
self.assertEqual(select_period.call_count, 2)
|
||||
self.assertEqual(select_category.call_count, 2)
|
||||
apply_price.assert_called_once_with(page, 400, 2000)
|
||||
wait_for_clearance.assert_called_once_with(
|
||||
page,
|
||||
"生意参谋 Baxia 风控遮罩",
|
||||
)
|
||||
|
||||
def test_manual_captcha_reapplies_filters_before_parsing_current_page(self):
|
||||
page = MagicMock()
|
||||
page.content.side_effect = [
|
||||
'<div class="baxia-dialog">captcha</div>',
|
||||
SAMPLE_HTML,
|
||||
]
|
||||
recover = MagicMock(return_value=page)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"collect_sycm_market_rank._risk_control_reason",
|
||||
side_effect=["生意参谋 Baxia 风控遮罩", ""],
|
||||
),
|
||||
patch(
|
||||
"collect_sycm_market_rank.wait_for_manual_risk_clearance"
|
||||
) as wait_for_clearance,
|
||||
patch("collect_sycm_market_rank._save_checkpoint"),
|
||||
):
|
||||
rows = collect_all_rank_pages(
|
||||
page,
|
||||
max_pages=1,
|
||||
category_name="旅行箱",
|
||||
category_path="箱包 > 旅行箱",
|
||||
recover_from_risk=recover,
|
||||
manual_risk_wait=True,
|
||||
)
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
wait_for_clearance.assert_called_once_with(
|
||||
page,
|
||||
"生意参谋 Baxia 风控遮罩",
|
||||
)
|
||||
recover.assert_called_once_with(
|
||||
page,
|
||||
1,
|
||||
1,
|
||||
"生意参谋 Baxia 风控遮罩",
|
||||
)
|
||||
|
||||
def test_target_market_categories_contains_the_nine_requested_paths(self):
|
||||
self.assertEqual(len(TARGET_MARKET_CATEGORIES), 9)
|
||||
self.assertEqual(
|
||||
[
|
||||
(
|
||||
item["name"],
|
||||
item["min_price"],
|
||||
item["max_price"],
|
||||
tuple(tuple(path) for path in item["paths"]),
|
||||
)
|
||||
for item in TARGET_MARKET_CATEGORIES
|
||||
],
|
||||
[
|
||||
(
|
||||
"旅行箱",
|
||||
400,
|
||||
2000,
|
||||
(("箱包皮具/热销女包/男包", "旅行箱"),),
|
||||
),
|
||||
(
|
||||
"运动包",
|
||||
200,
|
||||
1000,
|
||||
(("箱包皮具/热销女包/男包", "旅行袋"),),
|
||||
),
|
||||
(
|
||||
"托特包",
|
||||
200,
|
||||
3000,
|
||||
(("箱包皮具/热销女包/男包", "女士包袋新", "托特包"),),
|
||||
),
|
||||
(
|
||||
"双肩包",
|
||||
200,
|
||||
2000,
|
||||
(("箱包皮具/热销女包/男包", "双肩背包"),),
|
||||
),
|
||||
(
|
||||
"斜挎包",
|
||||
200,
|
||||
2000,
|
||||
(
|
||||
("箱包皮具/热销女包/男包", "男士包袋"),
|
||||
(
|
||||
"箱包皮具/热销女包/男包",
|
||||
"女士包袋新",
|
||||
"通用款女包",
|
||||
),
|
||||
),
|
||||
),
|
||||
(
|
||||
"胸包",
|
||||
200,
|
||||
1000,
|
||||
(("箱包皮具/热销女包/男包", "胸包"),),
|
||||
),
|
||||
(
|
||||
"手机包",
|
||||
100,
|
||||
1000,
|
||||
(("箱包皮具/热销女包/男包", "功能小包", "手机包"),),
|
||||
),
|
||||
(
|
||||
"电脑包",
|
||||
200,
|
||||
1000,
|
||||
(("3C数码配件", "笔记本电脑配件", "笔记本电脑包"),),
|
||||
),
|
||||
(
|
||||
"相机包",
|
||||
200,
|
||||
2000,
|
||||
(("3C数码配件", "数码相机配件", "数码相机包"),),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
def test_original_taobao_image_url_removes_rank_thumbnail_suffix(self):
|
||||
thumbnail = (
|
||||
"https://img.alicdn.com/bao/uploaded/demo-item_pic.jpg_36x36.jpg"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
_original_taobao_image_url(thumbnail),
|
||||
"https://img.alicdn.com/bao/uploaded/demo-item_pic.jpg",
|
||||
)
|
||||
|
||||
def test_product_identity_ignores_changing_mi_id(self):
|
||||
first = _product_item_id(
|
||||
"https://detail.tmall.com/item.htm?id=123&mi_id=first"
|
||||
)
|
||||
second = _product_item_id(
|
||||
"https://detail.tmall.com/item.htm?mi_id=second&id=123"
|
||||
)
|
||||
|
||||
self.assertEqual(first, "123")
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_parse_range_upper_supports_plain_and_wan_units(self):
|
||||
self.assertEqual(parse_range_upper("2500 ~ 5000"), 5000)
|
||||
self.assertEqual(parse_range_upper("7.5万～10万"), 100000)
|
||||
self.assertEqual(parse_range_upper("500"), 500)
|
||||
|
||||
def test_parse_market_rank_html_extracts_name_image_link_and_buyer_max(self):
|
||||
rows = parse_market_rank_html(SAMPLE_HTML)
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["rank"], 1)
|
||||
self.assertEqual(rows[0]["name"], "【新品】CECE大容量旅行箱")
|
||||
self.assertEqual(rows[0]["buyer_range"], "2500 ~ 5000")
|
||||
self.assertEqual(rows[0]["buyer_max"], 5000)
|
||||
self.assertEqual(
|
||||
rows[0]["image_url"],
|
||||
"https://img.alicdn.com/bao/uploaded/demo.jpg",
|
||||
)
|
||||
self.assertEqual(
|
||||
rows[0]["product_url"],
|
||||
"https://detail.tmall.com/item.htm?id=123",
|
||||
)
|
||||
|
||||
def test_parse_market_rank_html_skips_image_only_link_for_product_name(self):
|
||||
source = """
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>排名</th><th>商品</th><th>支付买家数</th>
|
||||
</tr></thead>
|
||||
<tbody><tr>
|
||||
<td>9 降1名</td>
|
||||
<td>
|
||||
<a href="//detail.tmall.com/item.htm?id=639">
|
||||
<img src="//img.alicdn.com/demo.jpg"/>
|
||||
</a>
|
||||
<p class="goodsName">
|
||||
<a title="MOFT电脑支架收纳包"
|
||||
href="//detail.tmall.com/item.htm?id=639">MOFT电脑支架收纳包</a>
|
||||
</p>
|
||||
</td>
|
||||
<td>50 ~ 100</td>
|
||||
</tr></tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
rows = parse_market_rank_html(source)
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["name"], "MOFT电脑支架收纳包")
|
||||
self.assertEqual(rows[0]["rank"], 9)
|
||||
self.assertEqual(rows[0]["buyer_max"], 100)
|
||||
|
||||
def test_rank_page_collector_has_no_detail_page_or_price_interface(self):
|
||||
parameters = inspect.signature(collect_all_rank_pages).parameters
|
||||
source = inspect.getsource(collect_all_rank_pages)
|
||||
|
||||
self.assertNotIn("context", parameters)
|
||||
self.assertNotIn("collect_prices", parameters)
|
||||
self.assertNotIn("price_cache", parameters)
|
||||
self.assertNotIn("_collect_price_from_rank_link", source)
|
||||
self.assertNotIn("expect_page", source)
|
||||
|
||||
def test_inspect_feishu_batch_tables_reports_start_rows_and_images(self):
|
||||
content = """
|
||||
<h2 id="heading-1">榜单记录 1–2</h2>
|
||||
<table id="table-1">
|
||||
<tbody>
|
||||
<tr><td><img src="img-1"/></td></tr>
|
||||
<tr><td>缺图</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h2 id="heading-2">榜单记录 3–3</h2>
|
||||
<table id="table-2">
|
||||
<tbody><tr><td><img src="img-2"/></td></tr></tbody>
|
||||
</table>
|
||||
"""
|
||||
|
||||
self.assertEqual(
|
||||
inspect_feishu_batch_tables(content),
|
||||
[
|
||||
{
|
||||
"block_id": "table-1",
|
||||
"start_rank": 1,
|
||||
"rows": 2,
|
||||
"images": 1,
|
||||
},
|
||||
{
|
||||
"block_id": "table-2",
|
||||
"start_rank": 3,
|
||||
"rows": 1,
|
||||
"images": 1,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
def test_build_doc_contains_image_metrics_and_link_without_product_price(self):
|
||||
xml = build_feishu_doc_xml(
|
||||
[
|
||||
{
|
||||
"rank": 1,
|
||||
"category_name": "旅行箱",
|
||||
"name": "CECE旅行箱",
|
||||
"buyer_range": "2500 ~ 5000",
|
||||
"buyer_max": 5000,
|
||||
"image_url": "https://img.alicdn.com/demo.jpg",
|
||||
"product_url": "https://detail.tmall.com/item.htm?id=123",
|
||||
}
|
||||
],
|
||||
min_price=400,
|
||||
max_price=2000,
|
||||
title="市场排行测试",
|
||||
)
|
||||
|
||||
self.assertIn("<title>市场排行测试</title>", xml)
|
||||
self.assertIn('img href="https://img.alicdn.com/demo.jpg"', xml)
|
||||
self.assertIn("5000", xml)
|
||||
self.assertIn("https://detail.tmall.com/item.htm?id=123", xml)
|
||||
self.assertIn("商品链接", xml)
|
||||
self.assertNotIn("商品价格", xml)
|
||||
self.assertNotIn("¥418", xml)
|
||||
self.assertIn("风格（运营手动区分）", xml)
|
||||
self.assertIn("场景用途（运营手动区分）", xml)
|
||||
self.assertIn(">品类</th>", xml)
|
||||
self.assertIn("<tr><td></td><td></td><td>旅行箱</td>", xml)
|
||||
self.assertIn(
|
||||
'<a href="https://detail.tmall.com/item.htm?id=123">打开商品</a>',
|
||||
xml,
|
||||
)
|
||||
self.assertNotIn(
|
||||
'<a href="https://detail.tmall.com/item.htm?id=123">CECE旅行箱</a>',
|
||||
xml,
|
||||
)
|
||||
self.assertEqual(
|
||||
xml.count('href="https://detail.tmall.com/item.htm?id=123"'),
|
||||
1,
|
||||
)
|
||||
self.assertIn('width="160"', xml)
|
||||
self.assertIn('height="160"', xml)
|
||||
self.assertIn('<col width="180"/>', xml)
|
||||
|
||||
def test_build_doc_uses_original_image_instead_of_thumbnail(self):
|
||||
xml = build_feishu_doc_xml(
|
||||
[
|
||||
{
|
||||
"rank": 1,
|
||||
"name": "CECE旅行箱",
|
||||
"buyer_range": "2500 ~ 5000",
|
||||
"buyer_max": 5000,
|
||||
"image_url": (
|
||||
"https://img.alicdn.com/demo.jpg_36x36.jpg"
|
||||
),
|
||||
"product_url": "https://detail.tmall.com/item.htm?id=123",
|
||||
}
|
||||
],
|
||||
min_price=400,
|
||||
max_price=2000,
|
||||
title="大图测试",
|
||||
)
|
||||
|
||||
self.assertIn('href="https://img.alicdn.com/demo.jpg"', xml)
|
||||
self.assertNotIn("_36x36.jpg", xml)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,343 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.product_commerce import (
|
||||
collect_erp_monthly_metrics as erp_monthly,
|
||||
)
|
||||
from gyxx_flow.modules.product_commerce.collect_erp_monthly_metrics import (
|
||||
monthly_report_interval,
|
||||
set_monthly_filters,
|
||||
)
|
||||
from gyxx_flow.modules.product_commerce.sync_monthly_sales_sheet import (
|
||||
ErpStylePlan,
|
||||
MonthlyMetric,
|
||||
MonthlySalesConfig,
|
||||
ProductTarget,
|
||||
SheetInfo,
|
||||
_write_rows,
|
||||
build_parser,
|
||||
choose_template,
|
||||
fetch_product_targets,
|
||||
match_metrics,
|
||||
merge_product_catalog,
|
||||
month_bounds,
|
||||
month_option,
|
||||
month_sheet_title,
|
||||
normalized_name,
|
||||
parse_month,
|
||||
plan_erp_styles,
|
||||
product_text,
|
||||
require_complete_erp_mappings,
|
||||
)
|
||||
|
||||
|
||||
def test_month_contract_uses_natural_month_and_existing_title_convention() -> None:
|
||||
month = parse_month("2026-08-19")
|
||||
|
||||
assert month == date(2026, 8, 1)
|
||||
assert month_bounds(month) == (date(2026, 8, 1), date(2026, 8, 31))
|
||||
assert month_option(month) == "2026.08"
|
||||
assert month_sheet_title(month) == "26年8月"
|
||||
|
||||
|
||||
def test_erp_month_interval_uses_inclusive_natural_month_for_day_input() -> None:
|
||||
assert monthly_report_interval(parse_month("2026-07-02")) == (
|
||||
"2026-07-01",
|
||||
"2026-07-31",
|
||||
)
|
||||
assert monthly_report_interval(parse_month("2026-12-31")) == (
|
||||
"2026-12-01",
|
||||
"2026-12-31",
|
||||
)
|
||||
assert monthly_report_interval(parse_month("2024-02-15")) == (
|
||||
"2024-02-01",
|
||||
"2024-02-29",
|
||||
)
|
||||
|
||||
|
||||
def test_selected_month_is_the_only_month_argument() -> None:
|
||||
arguments = build_parser().parse_args(["--month", "2026-07"])
|
||||
|
||||
assert parse_month(arguments.month) == date(2026, 7, 1)
|
||||
with pytest.raises(SystemExit):
|
||||
build_parser().parse_args(
|
||||
["--month", "2026-07", "--budget-month", "2026-08"]
|
||||
)
|
||||
|
||||
|
||||
def test_missing_erp_zero_override_is_explicit() -> None:
|
||||
default = build_parser().parse_args(["--month", "2026-07"])
|
||||
override = build_parser().parse_args(
|
||||
["--month", "2026-07", "--allow-missing-erp-as-zero"]
|
||||
)
|
||||
|
||||
assert default.allow_missing_erp_as_zero is False
|
||||
assert override.allow_missing_erp_as_zero is True
|
||||
|
||||
|
||||
def test_product_targets_are_read_without_a_month_filter() -> None:
|
||||
config = MonthlySalesConfig(
|
||||
spreadsheet_url="https://example.test/sheets/token",
|
||||
base_url="https://example.test/base/token",
|
||||
base_token="base",
|
||||
table_id="table",
|
||||
view_id="view",
|
||||
product_field="品名",
|
||||
target_field="目标销量",
|
||||
month_field="月份",
|
||||
product_aliases={},
|
||||
)
|
||||
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def lark(args: list[str]) -> dict:
|
||||
calls.append(args)
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["品名", "目标销量"],
|
||||
"data": [["布谷", 500], ["白鹭", 350], ["蓝鹊", 400]],
|
||||
"has_more": False,
|
||||
}
|
||||
}
|
||||
|
||||
result = fetch_product_targets(
|
||||
config,
|
||||
lark=lark,
|
||||
)
|
||||
|
||||
assert result == [
|
||||
ProductTarget("布谷", 500),
|
||||
ProductTarget("白鹭", 350),
|
||||
ProductTarget("蓝鹊", 400),
|
||||
]
|
||||
assert "--filter-json" not in calls[0]
|
||||
|
||||
|
||||
def test_full_catalog_unions_style_base_and_targets_and_sums_duplicates() -> None:
|
||||
targets = [
|
||||
ProductTarget("布谷", 500),
|
||||
ProductTarget("星迹&星迹2", 1000),
|
||||
ProductTarget("星迹2", 500),
|
||||
ProductTarget("预算孤儿", 200),
|
||||
]
|
||||
styles = {
|
||||
"布谷": {"erp_codes": ["10352", "10496"]},
|
||||
"星迹2": {"erp_codes": ["10504"]},
|
||||
"白鹭": {"erp_codes": ["10310"]},
|
||||
"蓝鹊": {"erp_codes": ["10279"]},
|
||||
}
|
||||
|
||||
result = merge_product_catalog(
|
||||
targets,
|
||||
styles,
|
||||
{"星迹&星迹2": "星迹2"},
|
||||
)
|
||||
|
||||
assert result == [
|
||||
ProductTarget("布谷", 500),
|
||||
ProductTarget("星迹&星迹2", 1500),
|
||||
ProductTarget("预算孤儿", 200),
|
||||
ProductTarget("白鹭", 0),
|
||||
ProductTarget("蓝鹊", 0),
|
||||
]
|
||||
|
||||
|
||||
def test_product_text_turns_base_markdown_links_into_plain_product_names() -> None:
|
||||
assert (
|
||||
product_text(
|
||||
"[盖亚微单Pro生命进程（新）](https://example.feishu.cn/base/token)"
|
||||
)
|
||||
== "盖亚微单Pro"
|
||||
)
|
||||
assert product_text("阿波罗X1生命进程（新）") == "阿波罗X1"
|
||||
assert product_text("拾影相机包（新） 副本") == "拾影相机包"
|
||||
|
||||
|
||||
def test_metric_matching_uses_explicit_aliases_and_casefolded_exact_names() -> None:
|
||||
targets = [
|
||||
ProductTarget("盖世M1", 300),
|
||||
ProductTarget("星迹&星迹2", 1500),
|
||||
ProductTarget("尚未入库", 200),
|
||||
]
|
||||
metrics = {
|
||||
"盖世m1": MonthlyMetric("盖世m1", 271, 36, 12),
|
||||
"星迹2": MonthlyMetric("星迹2", 1858, 451, 12),
|
||||
}
|
||||
|
||||
rows, unmatched = match_metrics(
|
||||
targets,
|
||||
metrics,
|
||||
{"星迹&星迹2": "星迹2"},
|
||||
)
|
||||
|
||||
assert normalized_name("星迹＆星迹2") == normalized_name("星迹+星迹2")
|
||||
assert rows[0]["sales"] == 271
|
||||
assert rows[1]["returns"] == 451
|
||||
assert rows[2]["sales"] == 0
|
||||
assert unmatched == ["尚未入库"]
|
||||
|
||||
|
||||
def test_template_is_latest_strictly_earlier_year_month_sheet() -> None:
|
||||
sheets = [
|
||||
SheetInfo("a", "26年6月", 0, 198),
|
||||
SheetInfo("b", "26年7月", 1, 198),
|
||||
SheetInfo("c", "7月销量", 2, 200),
|
||||
]
|
||||
|
||||
assert choose_template(sheets, date(2026, 8, 1)).sheet_id == "b"
|
||||
|
||||
|
||||
def test_erp_style_plan_uses_style_base_codes_and_aliases() -> None:
|
||||
targets = [
|
||||
ProductTarget("盖世M1", 300),
|
||||
ProductTarget("星迹&星迹2", 1500),
|
||||
ProductTarget("云栖相机双肩包", 200),
|
||||
]
|
||||
styles = {
|
||||
"盖世m1": {"erp_codes": ["10416", "10455", "10416"]},
|
||||
"星迹2": {"erp_codes": ["10504", "10394"]},
|
||||
"云栖": {"erp_codes": []},
|
||||
}
|
||||
|
||||
plans, missing = plan_erp_styles(
|
||||
targets,
|
||||
styles,
|
||||
{
|
||||
"星迹&星迹2": "星迹2",
|
||||
"云栖相机双肩包": "云栖",
|
||||
},
|
||||
)
|
||||
|
||||
assert plans == [
|
||||
ErpStylePlan("盖世m1", ("10416", "10455")),
|
||||
ErpStylePlan("星迹2", ("10504", "10394")),
|
||||
]
|
||||
assert missing == ["云栖相机双肩包"]
|
||||
|
||||
|
||||
def test_execute_guard_rejects_incomplete_erp_mapping_before_sheet_write() -> None:
|
||||
with pytest.raises(RuntimeError, match="目标表未清空"):
|
||||
require_complete_erp_mappings(["云栖相机双肩包", "轻风双肩包"])
|
||||
|
||||
require_complete_erp_mappings([])
|
||||
|
||||
|
||||
def test_monthly_filter_sets_natural_month_exact_shop_labels_and_erp_code() -> None:
|
||||
class Frame:
|
||||
def __init__(self) -> None:
|
||||
self.script = ""
|
||||
self.args = {}
|
||||
|
||||
def evaluate(self, script, args):
|
||||
self.script = script
|
||||
self.args = args
|
||||
return {"selected_shop_ids": ["shop_1"], "selected_shop_labels": ["甲店"]}
|
||||
|
||||
frame = Frame()
|
||||
|
||||
result = set_monthly_filters(
|
||||
frame,
|
||||
erp_code="10439",
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-07-31",
|
||||
shop_labels=["甲店"],
|
||||
)
|
||||
|
||||
assert frame.args == {
|
||||
"erpCode": "10439",
|
||||
"startDate": "2026-07-01",
|
||||
"endDate": "2026-07-31",
|
||||
"shopLabels": ["甲店"],
|
||||
}
|
||||
assert 'input[name="shop_id"]' in frame.script
|
||||
assert "#i_id" in frame.script
|
||||
assert result["selected_shop_labels"] == ["甲店"]
|
||||
|
||||
|
||||
def test_monthly_collector_sums_all_erp_codes_for_one_style(monkeypatch) -> None:
|
||||
monkeypatch.setattr(erp_monthly.erp, "find_report_frames", lambda _page: (object(), object()))
|
||||
observed_ranges: list[tuple[str, str, str]] = []
|
||||
|
||||
def run_one_code(_page, _frame, code, start_date, end_date, *_args):
|
||||
observed_ranges.append((code, start_date, end_date))
|
||||
return {
|
||||
"ok": True,
|
||||
"yesterday_sales": {"10416": 1185, "10455": 904}[code],
|
||||
"yesterday_returns": {"10416": 279, "10455": 159}[code],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
erp_monthly,
|
||||
"_run_one_code",
|
||||
run_one_code,
|
||||
)
|
||||
|
||||
result = erp_monthly.collect_monthly_in_page(
|
||||
object(),
|
||||
[{"style_name": "盖世m1", "erp_style_codes": ["10416", "10455"]}],
|
||||
date(2026, 7, 1),
|
||||
shop_labels=["甲店"],
|
||||
)
|
||||
|
||||
assert result["盖世m1"].sales == 2089
|
||||
assert result["盖世m1"].returns == 438
|
||||
assert observed_ranges == [
|
||||
("10416", "2026-07-01", "2026-07-31"),
|
||||
("10455", "2026-07-01", "2026-07-31"),
|
||||
]
|
||||
|
||||
|
||||
def test_monthly_collector_rejects_partial_erp_code_failure(monkeypatch) -> None:
|
||||
monkeypatch.setattr(erp_monthly.erp, "find_report_frames", lambda _page: (object(), object()))
|
||||
monkeypatch.setattr(
|
||||
erp_monthly,
|
||||
"_run_one_code",
|
||||
lambda _page, _frame, code, *_args: {
|
||||
"ok": code == "10416",
|
||||
"yesterday_sales": 10 if code == "10416" else 0,
|
||||
"yesterday_returns": 1 if code == "10416" else 0,
|
||||
"error": "timeout" if code != "10416" else "",
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="10455"):
|
||||
erp_monthly.collect_monthly_in_page(
|
||||
object(),
|
||||
[{"style_name": "盖世m1", "erp_style_codes": ["10416", "10455"]}],
|
||||
date(2026, 7, 1),
|
||||
shop_labels=["甲店"],
|
||||
)
|
||||
|
||||
|
||||
def test_write_rows_repeats_first_data_row_format_without_overwriting_values() -> None:
|
||||
calls: list[list[str]] = []
|
||||
config = MonthlySalesConfig(
|
||||
spreadsheet_url="https://example.test/sheets/token",
|
||||
base_url="https://example.test/base/token",
|
||||
base_token="base",
|
||||
table_id="table",
|
||||
view_id="view",
|
||||
product_field="品名",
|
||||
target_field="目标销量",
|
||||
month_field="月份",
|
||||
product_aliases={},
|
||||
)
|
||||
rows = [
|
||||
{"product": "甲", "target_sales": 10, "sales": 8, "returns": 1},
|
||||
{"product": "乙", "target_sales": 20, "sales": 9, "returns": 2},
|
||||
]
|
||||
|
||||
_write_rows(
|
||||
config,
|
||||
SheetInfo("sheet", "26年7月", 0, 198),
|
||||
rows,
|
||||
lark=lambda args: calls.append(args) or {},
|
||||
)
|
||||
|
||||
format_call = next(args for args in calls if "+range-copy" in args)
|
||||
assert format_call[format_call.index("--source-range") + 1] == "A2:G2"
|
||||
assert format_call[format_call.index("--target-range") + 1] == "A3:G3"
|
||||
assert format_call[format_call.index("--paste-type") + 1] == "formats"
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import taobao_sycm_products as tm
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self) -> None:
|
||||
self.context = object()
|
||||
self.closed = False
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_tm_main_fails_when_browser_connection_is_missing(monkeypatch) -> None:
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (None, None))
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
|
||||
|
||||
def test_tm_main_fails_when_login_does_not_complete(monkeypatch) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page))
|
||||
monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: None)
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_main_fails_when_target_date_cannot_be_selected(monkeypatch) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page))
|
||||
monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page)
|
||||
monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None)
|
||||
monkeypatch.setattr(tm, "select_yesterday", lambda _page: False)
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_main_fails_when_download_is_missing(monkeypatch) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page))
|
||||
monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page)
|
||||
monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None)
|
||||
monkeypatch.setattr(tm, "select_yesterday", lambda _page: True)
|
||||
monkeypatch.setattr(tm, "click_download", lambda _page: None)
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert session.closed is True
|
||||
|
||||
|
||||
def test_tm_main_fails_when_download_has_no_valid_style(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
session = _Session()
|
||||
page = object()
|
||||
report = tmp_path / "report.xlsx"
|
||||
monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page))
|
||||
monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page)
|
||||
monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None)
|
||||
monkeypatch.setattr(tm, "select_yesterday", lambda _page: True)
|
||||
monkeypatch.setattr(tm, "click_download", lambda _page: report)
|
||||
monkeypatch.setattr(tm, "export_excel_to_json_and_md", lambda _path: (None, None))
|
||||
monkeypatch.setattr(tm, "summarize_styles_from_excel", lambda _path: [])
|
||||
monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0)
|
||||
|
||||
assert tm.main(["--target-date", "2026-08-03"]) == 1
|
||||
assert session.closed is True
|
||||
@@ -0,0 +1,69 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
||||
|
||||
|
||||
class TmPersonaRecoveryTests(unittest.TestCase):
|
||||
def test_zero_chart_skip_is_not_platform_success(self):
|
||||
self.assertEqual(
|
||||
dmp.records_exit_code(
|
||||
{"款A": ["123"]},
|
||||
[{"style_name": "款A", "status": "skipped"}],
|
||||
),
|
||||
2,
|
||||
)
|
||||
|
||||
def test_switch_drawer_failure_retries_page_then_rebuilds_session(self):
|
||||
failed = {
|
||||
"status": "failed",
|
||||
"error": "切换分析单品弹窗未加载（未出现搜索框）",
|
||||
}
|
||||
succeeded = {"status": "ok", "chart_values": {"用户性别": []}}
|
||||
collect = Mock(side_effect=[failed, failed, succeeded])
|
||||
goto = Mock()
|
||||
reopen = Mock(return_value=("new-session", "new-page"))
|
||||
|
||||
session, page, record = dmp.collect_item_with_recovery(
|
||||
"old-session",
|
||||
"old-page",
|
||||
"盖亚斜挎",
|
||||
"921336775006",
|
||||
Path("data/tm人物画像"),
|
||||
collect_fn=collect,
|
||||
goto_fn=goto,
|
||||
reopen_session=reopen,
|
||||
)
|
||||
|
||||
self.assertEqual(record["status"], "ok")
|
||||
self.assertEqual((session, page), ("new-session", "new-page"))
|
||||
self.assertEqual(collect.call_count, 3)
|
||||
goto.assert_called_once_with("old-page")
|
||||
reopen.assert_called_once_with("old-session", "old-page")
|
||||
|
||||
def test_non_navigation_failure_is_not_retried(self):
|
||||
skipped = {"status": "skipped", "skip_reason": "分析人群规模过小"}
|
||||
collect = Mock(return_value=skipped)
|
||||
goto = Mock()
|
||||
reopen = Mock()
|
||||
|
||||
_, _, record = dmp.collect_item_with_recovery(
|
||||
"session",
|
||||
"page",
|
||||
"盖亚斜挎",
|
||||
"921336775006",
|
||||
Path("data/tm人物画像"),
|
||||
collect_fn=collect,
|
||||
goto_fn=goto,
|
||||
reopen_session=reopen,
|
||||
)
|
||||
|
||||
self.assertEqual(record, skipped)
|
||||
self.assertEqual(collect.call_count, 1)
|
||||
goto.assert_not_called()
|
||||
reopen.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,152 @@
|
||||
import unittest
|
||||
from argparse import Namespace
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import taobao_wanxiang_ai_creative_report as wanxiang
|
||||
from taobao_wanxiang_ai_creative_report import select_report_template
|
||||
|
||||
|
||||
class WanxiangReportTemplateTests(unittest.TestCase):
|
||||
def test_authenticated_account_chooser_on_login_route_is_valid_session(self):
|
||||
page = Mock()
|
||||
page.url = "https://one.alimama.com/index.html#!/login/index"
|
||||
page.locator.return_value.inner_text.return_value = (
|
||||
"欢迎登录\nHi，测试店铺\n进入后台\n退出账户"
|
||||
)
|
||||
|
||||
self.assertTrue(wanxiang.is_logged_in(page))
|
||||
|
||||
def test_credential_form_on_login_route_is_not_valid_session(self):
|
||||
page = Mock()
|
||||
page.url = "https://one.alimama.com/index.html#!/login/index"
|
||||
page.locator.return_value.inner_text.return_value = "账号名\n请输入登录密码"
|
||||
|
||||
self.assertFalse(wanxiang.is_logged_in(page))
|
||||
|
||||
def test_supports_actual_wanxiang_template_label_spelling(self):
|
||||
self.assertIn("报表模板", wanxiang.REPORT_TEMPLATE_LABELS)
|
||||
self.assertIn("报表模版", wanxiang.REPORT_TEMPLATE_LABELS)
|
||||
|
||||
def test_already_selected_keyword_template_does_not_click(self):
|
||||
page = Mock()
|
||||
page.evaluate.return_value = {
|
||||
"found": True,
|
||||
"current": "关键词推广",
|
||||
}
|
||||
|
||||
select_report_template(page, verify_attempts=1)
|
||||
|
||||
self.assertEqual(page.evaluate.call_count, 1)
|
||||
page.wait_for_timeout.assert_not_called()
|
||||
|
||||
def test_waits_for_async_template_filter_render(self):
|
||||
page = Mock()
|
||||
page.evaluate.side_effect = [
|
||||
{"found": False, "current": ""},
|
||||
{"found": True, "current": "关键词推广"},
|
||||
]
|
||||
|
||||
select_report_template(page, find_attempts=2, verify_attempts=1)
|
||||
|
||||
self.assertEqual(page.evaluate.call_count, 2)
|
||||
page.wait_for_timeout.assert_called_once_with(1000)
|
||||
|
||||
def test_missing_template_filter_saves_debug_and_stops(self):
|
||||
page = Mock()
|
||||
page.evaluate.return_value = {"found": False, "current": ""}
|
||||
|
||||
with (
|
||||
patch.object(wanxiang, "dump_report_template_debug") as dump,
|
||||
self.assertRaisesRegex(RuntimeError, "报表模板"),
|
||||
):
|
||||
select_report_template(page, find_attempts=2, verify_attempts=1)
|
||||
|
||||
dump.assert_called_once_with(page, "filter_not_found")
|
||||
|
||||
def test_switches_from_all_plans_to_keyword_template(self):
|
||||
page = Mock()
|
||||
page.evaluate.side_effect = [
|
||||
{"found": True, "current": "全部计划"},
|
||||
True,
|
||||
True,
|
||||
{"found": True, "current": "关键词推广"},
|
||||
]
|
||||
|
||||
with patch.object(wanxiang, "wait_table_change") as wait_table:
|
||||
select_report_template(page, verify_attempts=1)
|
||||
|
||||
self.assertEqual(page.evaluate.call_count, 4)
|
||||
self.assertGreaterEqual(page.wait_for_timeout.call_count, 2)
|
||||
wait_table.assert_called_once_with(page, seconds=2.0)
|
||||
|
||||
def test_raises_when_keyword_template_cannot_be_verified(self):
|
||||
page = Mock()
|
||||
page.evaluate.side_effect = [
|
||||
{"found": True, "current": "全部计划"},
|
||||
True,
|
||||
True,
|
||||
{"found": True, "current": "全部计划"},
|
||||
{"found": True, "current": "全部计划"},
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(wanxiang, "dump_report_template_debug"),
|
||||
self.assertRaisesRegex(RuntimeError, "关键词推广"),
|
||||
):
|
||||
select_report_template(page, verify_attempts=2)
|
||||
|
||||
def test_run_selects_keyword_template_before_collecting_styles(self):
|
||||
events = []
|
||||
page = Mock()
|
||||
session = Mock()
|
||||
args = Namespace(
|
||||
output=str(Path("data") / "test_wanxiang_keyword"),
|
||||
styles="星云2",
|
||||
date="2026-07-26",
|
||||
headless=True,
|
||||
user_data_dir="test-profile",
|
||||
account="account",
|
||||
password="placeholder",
|
||||
max_pages=1,
|
||||
report_template="关键词推广",
|
||||
keep_open=False,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(wanxiang, "start_stealthy_session", return_value=(session, page)),
|
||||
patch.object(wanxiang, "ensure_logged_in"),
|
||||
patch.object(wanxiang, "goto_wanxiang_from_seller"),
|
||||
patch.object(wanxiang, "goto_creative_report"),
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"select_report_template",
|
||||
side_effect=lambda *_args, **_kwargs: events.append("select"),
|
||||
) as select,
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"collect_style",
|
||||
side_effect=lambda *_args, **_kwargs: (
|
||||
events.append("collect") or {"style_name": "星云2"}
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"write_style_outputs",
|
||||
return_value=(Path("result.json"), Path("result.md")),
|
||||
),
|
||||
patch.object(
|
||||
wanxiang,
|
||||
"write_overall_outputs",
|
||||
return_value=Path("overall.json"),
|
||||
),
|
||||
):
|
||||
wanxiang.run(args)
|
||||
|
||||
select.assert_called_once_with(page, "关键词推广")
|
||||
self.assertEqual(events, ["select", "collect"])
|
||||
session.close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.browser_session import (
|
||||
install_jd_dom_compatibility,
|
||||
reuse_or_run_jd_login,
|
||||
)
|
||||
|
||||
|
||||
class _AccountInput:
|
||||
def __init__(self, *, visible: bool) -> None:
|
||||
self._visible = visible
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return int(self._visible)
|
||||
|
||||
def is_visible(self, *, timeout: int) -> bool:
|
||||
assert timeout == 1_000
|
||||
return self._visible
|
||||
|
||||
|
||||
class _Context:
|
||||
def __init__(self) -> None:
|
||||
self.scripts: list[str] = []
|
||||
|
||||
def add_init_script(self, *, script: str) -> None:
|
||||
self.scripts.append(script)
|
||||
|
||||
|
||||
def test_jd_session_adapter_reuses_authenticated_profile() -> None:
|
||||
inputs = {"shop": "account", "credential": "credential"}
|
||||
page = SimpleNamespace(
|
||||
url="https://shop.jd.com/jdm/home",
|
||||
goto=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args: None,
|
||||
locator=lambda _selector: _AccountInput(visible=False),
|
||||
)
|
||||
|
||||
result = reuse_or_run_jd_login(
|
||||
page,
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("authenticated profile must not repeat source login")
|
||||
),
|
||||
shop=inputs["shop"],
|
||||
password=inputs["credential"],
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_jd_session_adapter_invokes_source_login_when_form_is_visible() -> None:
|
||||
calls: list[tuple[object, str, str]] = []
|
||||
inputs = {"shop": "account", "credential": "credential"}
|
||||
page = SimpleNamespace(
|
||||
url="https://shop.jd.com/jdm/home",
|
||||
goto=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args: None,
|
||||
locator=lambda _selector: _AccountInput(visible=True),
|
||||
)
|
||||
|
||||
reuse_or_run_jd_login(
|
||||
page,
|
||||
lambda current_page, shop, password: calls.append(
|
||||
(current_page, shop, password)
|
||||
),
|
||||
shop=inputs["shop"],
|
||||
password=inputs["credential"],
|
||||
)
|
||||
|
||||
assert calls == [(page, "account", "credential")]
|
||||
|
||||
|
||||
def test_jd_dom_compatibility_aliases_current_and_future_pages() -> None:
|
||||
context = _Context()
|
||||
evaluated: list[str] = []
|
||||
page = SimpleNamespace(
|
||||
context=context,
|
||||
evaluate=lambda script: evaluated.append(script),
|
||||
)
|
||||
|
||||
install_jd_dom_compatibility(page)
|
||||
|
||||
assert len(context.scripts) == 1
|
||||
assert evaluated == context.scripts
|
||||
assert "jmtd-date-picker-combo" in context.scripts[0]
|
||||
assert "jmt-combo-date-picker" in context.scripts[0]
|
||||
assert "jmt-date-picker" in context.scripts[0]
|
||||
assert "classList.remove('jmt-date-picker')" in context.scripts[0]
|
||||
assert "element.dataset.eventContent === 'realtime'" in context.scripts[0]
|
||||
assert "gyxx-disabled-week-number" in context.scripts[0]
|
||||
assert "if (!document.documentElement)" in context.scripts[0]
|
||||
assert "setTimeout(installObserver, 0)" in context.scripts[0]
|
||||
assert "childList: true" in context.scripts[0]
|
||||
assert "attributes: true" not in context.scripts[0]
|
||||
assert 'data-event-content="week"' in context.scripts[0]
|
||||
assert "currentWeekTag.click()" in context.scripts[0]
|
||||
assert "__gyxxJdWeekActivationAt" in context.scripts[0]
|
||||
assert "gyxxWeekActivationAttempted" in context.scripts[0]
|
||||
assert ").find(visible)" in context.scripts[0]
|
||||
assert "gyxxWeekAdapterInstalled" in context.scripts[0]
|
||||
assert "addEventListener('mouseenter'" in context.scripts[0]
|
||||
@@ -0,0 +1,386 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
dy_store_competitor_store_scraping as dy_collector,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_data_collector,
|
||||
jd_peer_store_data_collector,
|
||||
)
|
||||
|
||||
|
||||
class _PlaywrightContext:
|
||||
def __init__(self, page: object) -> None:
|
||||
self.page = page
|
||||
|
||||
def __enter__(self):
|
||||
context = SimpleNamespace(
|
||||
new_page=lambda: self.page,
|
||||
close=lambda: None,
|
||||
)
|
||||
browser = SimpleNamespace(
|
||||
new_context=lambda **kwargs: context,
|
||||
close=lambda: None,
|
||||
)
|
||||
chromium = SimpleNamespace(launch=lambda **kwargs: browser)
|
||||
return SimpleNamespace(chromium=chromium)
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_jd_shop_collector_returns_nonzero_when_collection_fails(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_login",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")),
|
||||
)
|
||||
|
||||
assert jd_data_collector.main([]) == 1
|
||||
|
||||
|
||||
def test_jd_shop_login_preserves_source_account_password_flow() -> None:
|
||||
events: list[tuple[str, object]] = []
|
||||
|
||||
class Field:
|
||||
def fill(self, value: str) -> None:
|
||||
events.append(("fill", value))
|
||||
|
||||
def click(self) -> None:
|
||||
events.append(("click", "login"))
|
||||
|
||||
fields = {
|
||||
'input[placeholder="请输入账号名/邮箱"]': Field(),
|
||||
'input[placeholder="请输入登录密码"]': Field(),
|
||||
'button:has-text("立即登录")': Field(),
|
||||
}
|
||||
page = SimpleNamespace(
|
||||
goto=lambda url, **kwargs: events.append(("goto", (url, kwargs))),
|
||||
wait_for_timeout=lambda timeout: events.append(("wait", timeout)),
|
||||
wait_for_load_state=lambda *_args, **_kwargs: None,
|
||||
locator=lambda selector: fields[selector],
|
||||
evaluate=lambda script: events.append(("evaluate", script)),
|
||||
keyboard=SimpleNamespace(
|
||||
press=lambda key: events.append(("key", key)),
|
||||
),
|
||||
)
|
||||
|
||||
assert jd_data_collector.step_login(page, "account", "credential") is None
|
||||
assert ("fill", "account") in events
|
||||
assert ("fill", "credential") in events
|
||||
assert ("click", "login") in events
|
||||
|
||||
|
||||
def test_jd_peer_collector_returns_nonzero_when_collection_fails(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"sync_playwright",
|
||||
lambda: _PlaywrightContext(object()),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_login",
|
||||
lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")),
|
||||
)
|
||||
|
||||
assert jd_peer_store_data_collector.main([]) == 1
|
||||
|
||||
|
||||
def test_dy_callback_preserves_failure_for_main_process(monkeypatch) -> None:
|
||||
failure = RuntimeError("date control changed")
|
||||
persisted = []
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(failure),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_save_browser_state",
|
||||
lambda *args: persisted.append(args),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="date control changed"):
|
||||
dy_collector._page_automation(object())
|
||||
|
||||
assert dy_collector._automation_error is failure
|
||||
assert persisted == []
|
||||
|
||||
|
||||
def test_dy_outer_callback_persists_final_cross_domain_state(monkeypatch) -> None:
|
||||
events = []
|
||||
page = SimpleNamespace(
|
||||
url="https://fxg.jinritemai.com/ffa/eco/experience-score"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda actual_page: events.append(("source", actual_page)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_save_browser_state",
|
||||
lambda actual_page, stage: events.append(("persist", actual_page, stage)),
|
||||
)
|
||||
|
||||
dy_collector._page_automation(page)
|
||||
|
||||
assert events == [
|
||||
("source", page),
|
||||
("persist", page, "after cross-domain stages"),
|
||||
]
|
||||
assert dy_collector._automation_error is None
|
||||
|
||||
|
||||
def test_dy_final_state_snapshot_keeps_all_cross_domain_cookies(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
cookie_file = tmp_path / "cookies.json"
|
||||
storage_file = tmp_path / "storage_state.json"
|
||||
cookies = [
|
||||
{"name": "compass", "value": "one", "domain": ".jinritemai.com"},
|
||||
{"name": "qianchuan", "value": "two", "domain": ".oceanengine.com"},
|
||||
{"name": "fxg", "value": "three", "domain": "fxg.jinritemai.com"},
|
||||
]
|
||||
|
||||
class Context:
|
||||
def cookies(self):
|
||||
return cookies
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
with open(path, "w", encoding="utf-8") as stream:
|
||||
json.dump({"cookies": cookies, "origins": []}, stream)
|
||||
|
||||
monkeypatch.setattr(dy_collector, "COOKIES_FILE", str(cookie_file))
|
||||
monkeypatch.setattr(dy_collector, "STORAGE_STATE_FILE", str(storage_file))
|
||||
|
||||
dy_collector._save_browser_state(
|
||||
SimpleNamespace(context=Context()),
|
||||
"after cross-domain stages",
|
||||
)
|
||||
|
||||
saved_cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
|
||||
saved_storage = json.loads(storage_file.read_text(encoding="utf-8"))
|
||||
assert {item["domain"] for item in saved_cookies} == {
|
||||
".jinritemai.com",
|
||||
".oceanengine.com",
|
||||
"fxg.jinritemai.com",
|
||||
}
|
||||
assert saved_storage["cookies"] == cookies
|
||||
|
||||
|
||||
def test_dy_main_returns_nonzero_when_fetcher_swallows_callback_failure(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=True),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(RuntimeError("date control changed")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"save_results",
|
||||
lambda data: (_ for _ in ()).throw(
|
||||
AssertionError("failed automation must not save fallback data")
|
||||
),
|
||||
)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
try:
|
||||
page_action(object())
|
||||
except RuntimeError:
|
||||
pass
|
||||
return SimpleNamespace(url=url, status=200)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 1
|
||||
|
||||
|
||||
def test_dy_main_treats_final_login_response_as_cookie_skip(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=True),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(dy_collector, "_session_expired_exit_code", lambda: 75)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda page: (_ for _ in ()).throw(RuntimeError("navigation aborted")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"save_results",
|
||||
lambda data: (_ for _ in ()).throw(
|
||||
AssertionError("expired session must not save fallback data")
|
||||
),
|
||||
)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
try:
|
||||
page_action(object())
|
||||
except RuntimeError:
|
||||
pass
|
||||
return SimpleNamespace(
|
||||
url="https://compass.jinritemai.com/login?roleType=shop",
|
||||
status=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 75
|
||||
assert isinstance(dy_collector._automation_error, RuntimeError)
|
||||
|
||||
|
||||
def test_dy_production_keeps_source_success_after_optional_page_redirect(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module"))
|
||||
monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store"))
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"COMPETITOR_DATA_DIR",
|
||||
str(tmp_path / "competitor"),
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile"))
|
||||
monkeypatch.setattr(dy_collector, "_load_cookies", lambda: [])
|
||||
monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None)
|
||||
monkeypatch.setattr(dy_collector, "_results_saved", True)
|
||||
monkeypatch.setattr(dy_collector, "_automation_error", None)
|
||||
monkeypatch.setattr(dy_collector, "_page_automation_impl", lambda page: None)
|
||||
|
||||
def fake_fetch(url, *, page_action, **kwargs):
|
||||
page_action(SimpleNamespace(url=url))
|
||||
return SimpleNamespace(
|
||||
url="https://fxg.jinritemai.com/login/common",
|
||||
status=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch)
|
||||
|
||||
assert dy_collector.main() == 0
|
||||
|
||||
|
||||
def test_dy_login_redirect_is_not_treated_as_business_page() -> None:
|
||||
assert dy_collector._is_login_page_url(
|
||||
"https://fxg.jinritemai.com/login/common"
|
||||
)
|
||||
assert not dy_collector._is_login_page_url(
|
||||
"https://compass.jinritemai.com/shop/business-part"
|
||||
)
|
||||
|
||||
|
||||
def test_dy_login_form_is_detected_when_spa_url_does_not_change() -> None:
|
||||
counts = {
|
||||
'input[name="mobile"]': 1,
|
||||
'input[name="mobilecaptcha"]': 1,
|
||||
".account-center-action-button": 1,
|
||||
}
|
||||
page = SimpleNamespace(
|
||||
locator=lambda selector: SimpleNamespace(count=lambda: counts[selector])
|
||||
)
|
||||
|
||||
assert dy_collector._has_interactive_login_form(page)
|
||||
|
||||
|
||||
def test_dy_acceptance_login_redirect_returns_cookie_skip(monkeypatch) -> None:
|
||||
recorded = []
|
||||
|
||||
class Policy:
|
||||
enabled = True
|
||||
|
||||
def record(self, event, **payload):
|
||||
recorded.append((event, payload))
|
||||
|
||||
monkeypatch.setattr(dy_collector, "current_acceptance_policy", lambda: Policy())
|
||||
|
||||
assert dy_collector._session_expired_exit_code() == 75
|
||||
assert recorded[0][0] == "cookie_skipped"
|
||||
assert recorded[0][1]["details"]["status"] == "SKIPPED_COOKIE"
|
||||
|
||||
|
||||
def test_dy_session_expiry_is_classified_by_outer_callback(monkeypatch) -> None:
|
||||
class Policy:
|
||||
enabled = True
|
||||
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"current_acceptance_policy",
|
||||
lambda: Policy(),
|
||||
)
|
||||
page = SimpleNamespace(
|
||||
url="https://compass.jinritemai.com/login?roleType=shop",
|
||||
)
|
||||
monkeypatch.setattr(dy_collector, "_save_browser_debug", lambda *args: None)
|
||||
monkeypatch.setattr(
|
||||
dy_collector,
|
||||
"_page_automation_impl",
|
||||
lambda *_args: (_ for _ in ()).throw(
|
||||
AssertionError("expired session must not enter source business logic")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
dy_collector.BrowserSessionExpiredError,
|
||||
match="redirected to an interactive login page",
|
||||
):
|
||||
dy_collector._page_automation(page)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
|
||||
|
||||
class _Cursor:
|
||||
rowcount = 1
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _Connection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _Cursor()
|
||||
|
||||
def cursor(self) -> _Cursor:
|
||||
return self.cursor_instance
|
||||
|
||||
|
||||
def test_dynamic_batch_upsert_reports_all_input_records(monkeypatch) -> None:
|
||||
records = [
|
||||
{"week_start": "2026-07-27", "brand": "A", "rank": 1},
|
||||
{"week_start": "2026-07-27", "brand": "B", "rank": 2},
|
||||
{"week_start": "2026-07-27", "brand": "C", "rank": 3},
|
||||
]
|
||||
executed = {}
|
||||
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"_get_table_columns",
|
||||
lambda conn, table: {"week_start", "brand", "rank"},
|
||||
)
|
||||
|
||||
def fake_execute_batch(cursor, sql, data, *, page_size):
|
||||
executed.update(
|
||||
cursor=cursor,
|
||||
sql=sql,
|
||||
data=data,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(shop_db.psycopg2.extras, "execute_batch", fake_execute_batch)
|
||||
|
||||
connection = _Connection()
|
||||
persisted = shop_db._dynamic_batch_upsert(
|
||||
connection,
|
||||
"jd_peer_store_ranking",
|
||||
records,
|
||||
("week_start", "brand"),
|
||||
)
|
||||
|
||||
assert connection.cursor_instance.rowcount == 1
|
||||
assert persisted == len(records)
|
||||
assert executed["cursor"] is connection.cursor_instance
|
||||
assert executed["data"] == [
|
||||
["2026-07-27", "A", 1],
|
||||
["2026-07-27", "B", 2],
|
||||
["2026-07-27", "C", 3],
|
||||
]
|
||||
assert executed["page_size"] == 200
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,196 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
dy_store_competitor_store_scraping as collector,
|
||||
)
|
||||
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, *, count: int = 1) -> None:
|
||||
self._count = count
|
||||
self.clicked = False
|
||||
self.waited = False
|
||||
|
||||
@property
|
||||
def first(self):
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.clicked = True
|
||||
|
||||
def wait_for(self, **_kwargs) -> None:
|
||||
self.waited = True
|
||||
|
||||
|
||||
def test_market_category_controls_are_bound_to_visible_cascader() -> None:
|
||||
selectors: list[str] = []
|
||||
closed_menu = _Locator(count=0)
|
||||
trigger = _Locator()
|
||||
opened_menu = _Locator()
|
||||
|
||||
class Page:
|
||||
def locator(self, selector: str):
|
||||
selectors.append(selector)
|
||||
if selector == ".aurora-cascader-menu:visible":
|
||||
return closed_menu if selectors.count(selector) == 1 else opened_menu
|
||||
if selector == ".aurora-form-item-control .aurora-cascader:visible":
|
||||
return trigger
|
||||
raise AssertionError(selector)
|
||||
|
||||
assert collector._click_market_category_dropdown(Page())
|
||||
assert trigger.clicked
|
||||
assert opened_menu.waited
|
||||
|
||||
|
||||
def test_market_category_option_uses_visible_titled_menu_item() -> None:
|
||||
located: list[str] = []
|
||||
option = _Locator()
|
||||
page = SimpleNamespace(
|
||||
locator=lambda selector: located.append(selector) or option,
|
||||
)
|
||||
|
||||
assert collector._click_market_category_option(page, "双肩包")
|
||||
assert located == [
|
||||
'.aurora-cascader-menu '
|
||||
'li.aurora-cascader-menu-item[role="menuitemcheckbox"]'
|
||||
'[title="双肩包"]:visible'
|
||||
]
|
||||
assert option.waited
|
||||
assert option.clicked
|
||||
|
||||
|
||||
def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None:
|
||||
expected = collector._expected_market_category_path("双肩包")
|
||||
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
|
||||
signatures = iter(["old rows", "new rows"])
|
||||
clicked: list[str] = []
|
||||
refresh_checks: list[tuple[str, str]] = []
|
||||
page = SimpleNamespace(
|
||||
evaluate=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_selected_market_category_path",
|
||||
lambda _page: next(paths),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_market_table_signature",
|
||||
lambda _page: next(signatures),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_dropdown",
|
||||
lambda _page: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_option",
|
||||
lambda _page, label: clicked.append(label) or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_market_category_refresh",
|
||||
lambda _page, *, expected_path, previous_signature: refresh_checks.append(
|
||||
(expected_path, previous_signature)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None)
|
||||
|
||||
collector._select_market_category(page, "双肩包")
|
||||
|
||||
assert clicked == ["鞋靴箱包", "箱包", "时尚箱包", "双肩包"]
|
||||
assert refresh_checks == [(expected, "old rows")]
|
||||
|
||||
|
||||
def test_select_market_category_rejects_stale_table(monkeypatch) -> None:
|
||||
expected = collector._expected_market_category_path("双肩包")
|
||||
paths = iter([collector._expected_market_category_path("腰包/胸包"), expected])
|
||||
page = SimpleNamespace(
|
||||
evaluate=lambda *_args, **_kwargs: None,
|
||||
wait_for_timeout=lambda *_args, **_kwargs: None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_selected_market_category_path",
|
||||
lambda _page: next(paths),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_market_table_signature",
|
||||
lambda _page: "same rows",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_dropdown",
|
||||
lambda _page: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_click_market_category_option",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_market_category_refresh",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None)
|
||||
|
||||
with pytest.raises(
|
||||
collector.MarketCategorySelectionError,
|
||||
match="still contains stale rows",
|
||||
):
|
||||
collector._select_market_category(page, "双肩包")
|
||||
|
||||
|
||||
def test_market_ranking_collection_never_extracts_after_selection_failure(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
extracted: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_previous_week_range",
|
||||
lambda: (date(2026, 7, 27), date(2026, 8, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_open_market_rank_shop_page",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_market_previous_week",
|
||||
lambda *_args, **_kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_market_category",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
||||
collector.MarketCategorySelectionError("category did not change")
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_extract_market_ranking",
|
||||
lambda _page, category: extracted.append(category),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
collector.MarketCategorySelectionError,
|
||||
match="category did not change",
|
||||
):
|
||||
collector._collect_market_rankings(object())
|
||||
|
||||
assert extracted == []
|
||||
@@ -0,0 +1,359 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli
|
||||
from gyxx_flow.modules.shop_intelligence.config import (
|
||||
DY_PEER_BASE_FIELDS,
|
||||
DY_PEER_CAT_MAP,
|
||||
JD_PEER_BASE_FIELDS,
|
||||
JD_PEER_BRAND_MAP,
|
||||
JD_PEER_CATEGORY_MAP,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.writers import peer_store_writer
|
||||
|
||||
|
||||
def test_shop_idempotency_reads_all_pages_and_normalizes_month(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
offsets: list[int] = []
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
payload = json.loads(args[-1])
|
||||
offsets.append(payload["offset"])
|
||||
if payload["offset"] == 0:
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["JD"], ["2026.7月"]]],
|
||||
"record_id_list": ["other-platform"],
|
||||
"has_more": True,
|
||||
}
|
||||
}
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["DY"], ["2026.7月"]]],
|
||||
"record_id_list": ["target"],
|
||||
"has_more": False,
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run)
|
||||
|
||||
record_id = lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{"时间": "time", "平台": "platform", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "target"
|
||||
assert offsets == [0, 1]
|
||||
|
||||
|
||||
def test_shop_idempotency_fails_closed_when_identity_index_is_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time", "month"],
|
||||
"data": [],
|
||||
"record_id_list": [],
|
||||
"has_more": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="平台"):
|
||||
lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{"时间": "time", "平台": "platform", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
|
||||
def test_record_search_pagination_has_a_hard_page_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(lark_cli, "_MAX_RECORD_SEARCH_PAGES", 1)
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time"],
|
||||
"data": [["7月第5周（7.27-8.2）"]],
|
||||
"record_id_list": ["record-1"],
|
||||
"has_more": True,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="安全上限"):
|
||||
lark_cli.search_record_pages(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
{
|
||||
"keyword": "7月第5周",
|
||||
"search_fields": ["time"],
|
||||
"select_fields": ["time"],
|
||||
"limit": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_peer_idempotency_reads_all_pages_and_requires_identity_indexes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
offsets: list[int] = []
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
payload = json.loads(args[-1])
|
||||
offsets.append(payload["offset"])
|
||||
if payload["offset"] == 0:
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "brand", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["other"], ["2026.7月"]]],
|
||||
"record_id_list": ["other-brand"],
|
||||
"has_more": True,
|
||||
}
|
||||
}
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "brand", "month"],
|
||||
"data": [["7月第5周（7.27-8.02）", ["NIID"], ["2026.7月"]]],
|
||||
"record_id_list": ["target"],
|
||||
"has_more": False,
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run)
|
||||
|
||||
record_id = peer_store_writer._find_existing_peer_record(
|
||||
"node",
|
||||
"script",
|
||||
"table",
|
||||
{"时间": "time", "竞店名称": "brand", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"NIID",
|
||||
"2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "target"
|
||||
assert offsets == [0, 1]
|
||||
|
||||
monkeypatch.setattr(
|
||||
lark_cli,
|
||||
"run_lark_cli",
|
||||
lambda *_args, **_kwargs: {
|
||||
"data": {
|
||||
"fields": ["time", "month"],
|
||||
"data": [],
|
||||
"record_id_list": [],
|
||||
"has_more": False,
|
||||
}
|
||||
},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="竞店名称"):
|
||||
peer_store_writer._find_existing_peer_record(
|
||||
"node",
|
||||
"script",
|
||||
"table",
|
||||
{"时间": "time", "竞店名称": "brand", "月份": "month"},
|
||||
"7月第5周（7.27-8.2）",
|
||||
"NIID",
|
||||
"2026.07月",
|
||||
)
|
||||
|
||||
|
||||
def test_jd_rerun_explicitly_clears_only_na_category(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
brand_json_name = next(iter(JD_PEER_BRAND_MAP))
|
||||
category_names = list(JD_PEER_CATEGORY_MAP)
|
||||
artifact = tmp_path / "jd-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"数据周期": "2026-07-27 ~ 2026-08-02",
|
||||
"数据": {
|
||||
brand_json_name: {
|
||||
category_names[0]: {"rank": "N/A"},
|
||||
category_names[1]: {"rank": "7"},
|
||||
category_names[2]: {"rank": "3"},
|
||||
}
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
writes: list[dict] = []
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
|
||||
assert peer_store_writer._write_jd_peer_store(str(artifact)) is True
|
||||
assert len(writes) == 1
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[0]]] is None
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[1]]] == 7
|
||||
assert writes[0][JD_PEER_CATEGORY_MAP[category_names[2]]] == 3
|
||||
|
||||
|
||||
def test_dy_brand_iteration_uses_both_category_brand_sets(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "dy-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"results": {
|
||||
DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}},
|
||||
DY_PEER_CAT_MAP["双肩包"]: {"beta": {"rank": "22"}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
writes: list[dict] = []
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_brand_options_cache",
|
||||
lambda *_args: {"alpha": "Alpha", "beta": "Beta"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
|
||||
assert peer_store_writer._write_dy_peer_store(str(artifact)) is True
|
||||
assert [row[DY_PEER_BASE_FIELDS["竞店名称"]] for row in writes] == [
|
||||
"Alpha",
|
||||
"Beta",
|
||||
]
|
||||
|
||||
|
||||
def test_dy_local_shop_validation_failure_propagates_false(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
peer_artifact = tmp_path / "dy-peer.json"
|
||||
peer_artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"results": {
|
||||
DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}},
|
||||
DY_PEER_CAT_MAP["双肩包"]: {"alpha": {"rank": "22"}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_artifact = tmp_path / "dy-shop.json"
|
||||
shop_artifact.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, ""))
|
||||
monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script"))
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_brand_options_cache",
|
||||
lambda *_args: {"alpha": "Alpha"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_build_month_options_cache",
|
||||
lambda *_args: {"2026.07月": "2026.07月"},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: {"data": {"record_id_list": ["existing"]}},
|
||||
)
|
||||
monkeypatch.setattr(peer_store_writer, "_write_dy_local_shop_row", lambda *_args: False)
|
||||
|
||||
assert (
|
||||
peer_store_writer._write_dy_peer_store(
|
||||
str(peer_artifact),
|
||||
str(shop_artifact),
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_dy_local_shop_rankings_return_false_without_upsert(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "shop.json"
|
||||
artifact.write_text(json.dumps({"market_rankings": {}}), encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"validate_market_rankings",
|
||||
lambda *_args: (False, "missing rankings"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
peer_store_writer,
|
||||
"_upsert_peer_record",
|
||||
lambda *_args: pytest.fail("invalid local rankings must not be written"),
|
||||
)
|
||||
|
||||
assert (
|
||||
peer_store_writer._write_dy_local_shop_row(
|
||||
"node",
|
||||
"script",
|
||||
"7月第5周（7.27-8.2）",
|
||||
"2026.07月",
|
||||
str(artifact),
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_configured_peer_identity_fields_are_non_empty() -> None:
|
||||
for fields in (JD_PEER_BASE_FIELDS, DY_PEER_BASE_FIELDS):
|
||||
assert fields["时间"]
|
||||
assert fields["月份"]
|
||||
assert fields["竞店名称"]
|
||||
@@ -0,0 +1,257 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_peer_store_data_collector as collector,
|
||||
)
|
||||
|
||||
|
||||
class _HiddenMenuLocator:
|
||||
def count(self) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
class _Page:
|
||||
def __init__(self) -> None:
|
||||
self.keyboard = SimpleNamespace(press=lambda _key: None)
|
||||
|
||||
def locator(self, _selector: str) -> _HiddenMenuLocator:
|
||||
return _HiddenMenuLocator()
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_category_path_normalization_is_exact() -> None:
|
||||
assert collector._normalize_category_path(" 男包 → 男士双肩包 ") == (
|
||||
"男包 > 男士双肩包"
|
||||
)
|
||||
assert collector._expected_category_path("男士双肩包", "男包") == (
|
||||
"男包 > 男士双肩包"
|
||||
)
|
||||
assert collector._expected_category_path("男包", None) == "男包"
|
||||
|
||||
|
||||
def test_category_menu_closes_stale_portals_before_opening_selector() -> None:
|
||||
class Menus:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
self.first = self
|
||||
|
||||
def count(self) -> int:
|
||||
return self.page.menu_count
|
||||
|
||||
def wait_for(self, **_kwargs) -> None:
|
||||
return None
|
||||
|
||||
class Trigger:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.page.menu_count = 1
|
||||
|
||||
class Selector:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def locator(self, _selector: str) -> Trigger:
|
||||
return Trigger(self.page)
|
||||
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.menu_count = 2
|
||||
self.escapes = 0
|
||||
self.keyboard = SimpleNamespace(press=self.press)
|
||||
|
||||
def press(self, key: str) -> None:
|
||||
assert key == "Escape"
|
||||
self.escapes += 1
|
||||
self.menu_count = 0
|
||||
|
||||
def locator(self, _selector: str) -> Menus:
|
||||
return Menus(self)
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
page = Page()
|
||||
|
||||
menu = collector._visible_category_menu(page, Selector(page))
|
||||
|
||||
assert menu.count() == 1
|
||||
assert page.escapes == 1
|
||||
|
||||
|
||||
def test_category_menu_targets_only_outer_dropdown_panel() -> None:
|
||||
source = __import__("inspect").getsource(collector._visible_category_menu)
|
||||
|
||||
assert ".jmtd-dropdown-panel.jmtd-dropdown-lists:visible" in source
|
||||
|
||||
|
||||
def test_successful_category_switch_closes_its_dropdown_portal(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class Menus:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def count(self) -> int:
|
||||
return self.page.menu_count
|
||||
|
||||
class Trigger:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def click(self, **_kwargs) -> None:
|
||||
self.page.menu_count = 0
|
||||
|
||||
class Selector:
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
|
||||
def locator(self, _selector: str) -> Trigger:
|
||||
return Trigger(self.page)
|
||||
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.menu_count = 1
|
||||
self.keyboard = SimpleNamespace(press=lambda _key: None)
|
||||
|
||||
def locator(self, _selector: str) -> Menus:
|
||||
return Menus(self)
|
||||
|
||||
def wait_for_timeout(self, _timeout_ms: int) -> None:
|
||||
return None
|
||||
|
||||
page = Page()
|
||||
selector = Selector(page)
|
||||
monkeypatch.setattr(collector, "_visible_category_selector", lambda _page: selector)
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before")
|
||||
monkeypatch.setattr(
|
||||
collector, "_selected_category_path", lambda _selector: "男包 > 男士双肩包"
|
||||
)
|
||||
|
||||
assert collector.switch_category(page, "男士双肩包", "男包") is True
|
||||
assert page.menu_count == 0
|
||||
|
||||
|
||||
def test_ranking_refresh_accepts_only_explicit_empty_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "")
|
||||
monkeypatch.setattr(
|
||||
collector, "_ranking_table_is_explicitly_empty", lambda _page: True
|
||||
)
|
||||
|
||||
assert collector._wait_for_ranking_table_refresh(
|
||||
object(), "previous", timeout_ms=1
|
||||
) is True
|
||||
|
||||
|
||||
def test_ranking_refresh_rejects_unverified_empty_state(monkeypatch) -> None:
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "")
|
||||
monkeypatch.setattr(
|
||||
collector, "_ranking_table_is_explicitly_empty", lambda _page: False
|
||||
)
|
||||
monkeypatch.setattr(collector.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert collector._wait_for_ranking_table_refresh(
|
||||
object(), "previous", timeout_ms=0
|
||||
) is False
|
||||
|
||||
|
||||
def test_switch_category_requires_selected_state_and_table_refresh(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
page = _Page()
|
||||
selector = object()
|
||||
menu = object()
|
||||
observed: list[object] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_visible_category_selector",
|
||||
lambda actual_page: selector if actual_page is page else None,
|
||||
)
|
||||
monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before")
|
||||
monkeypatch.setattr(collector, "_selected_category_path", lambda _selector: "男包")
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_visible_category_menu",
|
||||
lambda actual_page, actual_selector: (
|
||||
menu if (actual_page, actual_selector) == (page, selector) else None
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_category_option",
|
||||
lambda actual_page, actual_menu, category, parent: observed.append(
|
||||
(actual_page, actual_menu, category, parent)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_selected_category",
|
||||
lambda actual_selector, expected: (
|
||||
observed.append((actual_selector, expected, "selected")) or True
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_wait_for_ranking_table_refresh",
|
||||
lambda actual_page, previous: (
|
||||
observed.append((actual_page, previous, "refreshed")) or True
|
||||
),
|
||||
)
|
||||
|
||||
assert collector.switch_category(page, "男士双肩包", "男包") is True
|
||||
assert observed == [
|
||||
(page, menu, "男士双肩包", "男包"),
|
||||
(selector, "男包 > 男士双肩包", "selected"),
|
||||
(page, "before", "refreshed"),
|
||||
]
|
||||
|
||||
|
||||
def test_failed_category_switch_never_extracts_previous_table(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
page = _Page()
|
||||
extracted: list[str] = []
|
||||
|
||||
monkeypatch.setattr(collector, "SHOPS", ["peer-shop"])
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"CATEGORIES",
|
||||
[("男士双肩包", "男包"), ("男士单肩/斜挎包", "男包")],
|
||||
)
|
||||
monkeypatch.setattr(collector, "select_last_week_any_day", lambda _page: None)
|
||||
monkeypatch.setattr(collector, "search_shop", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"switch_category",
|
||||
lambda _page, category, _parent: category == "男士单肩/斜挎包",
|
||||
)
|
||||
|
||||
def extract_rows(_page):
|
||||
extracted.append("called")
|
||||
return [
|
||||
{
|
||||
"rank": "7",
|
||||
"shop": "peer-shop",
|
||||
"amount": "￥1万~￥2万",
|
||||
"orders": "10~50",
|
||||
"followers": "3",
|
||||
"visitors": "1,000~2,000",
|
||||
"searchClicks": "600~800",
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(collector, "extract_ranking_table", extract_rows)
|
||||
|
||||
result = collector.step_collect_competitor_data(page)["peer-shop"]
|
||||
|
||||
assert result["男士双肩包"]["rank"] == "N/A"
|
||||
assert "禁止复用" not in result["男士双肩包"]["error"]
|
||||
assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"]
|
||||
assert result["男士单肩/斜挎包"]["rank"] == "7"
|
||||
assert extracted == ["called"]
|
||||
@@ -0,0 +1,225 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_self_operated_brand_daily as brand_daily,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily import (
|
||||
browser_runtime_options,
|
||||
build_db_records,
|
||||
is_authenticated_shop_url,
|
||||
iter_dates,
|
||||
parse_decimal,
|
||||
parse_percent,
|
||||
resolve_account_identifier,
|
||||
)
|
||||
|
||||
|
||||
def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch):
|
||||
monkeypatch.delenv("GYXX_BROWSER_CHANNEL", raising=False)
|
||||
|
||||
assert browser_runtime_options() == {}
|
||||
|
||||
|
||||
def test_browser_runtime_can_use_installed_chrome(monkeypatch):
|
||||
monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome")
|
||||
|
||||
assert browser_runtime_options() == {"channel": "chrome"}
|
||||
|
||||
|
||||
class DateRangeTests(unittest.TestCase):
|
||||
def test_iter_dates_is_inclusive(self):
|
||||
self.assertEqual(
|
||||
list(iter_dates(date(2026, 6, 29), date(2026, 7, 2))),
|
||||
[
|
||||
date(2026, 6, 29),
|
||||
date(2026, 6, 30),
|
||||
date(2026, 7, 1),
|
||||
date(2026, 7, 2),
|
||||
],
|
||||
)
|
||||
|
||||
def test_iter_dates_rejects_reverse_range(self):
|
||||
with self.assertRaisesRegex(ValueError, "start_date"):
|
||||
list(iter_dates(date(2026, 7, 2), date(2026, 7, 1)))
|
||||
|
||||
|
||||
class LoginStateTests(unittest.TestCase):
|
||||
def test_shop_page_is_authenticated(self):
|
||||
self.assertTrue(
|
||||
is_authenticated_shop_url(
|
||||
"https://shop.jd.com/jdm/vc/data-management/performance/brandPerformance"
|
||||
)
|
||||
)
|
||||
|
||||
def test_passport_and_security_pages_are_not_authenticated(self):
|
||||
self.assertFalse(
|
||||
is_authenticated_shop_url(
|
||||
"https://passport.shop.jd.com/login/index.action/jdm"
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
is_authenticated_shop_url("https://aq.jd.com/certified/index?p=abc")
|
||||
)
|
||||
|
||||
|
||||
class _FakeLocator:
|
||||
def __init__(self, value=None):
|
||||
self.value = value
|
||||
self.first = self
|
||||
|
||||
def count(self):
|
||||
return int(self.value is not None)
|
||||
|
||||
def get_attribute(self, name):
|
||||
assert name == "title"
|
||||
return self.value
|
||||
|
||||
def inner_text(self):
|
||||
return self.value or ""
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, values, cookies=None):
|
||||
self.values = values
|
||||
self.context = SimpleNamespace(cookies=lambda: cookies or [])
|
||||
|
||||
def locator(self, selector):
|
||||
return _FakeLocator(self.values.get(selector))
|
||||
|
||||
|
||||
class AccountIdentifierTests(unittest.TestCase):
|
||||
def test_configured_account_takes_priority(self):
|
||||
self.assertEqual(
|
||||
resolve_account_identifier(_FakePage({}), " gyxx2022 "),
|
||||
"gyxx2022",
|
||||
)
|
||||
|
||||
def test_cookie_session_uses_signed_in_pin(self):
|
||||
page = _FakePage(
|
||||
{
|
||||
".shop-menu-accountV1__account-dropdown .content-pin[title]": (
|
||||
"gyxx2022"
|
||||
),
|
||||
".shop-menu-accountV1__right-account-top-name[title]": (
|
||||
"惠州鑫华达科技有限公司"
|
||||
),
|
||||
}
|
||||
)
|
||||
self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022")
|
||||
|
||||
def test_cookie_session_falls_back_to_company_name(self):
|
||||
page = _FakePage(
|
||||
{
|
||||
".shop-menu-accountV1__right-account-top-name[title]": (
|
||||
"惠州鑫华达科技有限公司"
|
||||
)
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
resolve_account_identifier(page, ""),
|
||||
"惠州鑫华达科技有限公司",
|
||||
)
|
||||
|
||||
def test_cookie_session_uses_jd_pin_cookie_without_account_menu(self):
|
||||
page = _FakePage(
|
||||
{},
|
||||
cookies=[
|
||||
{
|
||||
"name": "pin",
|
||||
"value": "gyxx%32%30%32%32",
|
||||
"domain": ".jd.com",
|
||||
}
|
||||
],
|
||||
)
|
||||
self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022")
|
||||
|
||||
def test_cookie_session_requires_stable_identifier(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "无法识别当前京麦账号"):
|
||||
resolve_account_identifier(_FakePage({}), "")
|
||||
|
||||
|
||||
class ValueParsingTests(unittest.TestCase):
|
||||
def test_parse_decimal_handles_commas_and_currency(self):
|
||||
self.assertEqual(parse_decimal("¥34,605.80"), Decimal("34605.80"))
|
||||
self.assertEqual(parse_decimal("122.00"), Decimal("122.00"))
|
||||
|
||||
def test_parse_decimal_preserves_negative_values(self):
|
||||
self.assertEqual(parse_decimal("-2,173.20"), Decimal("-2173.20"))
|
||||
|
||||
def test_parse_decimal_returns_none_for_placeholder(self):
|
||||
for value in (None, "", "-", "--", "—", "N/A"):
|
||||
self.assertIsNone(parse_decimal(value))
|
||||
|
||||
def test_parse_percent_converts_display_percent_to_ratio(self):
|
||||
self.assertEqual(parse_percent("6.25%"), Decimal("0.0625"))
|
||||
self.assertEqual(parse_percent("0.06"), Decimal("0.06"))
|
||||
|
||||
|
||||
class RecordBuildingTests(unittest.TestCase):
|
||||
def test_build_db_records_maps_all_columns(self):
|
||||
rows = [
|
||||
{
|
||||
"品牌": "光影行星（GYXX）",
|
||||
"财务销量": "122.00",
|
||||
"收入": "34,605.80",
|
||||
"毛利": "2,173.20",
|
||||
"毛利率": "6.00%",
|
||||
"库存金额": "586,149.83",
|
||||
}
|
||||
]
|
||||
|
||||
records = build_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 29),
|
||||
rows=rows,
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
record = records[0]
|
||||
self.assertEqual(record["account"], "gyxx2022")
|
||||
self.assertEqual(record["data_date"], date(2026, 7, 29))
|
||||
self.assertEqual(record["brand"], "光影行星（GYXX）")
|
||||
self.assertEqual(record["financial_sales"], Decimal("122.00"))
|
||||
self.assertEqual(record["revenue"], Decimal("34605.80"))
|
||||
self.assertEqual(record["gross_profit"], Decimal("2173.20"))
|
||||
self.assertEqual(record["gross_margin"], Decimal("0.06"))
|
||||
self.assertEqual(record["inventory_amount"], Decimal("586149.83"))
|
||||
self.assertEqual(record["raw_row"]["收入"], "34,605.80")
|
||||
|
||||
def test_build_db_records_skips_empty_placeholder_row(self):
|
||||
records = build_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 29),
|
||||
rows=[{"品牌": "-", "收入": "-"}],
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
self.assertEqual(records, [])
|
||||
|
||||
|
||||
def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch):
|
||||
class FakePolicy:
|
||||
enabled = True
|
||||
|
||||
def preflight_cookie(self, binding, *, environment):
|
||||
return SimpleNamespace(should_skip=True, reason="browser state is missing")
|
||||
|
||||
monkeypatch.setattr(brand_daily, "current_acceptance_policy", lambda: FakePolicy())
|
||||
monkeypatch.setattr(brand_daily, "binding_from_environment", lambda: "binding")
|
||||
monkeypatch.setattr(
|
||||
brand_daily,
|
||||
"parse_args",
|
||||
lambda argv: (_ for _ in ()).throw(
|
||||
AssertionError("arguments must not be parsed after a cookie skip")
|
||||
),
|
||||
)
|
||||
|
||||
assert brand_daily.main([]) == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_self_operated_product_daily as product_daily,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily import (
|
||||
build_product_db_records,
|
||||
parse_total_count,
|
||||
validate_collected_rows,
|
||||
)
|
||||
|
||||
|
||||
class TotalCountTests(unittest.TestCase):
|
||||
def test_parse_total_count(self):
|
||||
self.assertEqual(parse_total_count("共50条"), 50)
|
||||
self.assertEqual(parse_total_count("共 1,234 条"), 1234)
|
||||
|
||||
def test_parse_total_count_rejects_unknown_text(self):
|
||||
with self.assertRaisesRegex(ValueError, "总条数"):
|
||||
parse_total_count("暂无数据")
|
||||
|
||||
|
||||
class ProductRecordTests(unittest.TestCase):
|
||||
def test_maps_all_product_columns(self):
|
||||
rows = [
|
||||
{
|
||||
"商品编码": "100232520894",
|
||||
"商品名称": "光影行星测试商品",
|
||||
"财务销量": "1.00",
|
||||
"收入": "145.14",
|
||||
"成本": "196.60",
|
||||
"毛利": "-51.46",
|
||||
"优惠券抵减数": "50.68",
|
||||
"积分抵减数": "0",
|
||||
"满返满减额": "0",
|
||||
"库存周转天数": "70.55",
|
||||
}
|
||||
]
|
||||
|
||||
records = build_product_db_records(
|
||||
account="gyxx2022",
|
||||
data_date=date(2026, 7, 30),
|
||||
rows=rows,
|
||||
source_url="https://shop.jd.com/example",
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 1)
|
||||
record = records[0]
|
||||
self.assertEqual(record["product_code"], "100232520894")
|
||||
self.assertEqual(record["product_name"], "光影行星测试商品")
|
||||
self.assertEqual(record["financial_sales"], Decimal("1.00"))
|
||||
self.assertEqual(record["revenue"], Decimal("145.14"))
|
||||
self.assertEqual(record["cost"], Decimal("196.60"))
|
||||
self.assertEqual(record["gross_profit"], Decimal("-51.46"))
|
||||
self.assertEqual(record["coupon_deduction"], Decimal("50.68"))
|
||||
self.assertEqual(record["points_deduction"], Decimal("0"))
|
||||
self.assertEqual(record["promotion_deduction"], Decimal("0"))
|
||||
self.assertEqual(record["inventory_turnover_days"], Decimal("70.55"))
|
||||
|
||||
def test_skips_row_without_product_code(self):
|
||||
self.assertEqual(
|
||||
build_product_db_records(
|
||||
"gyxx2022",
|
||||
date(2026, 7, 30),
|
||||
[{"商品编码": "-", "商品名称": "占位行"}],
|
||||
"https://shop.jd.com/example",
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
class PaginationValidationTests(unittest.TestCase):
|
||||
def test_accepts_complete_unique_rows(self):
|
||||
rows = [{"商品编码": "1"}, {"商品编码": "2"}]
|
||||
self.assertEqual(validate_collected_rows(rows, 2), rows)
|
||||
|
||||
def test_rejects_duplicate_product_codes(self):
|
||||
rows = [{"商品编码": "1"}, {"商品编码": "1"}]
|
||||
with self.assertRaisesRegex(RuntimeError, "重复"):
|
||||
validate_collected_rows(rows, 2)
|
||||
|
||||
def test_rejects_missing_rows(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "总条数"):
|
||||
validate_collected_rows([{"商品编码": "1"}], 2)
|
||||
|
||||
|
||||
def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
product_daily,
|
||||
"preflight_acceptance_login",
|
||||
lambda: COOKIE_SKIP_EXIT_CODE,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
product_daily,
|
||||
"parse_args",
|
||||
lambda argv: (_ for _ in ()).throw(
|
||||
AssertionError("arguments must not be parsed after a cookie skip")
|
||||
),
|
||||
)
|
||||
|
||||
assert product_daily.main([]) == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,517 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.catalog import WorkflowCatalog
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
|
||||
from gyxx_flow.modules.shop_intelligence.config import DY_PLATFORM, JD_PLATFORM
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
from gyxx_flow.modules.shop_intelligence.quality import validate_peer_payload
|
||||
|
||||
run_peer_store = import_module(
|
||||
"gyxx_flow.modules.shop_intelligence.runners.run_peer_store"
|
||||
)
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
JD_BRANDS = (
|
||||
"Bellroy官方旗舰店",
|
||||
"tomtoc官方旗舰店",
|
||||
"探迹者TAJEZZO官方旗舰店",
|
||||
"BAGSMART旗舰店",
|
||||
"GROTTO个乐官方旗舰店",
|
||||
"NIID箱包旗舰店",
|
||||
"光影行星 GYXX官方旗舰店",
|
||||
)
|
||||
JD_CATEGORIES = (
|
||||
"男士包袋",
|
||||
"男士双肩包",
|
||||
"男士单肩/斜挎包",
|
||||
"电脑包",
|
||||
"男士腰包/胸包",
|
||||
"旅行包",
|
||||
"休闲运动包",
|
||||
)
|
||||
DY_BRANDS = (
|
||||
"Bellroy",
|
||||
"tomtoc",
|
||||
"BAGSMART",
|
||||
"GROTTO",
|
||||
"NIID",
|
||||
"reeyee",
|
||||
"VAOPER",
|
||||
"探迹者",
|
||||
)
|
||||
DY_CATEGORIES = ("单肩包/斜挎包", "双肩包", "腰包/胸包")
|
||||
|
||||
|
||||
def _jd_ranked_entry(brand: str, seed: int) -> dict:
|
||||
return {
|
||||
"rank": str(seed + 1),
|
||||
"shop": brand,
|
||||
"amount": f"￥{seed + 10}万~￥{seed + 20}万",
|
||||
"orders": f"{seed + 20}~{seed + 30}",
|
||||
"followers": str(seed + 40),
|
||||
"visitors": f"{seed + 50}~{seed + 60}",
|
||||
"searchClicks": str(seed + 70),
|
||||
}
|
||||
|
||||
|
||||
def _jd_unranked_entry(brand: str) -> dict:
|
||||
return {
|
||||
"rank": "N/A",
|
||||
"amount": "N/A",
|
||||
"orders": "N/A",
|
||||
"followers": "N/A",
|
||||
"visitors": "N/A",
|
||||
"searchClicks": "N/A",
|
||||
"error": f"当前类目榜单未找到店铺: {brand}",
|
||||
}
|
||||
|
||||
|
||||
def _jd_payload() -> dict:
|
||||
return {
|
||||
"数据": {
|
||||
brand: {
|
||||
category: _jd_ranked_entry(brand, brand_index * 20 + category_index)
|
||||
for category_index, category in enumerate(JD_CATEGORIES)
|
||||
}
|
||||
for brand_index, brand in enumerate(JD_BRANDS)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _dy_range(seed: int) -> dict:
|
||||
return {
|
||||
"min": float(seed),
|
||||
"max": float(seed + 1),
|
||||
"open_ended": False,
|
||||
"raw_values": [str(seed)],
|
||||
}
|
||||
|
||||
|
||||
def _dy_entry(
|
||||
brand: str,
|
||||
category: str,
|
||||
seed: int,
|
||||
*,
|
||||
ranked: bool = True,
|
||||
matched: bool = True,
|
||||
) -> dict:
|
||||
rank = str(seed + 1) if ranked else ""
|
||||
stores = (
|
||||
[
|
||||
{
|
||||
"shop_name": f"{brand}旗舰店",
|
||||
"rank": rank,
|
||||
"row_text": f"{brand}-{seed}",
|
||||
}
|
||||
]
|
||||
if matched
|
||||
else []
|
||||
)
|
||||
return {
|
||||
"brand": brand,
|
||||
"category": category,
|
||||
"matched_shop_count": len(stores),
|
||||
"rank": rank,
|
||||
"rank_display": rank or "未进入前200名",
|
||||
"payment_amount": _dy_range(seed + 10),
|
||||
"order_count": _dy_range(seed + 20),
|
||||
"product_click_users": _dy_range(seed + 30),
|
||||
"stores": stores,
|
||||
"captured_at": "2026-08-04T10:00:00",
|
||||
}
|
||||
|
||||
|
||||
def _dy_payload() -> dict:
|
||||
return {
|
||||
"categories": list(DY_CATEGORIES),
|
||||
"brands": list(DY_BRANDS),
|
||||
"results": {
|
||||
category: {
|
||||
brand: _dy_entry(brand, category, category_index * 100 + brand_index)
|
||||
for brand_index, brand in enumerate(DY_BRANDS)
|
||||
}
|
||||
for category_index, category in enumerate(DY_CATEGORIES)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_runner_rejects_success_without_fresh_peer_artifact(monkeypatch) -> None:
|
||||
marker = ("D:/old-peer.json", 1, 10)
|
||||
monkeypatch.setattr(run_peer_store, "_artifact_marker", lambda _platform: marker)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "no fresh peer-store file" in result[2]
|
||||
|
||||
|
||||
def test_runner_passes_only_the_fresh_peer_artifact(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "new-peer.json"
|
||||
artifact.write_text(
|
||||
json.dumps(_jd_payload(), ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_artifact_marker",
|
||||
lambda _platform: next(markers),
|
||||
)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "ok"),
|
||||
)
|
||||
|
||||
assert result == (JD_PLATFORM, 0, "ok", str(artifact))
|
||||
|
||||
|
||||
def test_runner_rejects_identical_jd_payload_copied_across_categories(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "bad-peer.json"
|
||||
payload = _jd_payload()
|
||||
repeated = _jd_ranked_entry(JD_BRANDS[0], 22)
|
||||
for category in JD_CATEGORIES:
|
||||
payload["数据"][JD_BRANDS[0]][category] = repeated
|
||||
artifact.write_text(
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_artifact_marker",
|
||||
lambda _platform: next(markers),
|
||||
)
|
||||
|
||||
result = run_peer_store._run_with_fresh_artifact(
|
||||
JD_PLATFORM,
|
||||
lambda: (JD_PLATFORM, 0, "ok"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "identical payload copied across categories" in result[2]
|
||||
|
||||
|
||||
def test_quality_gate_allows_one_unranked_brand_when_other_brands_differ() -> None:
|
||||
payload = _jd_payload()
|
||||
brand = JD_BRANDS[0]
|
||||
payload["数据"][brand] = {
|
||||
category: _jd_unranked_entry(brand) for category in JD_CATEGORIES
|
||||
}
|
||||
|
||||
assert validate_peer_payload(JD_PLATFORM, payload) == (True, "")
|
||||
|
||||
|
||||
def test_quality_gate_rejects_explicit_category_switch_error() -> None:
|
||||
payload = _jd_payload()
|
||||
payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[1]] = {
|
||||
**_jd_unranked_entry(JD_BRANDS[0]),
|
||||
"error": "类目切换或榜单刷新失败",
|
||||
}
|
||||
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "类目切换或榜单刷新失败" in reason
|
||||
|
||||
|
||||
def test_direct_jd_peer_persistence_rejects_repeated_payload_before_connecting(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "bad-peer-direct.json"
|
||||
payload = _jd_payload()
|
||||
repeated = _jd_ranked_entry(JD_BRANDS[-2], 22)
|
||||
for category in JD_CATEGORIES:
|
||||
payload["数据"][JD_BRANDS[-2]][category] = repeated
|
||||
artifact.write_text(
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(shop_db, "DB_ENABLED", True)
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"get_conn",
|
||||
lambda: pytest.fail("invalid payload must not connect to PostgreSQL"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="JD peer-store data quality check failed"):
|
||||
shop_db.persist_jd_peer_store(str(artifact))
|
||||
|
||||
|
||||
def test_quality_gate_requires_exact_jd_brand_and_category_sets() -> None:
|
||||
missing_brand = _jd_payload()
|
||||
missing_brand["数据"].pop(JD_BRANDS[-1])
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, missing_brand)
|
||||
assert valid is False
|
||||
assert "brand set mismatch" in reason
|
||||
|
||||
missing_category = _jd_payload()
|
||||
missing_category["数据"][JD_BRANDS[0]].pop(JD_CATEGORIES[-1])
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, missing_category)
|
||||
assert valid is False
|
||||
assert "category set mismatch" in reason
|
||||
|
||||
|
||||
def test_quality_gate_rejects_malformed_jd_ranking_entry() -> None:
|
||||
payload = _jd_payload()
|
||||
payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[0]] = "23"
|
||||
|
||||
valid, reason = validate_peer_payload(JD_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "ranking entry is not an object" in reason
|
||||
|
||||
|
||||
def test_quality_gate_allows_verified_dy_unranked_results() -> None:
|
||||
payload = _dy_payload()
|
||||
for category in DY_CATEGORIES:
|
||||
for brand_index, brand in enumerate(DY_BRANDS):
|
||||
payload["results"][category][brand] = _dy_entry(
|
||||
brand,
|
||||
category,
|
||||
brand_index,
|
||||
ranked=False,
|
||||
matched=False,
|
||||
)
|
||||
|
||||
assert validate_peer_payload(DY_PLATFORM, payload) == (True, "")
|
||||
|
||||
|
||||
def test_quality_gate_rejects_identical_dy_data_in_two_categories() -> None:
|
||||
payload = _dy_payload()
|
||||
first_category, second_category = DY_CATEGORIES[:2]
|
||||
for brand in DY_BRANDS:
|
||||
repeated = deepcopy(payload["results"][first_category][brand])
|
||||
repeated["category"] = second_category
|
||||
payload["results"][second_category][brand] = repeated
|
||||
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
|
||||
assert valid is False
|
||||
assert "identical payload copied across categories" in reason
|
||||
|
||||
|
||||
def test_quality_gate_requires_exact_dy_sets_and_entry_structure() -> None:
|
||||
payload = _dy_payload()
|
||||
payload["results"][DY_CATEGORIES[0]].pop(DY_BRANDS[-1])
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
assert valid is False
|
||||
assert "brand set mismatch" in reason
|
||||
|
||||
payload = _dy_payload()
|
||||
payload["results"][DY_CATEGORIES[0]][DY_BRANDS[0]].pop("stores")
|
||||
valid, reason = validate_peer_payload(DY_PLATFORM, payload)
|
||||
assert valid is False
|
||||
assert "ranking fields missing stores" in reason
|
||||
|
||||
|
||||
def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["run_peer_store", "--platform", "dy", "--skip-feishu"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"setup_runner_logging",
|
||||
lambda _name: "test.log",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"skip_feishu_for_acceptance",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_run_with_fresh_artifact",
|
||||
lambda platform, _collector: (
|
||||
platform,
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
"expired",
|
||||
"",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_persist_and_write",
|
||||
lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"),
|
||||
)
|
||||
|
||||
assert run_peer_store.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_persist_zero_is_reported_as_failure_and_feishu_can_be_skipped(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("Feishu must remain skipped"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
skip_feishu=True,
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
def test_persist_zero_stops_peer_feishu_write(monkeypatch) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("Feishu must not run after DB failure"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
def test_dy_peer_rejects_shop_json_from_another_week_before_writes(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
peer_json = tmp_path / "peer.json"
|
||||
peer_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/27 - 2026/08/02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_json = tmp_path / "shop.json"
|
||||
shop_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/20 - 2026/07/26"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_latest_shop_json_for_dy",
|
||||
lambda: str(shop_json),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"persist_dy_peer_store",
|
||||
lambda _path: pytest.fail("mismatched snapshots must fail before PG"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"write_peer_store",
|
||||
lambda *_args, **_kwargs: pytest.fail("mismatched snapshots must not write Feishu"),
|
||||
)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
DY_PLATFORM,
|
||||
True,
|
||||
str(peer_json),
|
||||
)
|
||||
|
||||
assert len(errors) == 1
|
||||
assert "周区间不一致" in errors[0]
|
||||
|
||||
|
||||
def test_dy_peer_passes_matching_shop_json_to_writer(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
peer_json = tmp_path / "peer.json"
|
||||
peer_json.write_text(
|
||||
json.dumps({"date_range": "2026/07/27 - 2026/08/02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shop_json = tmp_path / "shop.json"
|
||||
shop_json.write_text(
|
||||
json.dumps({"date_range": "2026-07-27 ~ 2026-08-02"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
observed = {}
|
||||
monkeypatch.setattr(
|
||||
run_peer_store,
|
||||
"_latest_shop_json_for_dy",
|
||||
lambda: str(shop_json),
|
||||
)
|
||||
monkeypatch.setattr(run_peer_store, "persist_dy_peer_store", lambda _path: 3)
|
||||
|
||||
def fake_write(platform, actual_peer_json, actual_shop_json):
|
||||
observed.update(
|
||||
platform=platform,
|
||||
peer_json=actual_peer_json,
|
||||
shop_json=actual_shop_json,
|
||||
)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(run_peer_store, "write_peer_store", fake_write)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
DY_PLATFORM,
|
||||
True,
|
||||
str(peer_json),
|
||||
)
|
||||
|
||||
assert errors == []
|
||||
assert observed == {
|
||||
"platform": DY_PLATFORM,
|
||||
"peer_json": str(peer_json),
|
||||
"shop_json": str(shop_json),
|
||||
}
|
||||
|
||||
|
||||
def test_false_feishu_writer_result_is_reported(monkeypatch) -> None:
|
||||
monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 3)
|
||||
monkeypatch.setattr(run_peer_store, "write_peer_store", lambda *_args: False)
|
||||
|
||||
errors = run_peer_store._persist_and_write(
|
||||
JD_PLATFORM,
|
||||
True,
|
||||
"D:/fresh-peer.json",
|
||||
)
|
||||
|
||||
assert errors == [f"{JD_PLATFORM} 竞店飞书写入失败: writer 返回 False"]
|
||||
|
||||
|
||||
def test_competitor_platforms_are_independent_graph_steps() -> None:
|
||||
module = ShopIntelligenceModule.from_catalog(
|
||||
WorkflowCatalog.load(PROJECT_ROOT / "config")
|
||||
)
|
||||
competitor = next(
|
||||
definition
|
||||
for definition in module.workflow_definitions()
|
||||
if definition.workflow_id == "shop.competitor.weekly"
|
||||
)
|
||||
|
||||
assert tuple(step.step_id for step in competitor.steps) == ("jd", "dy")
|
||||
assert all(not step.depends_on for step in competitor.steps)
|
||||
assert tuple(step.action.command_args for step in competitor.steps) == (
|
||||
("--platform", "jd"),
|
||||
("--platform", "dy"),
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
import unittest
|
||||
from datetime import date
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence.runners.run_jd_self_operated_daily import (
|
||||
build_collector_args,
|
||||
previous_day,
|
||||
)
|
||||
|
||||
|
||||
class TargetDateTests(unittest.TestCase):
|
||||
def test_previous_day_handles_month_boundary(self):
|
||||
self.assertEqual(previous_day(date(2026, 8, 1)), date(2026, 7, 31))
|
||||
|
||||
def test_previous_day_handles_year_boundary(self):
|
||||
self.assertEqual(previous_day(date(2027, 1, 1)), date(2026, 12, 31))
|
||||
|
||||
|
||||
class CollectorArgsTests(unittest.TestCase):
|
||||
def test_builds_single_day_headless_args(self):
|
||||
self.assertEqual(
|
||||
build_collector_args(date(2026, 7, 30), headless=True),
|
||||
[
|
||||
"--start-date",
|
||||
"2026-07-30",
|
||||
"--end-date",
|
||||
"2026-07-30",
|
||||
"--headless",
|
||||
],
|
||||
)
|
||||
|
||||
def test_headful_args_omit_headless_flag(self):
|
||||
self.assertNotIn(
|
||||
"--headless",
|
||||
build_collector_args(date(2026, 7, 30), headless=False),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli
|
||||
|
||||
|
||||
def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path:
|
||||
evidence = tmp_path / "shop-evidence.jsonl"
|
||||
monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1")
|
||||
monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence))
|
||||
monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False)
|
||||
return evidence
|
||||
|
||||
|
||||
def test_shop_shared_lark_cli_physically_skips_bitable_write(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
evidence = _enable_acceptance(monkeypatch, tmp_path)
|
||||
|
||||
def fail_if_spawned(*args, **kwargs):
|
||||
raise AssertionError("lark-cli subprocess must not start")
|
||||
|
||||
monkeypatch.setattr(lark_cli.subprocess, "run", fail_if_spawned)
|
||||
result = lark_cli.run_lark_cli(
|
||||
"node",
|
||||
"run.js",
|
||||
["base", "+record-batch-update", "--table-id", "tbl_test"],
|
||||
)
|
||||
|
||||
assert result["acceptance_skipped"] is True
|
||||
payload = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
assert payload["event"] == "feishu_write_skipped"
|
||||
assert payload["operation"].endswith("record-batch-update")
|
||||
|
||||
|
||||
def test_shop_shared_lark_cli_keeps_production_write_behavior(monkeypatch) -> None:
|
||||
monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False)
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
||||
|
||||
monkeypatch.setattr(lark_cli.subprocess, "run", fake_run)
|
||||
result = lark_cli.run_lark_cli(
|
||||
"node",
|
||||
"run.js",
|
||||
["base", "+record-batch-create", "--table-id", "tbl_test"],
|
||||
)
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert len(calls) == 1
|
||||
@@ -0,0 +1,916 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence import lark_cli as shop_lark_cli
|
||||
from gyxx_flow.modules.shop_intelligence.collectors import (
|
||||
jd_data_collector,
|
||||
jd_peer_store_data_collector,
|
||||
jd_self_operated_brand_daily,
|
||||
taobao_sycm,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.db import db as shop_db
|
||||
from gyxx_flow.modules.shop_intelligence.lark_cli import (
|
||||
format_week,
|
||||
normalize_week_label,
|
||||
week_search_prefix,
|
||||
)
|
||||
from gyxx_flow.modules.shop_intelligence.quality import validate_shop_payload
|
||||
from gyxx_flow.modules.shop_intelligence.writers import shop_base_writer
|
||||
|
||||
run_shop = import_module("gyxx_flow.modules.shop_intelligence.runners.run_shop")
|
||||
|
||||
|
||||
def test_jd_playwright_wait_for_function_passes_argument_by_keyword() -> None:
|
||||
tree = ast.parse(
|
||||
Path(jd_data_collector.__file__).read_text(encoding="utf-8-sig")
|
||||
)
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "wait_for_function"
|
||||
]
|
||||
|
||||
assert calls
|
||||
assert all(len(call.args) == 1 for call in calls)
|
||||
|
||||
|
||||
def test_jd_cross_month_week_uses_visible_iso_week_row() -> None:
|
||||
source = inspect.getsource(jd_data_collector.select_last_week_any_day)
|
||||
|
||||
assert "target_year = last_sunday.year" in source
|
||||
assert "jmtd-date-picker-header-btn-prev-month" in source
|
||||
assert "jmtd-date-picker-week-number-cell" in source
|
||||
assert "clicked_iso_week_row_" in source
|
||||
assert "jmtd-date-picker-combo-tag-echo" in source
|
||||
|
||||
|
||||
def test_jd_navigation_retries_transient_failures() -> None:
|
||||
class Page:
|
||||
def __init__(self) -> None:
|
||||
self.attempts = 0
|
||||
self.waits: list[int] = []
|
||||
|
||||
def goto(self, *_args, **_kwargs) -> None:
|
||||
self.attempts += 1
|
||||
if self.attempts < 3:
|
||||
raise RuntimeError("net::ERR_PROXY_CONNECTION_FAILED")
|
||||
|
||||
def wait_for_timeout(self, timeout_ms: int) -> None:
|
||||
self.waits.append(timeout_ms)
|
||||
|
||||
page = Page()
|
||||
jd_data_collector.goto_with_retry(page, "https://example.invalid")
|
||||
|
||||
assert page.attempts == 3
|
||||
assert page.waits == [3000, 6000]
|
||||
|
||||
|
||||
def test_jd_waits_for_summary_card_hydration_after_week_switch() -> None:
|
||||
source = inspect.getsource(jd_data_collector.step_shop_star_and_trade)
|
||||
|
||||
assert "page.wait_for_timeout(8000)" in source
|
||||
assert ".sz-summary-item" in source
|
||||
assert ".sz-summary-item-index [data-value]" in source
|
||||
assert "real-time-module-right-shop-level-content-my-level-data-value" in source
|
||||
|
||||
|
||||
def test_jd_star_accepts_current_bare_score_value() -> None:
|
||||
assert jd_data_collector._extract_jd_star("5.0") == "5.0星"
|
||||
assert jd_data_collector._extract_jd_star("5.0详情") == "5.0星"
|
||||
assert jd_data_collector._extract_jd_star("未获取到") == "未获取到"
|
||||
|
||||
|
||||
def test_week_label_is_canonical_and_historical_padding_is_equivalent() -> None:
|
||||
week_label, month_label, *_ = format_week("2026-07-27", "2026-08-02")
|
||||
|
||||
assert week_label == "7月第5周（7.27-8.2）"
|
||||
assert month_label == "2026.07月"
|
||||
assert normalize_week_label("7月第5周（7.27-8.02）") == normalize_week_label(
|
||||
week_label
|
||||
)
|
||||
assert normalize_week_label(" 7月第5周 ( 7.27 - 8.02 ) ") == normalize_week_label(
|
||||
week_label
|
||||
)
|
||||
assert week_search_prefix("7月第5周（7.27-8.02）") == "7月第5周"
|
||||
|
||||
|
||||
def test_find_existing_record_matches_padding_but_not_another_year(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
observed = {}
|
||||
|
||||
def fake_run(_node, _script, args, **_kwargs):
|
||||
observed["payload"] = json.loads(args[-1])
|
||||
return {
|
||||
"data": {
|
||||
"fields": ["time", "platform", "month"],
|
||||
"data": [
|
||||
["7月第5周（7.27-8.02）", ["DY"], ["2025.07月"]],
|
||||
["7月第5周（7.27-8.02）", ["DY"], ["2026.07月"]],
|
||||
],
|
||||
"record_id_list": ["old-year", "same-week"],
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(shop_lark_cli, "run_lark_cli", fake_run)
|
||||
fields = {
|
||||
"时间": "time",
|
||||
"平台": "platform",
|
||||
"月份": "month",
|
||||
}
|
||||
|
||||
record_id = shop_lark_cli.find_existing_record(
|
||||
"node",
|
||||
"script",
|
||||
"base",
|
||||
"table",
|
||||
fields,
|
||||
"7月第5周（7.27-8.2）",
|
||||
"DY",
|
||||
month_str="2026.07月",
|
||||
)
|
||||
|
||||
assert record_id == "same-week"
|
||||
assert observed["payload"]["keyword"] == "7月第5周"
|
||||
|
||||
|
||||
def test_dy_shop_feishu_sales_uses_confirmed_net_sales_amount() -> None:
|
||||
extracted = shop_base_writer._extract_dy_metrics(
|
||||
{
|
||||
"date_range": "2026/07/27 - 2026/08/02",
|
||||
"metrics": {
|
||||
"成交金额": "¥413,988.46",
|
||||
"成交退款金额": "¥119,095.48",
|
||||
},
|
||||
"derived_metrics": {"raw": {"销售额": 294892.98}},
|
||||
"experience_score": {"score": "95"},
|
||||
}
|
||||
)
|
||||
|
||||
assert extracted["sales_amount"] == 294892.98
|
||||
assert extracted["star_rating"] == 95.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_value",
|
||||
["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, "5星"],
|
||||
)
|
||||
def test_jd_shop_quality_rejects_invalid_or_negative_sales(invalid_value) -> None:
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": invalid_value}},
|
||||
"star_rating_raw": "5.0星",
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "成交金额" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_value",
|
||||
["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, 5.01, "¥4.9"],
|
||||
)
|
||||
def test_jd_shop_quality_rejects_invalid_or_out_of_range_score(
|
||||
invalid_value,
|
||||
) -> None:
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥311,671.16"}},
|
||||
"star_rating_raw": invalid_value,
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert "店铺评分" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "invalid_value", "reason_text"),
|
||||
[
|
||||
("gross", "加载失败", "成交金额"),
|
||||
("gross", "¥", "成交金额"),
|
||||
("gross", float("nan"), "成交金额"),
|
||||
("gross", -1, "成交金额"),
|
||||
("net", "abc", "净销售额"),
|
||||
("net", float("nan"), "净销售额"),
|
||||
("net", -1, "净销售额"),
|
||||
("score", "未采集到", "店铺评分"),
|
||||
("score", float("nan"), "店铺评分"),
|
||||
("score", -1, "店铺评分"),
|
||||
("score", 100.01, "店铺评分"),
|
||||
],
|
||||
)
|
||||
def test_dy_shop_quality_rejects_invalid_metrics(
|
||||
field,
|
||||
invalid_value,
|
||||
reason_text,
|
||||
) -> None:
|
||||
gross = "¥413,988.46"
|
||||
net = 294892.98
|
||||
score = "95分"
|
||||
if field == "gross":
|
||||
gross = invalid_value
|
||||
elif field == "net":
|
||||
net = invalid_value
|
||||
else:
|
||||
score = invalid_value
|
||||
|
||||
valid, reason = validate_shop_payload(
|
||||
run_shop.DY_PLATFORM,
|
||||
{
|
||||
"metrics": {"成交金额": gross},
|
||||
"derived_metrics": {"raw": {"销售额": net}},
|
||||
"experience_score": {"score": score},
|
||||
},
|
||||
)
|
||||
|
||||
assert valid is False
|
||||
assert reason_text in reason
|
||||
|
||||
|
||||
def test_shop_quality_accepts_strict_real_metric_formats() -> None:
|
||||
assert validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥311,671.16"}},
|
||||
"star_rating_raw": "5.0星",
|
||||
},
|
||||
) == (True, "")
|
||||
assert validate_shop_payload(
|
||||
run_shop.JD_PLATFORM,
|
||||
{
|
||||
"transaction": {"成交金额": {"value": "￥264,650.81"}},
|
||||
"star_rating_raw": (
|
||||
"店铺星级星级运营店铺星级我的店铺星级4.7详情较昨日持平"
|
||||
),
|
||||
},
|
||||
) == (True, "")
|
||||
assert validate_shop_payload(
|
||||
run_shop.DY_PLATFORM,
|
||||
{
|
||||
"metrics": {"成交金额": "¥413,988.46"},
|
||||
"derived_metrics": {"raw": {"销售额": 294892.98}},
|
||||
"experience_score": {"score": "95分"},
|
||||
},
|
||||
) == (True, "")
|
||||
|
||||
|
||||
def test_jd_shop_feishu_rejects_missing_metrics_instead_of_writing_zero() -> None:
|
||||
with pytest.raises(ValueError, match="缺少有效成交金额、店铺评分"):
|
||||
shop_base_writer._extract_jd_metrics(
|
||||
{
|
||||
"week_start": "2026-07-27",
|
||||
"week_end": "2026-08-02",
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_jd_current_dom_metric_text_and_star_are_normalized() -> None:
|
||||
assert jd_data_collector._normalize_jd_metric_text(
|
||||
"￥\n3\n1\n1\n,\n6\n7\n1\n.\n1\n6"
|
||||
) == "￥311,671.16"
|
||||
assert jd_data_collector._extract_jd_star(
|
||||
"店铺星级\n星级运营\n我的店铺星级\n5.0详情\n较昨日持平"
|
||||
) == "5.0星"
|
||||
|
||||
|
||||
def test_jd_week_event_suffix_uses_iso_week() -> None:
|
||||
assert jd_data_collector._week_event_suffix(
|
||||
jd_data_collector.datetime(2026, 7, 27)
|
||||
) == "2026-31"
|
||||
|
||||
|
||||
def test_runner_rejects_success_without_fresh_artifact(monkeypatch) -> None:
|
||||
marker = ("D:/old.json", 1, 10)
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: marker)
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
"JD",
|
||||
lambda: ("JD", 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "no fresh metrics file" in result[2]
|
||||
|
||||
|
||||
def test_runner_passes_only_the_new_artifact(monkeypatch) -> None:
|
||||
markers = iter((None, ("D:/new.json", 2, 20)))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_validate_metrics_artifact",
|
||||
lambda _platform, _path: (True, ""),
|
||||
)
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
"JD",
|
||||
lambda: ("JD", 0, "ok"),
|
||||
)
|
||||
|
||||
assert result == ("JD", 0, "ok", "D:/new.json")
|
||||
|
||||
|
||||
def test_runner_rejects_fresh_but_empty_metrics_artifact(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "dy.json"
|
||||
artifact.write_text('{"metrics": {}}', encoding="utf-8")
|
||||
markers = iter((None, (str(artifact), 2, artifact.stat().st_size)))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
run_shop.DY_PLATFORM,
|
||||
lambda: (run_shop.DY_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "shop metrics are empty" in result[2]
|
||||
|
||||
|
||||
def test_runner_rejects_jd_placeholder_metrics_before_external_writes(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "jd.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size)
|
||||
markers = iter((None, marker))
|
||||
monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers))
|
||||
|
||||
result = run_shop._run_with_fresh_artifact(
|
||||
run_shop.JD_PLATFORM,
|
||||
lambda: (run_shop.JD_PLATFORM, 0, "collector output"),
|
||||
)
|
||||
|
||||
assert result[1] == 1
|
||||
assert result[3] == ""
|
||||
assert "missing valid 成交金额/店铺评分" in result[2]
|
||||
|
||||
|
||||
def test_direct_jd_persistence_rejects_placeholder_metrics_before_connecting(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact = tmp_path / "jd-invalid.json"
|
||||
artifact.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"star_rating_raw": "未获取到",
|
||||
"transaction": {"成交金额": {"value": "N/A"}},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(shop_db, "DB_ENABLED", True)
|
||||
monkeypatch.setattr(
|
||||
shop_db,
|
||||
"get_conn",
|
||||
lambda: pytest.fail("invalid payload must not connect to PostgreSQL"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="JD shop data quality check failed"):
|
||||
shop_db.persist_jd_shop(str(artifact))
|
||||
|
||||
|
||||
def test_jd_pg_transaction_value_does_not_depend_on_item_key_order() -> None:
|
||||
metrics = {
|
||||
"shop": "test-shop",
|
||||
"week_start": "2026-07-27",
|
||||
"week_end": "2026-08-02",
|
||||
"star_rating_raw": "5.0",
|
||||
"transaction": {
|
||||
"成交金额": {
|
||||
"环比": "-19.8%",
|
||||
"同行同级均值": "¥999.00",
|
||||
"value": "¥311,671.16",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
row = shop_db._build_jd_shop_row(metrics)
|
||||
|
||||
assert row is not None
|
||||
assert row["deal_amount"] == 311671.16
|
||||
assert row["deal_amount_qoq"] == "-19.8%"
|
||||
assert row["deal_amount_peer_avg"] == "¥999.00"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name", "platform_code"),
|
||||
[
|
||||
(run_shop.JD_PLATFORM, "persist_jd_shop", "JD"),
|
||||
(run_shop.DY_PLATFORM, "persist_dy_shop", "DY"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("persisted_count", [0, -1])
|
||||
def test_shop_nonpositive_persistence_stops_feishu_write(
|
||||
monkeypatch,
|
||||
platform: str,
|
||||
persist_name: str,
|
||||
platform_code: str,
|
||||
persisted_count: int,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
|
||||
monkeypatch.setattr(run_shop, persist_name, lambda _path: persisted_count)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"write_shop",
|
||||
lambda *_args: pytest.fail("Feishu must not run after DB failure"),
|
||||
)
|
||||
|
||||
errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json")
|
||||
|
||||
assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "persist_name", "platform_code"),
|
||||
[
|
||||
(run_shop.JD_PLATFORM, "persist_jd_shop", "JD"),
|
||||
(run_shop.DY_PLATFORM, "persist_dy_shop", "DY"),
|
||||
],
|
||||
)
|
||||
def test_shop_false_writer_result_is_reported(
|
||||
monkeypatch,
|
||||
platform: str,
|
||||
persist_name: str,
|
||||
platform_code: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "")
|
||||
monkeypatch.setattr(run_shop, persist_name, lambda _path: 1)
|
||||
monkeypatch.setattr(run_shop, "write_shop", lambda *_args: False)
|
||||
|
||||
errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json")
|
||||
|
||||
assert errors == [f"{platform_code} 飞书写入失败: writer 返回 False"]
|
||||
|
||||
|
||||
def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["run_shop", "--platform", "dy", "--skip-feishu"],
|
||||
)
|
||||
monkeypatch.setattr(run_shop, "setup_runner_logging", lambda _name: "test.log")
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"skip_feishu_for_acceptance",
|
||||
lambda *_args, **_kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_run_with_fresh_artifact",
|
||||
lambda platform, _collector: (
|
||||
platform,
|
||||
COOKIE_SKIP_EXIT_CODE,
|
||||
"expired",
|
||||
"",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
run_shop,
|
||||
"_persist_and_write",
|
||||
lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"),
|
||||
)
|
||||
|
||||
assert run_shop.main() == COOKIE_SKIP_EXIT_CODE
|
||||
|
||||
|
||||
def test_shop_analysis_uses_local_analyzer_hermes(monkeypatch) -> None:
|
||||
observed = {}
|
||||
|
||||
class Response:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"choices": [{"message": {"content": "analysis"}}]}
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, url, **kwargs):
|
||||
observed["session"] = self
|
||||
observed["url"] = url
|
||||
observed.update(kwargs)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setenv(
|
||||
"HERMES_ANALYZER_URL",
|
||||
"http://127.0.0.1:8642/v1/chat/completions",
|
||||
)
|
||||
monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "test-token")
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
|
||||
assert shop_base_writer._call_hermes("test prompt") == "analysis"
|
||||
assert observed["url"].startswith("http://127.0.0.1:8642/")
|
||||
assert observed["session"].trust_env is False
|
||||
assert observed["headers"]["Authorization"] == "Bearer test-token"
|
||||
|
||||
|
||||
def test_shop_analysis_falls_back_to_local_cli_when_gateway_requires_auth(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class Response:
|
||||
status_code = 401
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
error = shop_base_writer.requests.HTTPError("unauthorized")
|
||||
error.response = self
|
||||
raise error
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, *_args, **_kwargs):
|
||||
return Response()
|
||||
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
monkeypatch.setattr(
|
||||
shop_base_writer,
|
||||
"_call_hermes_cli",
|
||||
lambda prompt, *, timeout: f"cli:{prompt}:{timeout}",
|
||||
)
|
||||
|
||||
assert shop_base_writer._call_hermes("prompt", timeout=12) == "cli:prompt:12"
|
||||
|
||||
|
||||
def test_shop_analysis_reads_analyzer_profile_gateway_key(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
profile = tmp_path / "profiles" / "data-analyzer"
|
||||
profile.mkdir(parents=True)
|
||||
(profile / ".env").write_text(
|
||||
"API_SERVER_KEY=profile-key\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
observed = {}
|
||||
|
||||
class Response:
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"choices": [{"message": {"content": "analysis"}}]}
|
||||
|
||||
class Session:
|
||||
trust_env = True
|
||||
|
||||
def post(self, _url, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return Response()
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
monkeypatch.delenv("HERMES_ANALYZER_TOKEN", raising=False)
|
||||
monkeypatch.delenv("GYXX_HERMES_API_KEY", raising=False)
|
||||
monkeypatch.delenv("HERMES_API_KEY", raising=False)
|
||||
monkeypatch.setattr(shop_base_writer.requests, "Session", Session)
|
||||
|
||||
assert shop_base_writer._call_hermes("prompt") == "analysis"
|
||||
assert observed["headers"]["Authorization"] == "Bearer profile-key"
|
||||
|
||||
|
||||
def test_jd_shop_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
page = object()
|
||||
observed = {}
|
||||
|
||||
class Context:
|
||||
pages = [page]
|
||||
closed = False
|
||||
|
||||
def cookies(self):
|
||||
return [{"name": "session", "value": "saved"}]
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
Path(path).write_text('{"cookies": []}', encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_enter_shangzhi",
|
||||
lambda *_args: page,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"step_shop_star_and_trade",
|
||||
lambda *_args: ({}, {}),
|
||||
)
|
||||
monkeypatch.setattr(jd_data_collector, "step_deal_summary", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_merchant_ranking", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_jzt_non_full_site", lambda *_args: {})
|
||||
monkeypatch.setattr(jd_data_collector, "step_jzt_full_site", lambda *_args: {})
|
||||
monkeypatch.setattr(
|
||||
jd_data_collector,
|
||||
"generate_report",
|
||||
lambda *_args: str(tmp_path / "report.json"),
|
||||
)
|
||||
|
||||
assert jd_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22105" in observed["args"]
|
||||
assert "--no-proxy-server" in observed["args"]
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
|
||||
|
||||
def test_jd_peer_uses_bound_profile_port_and_persists_state(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
page = object()
|
||||
observed = {}
|
||||
|
||||
class Context:
|
||||
pages = [page]
|
||||
closed = False
|
||||
|
||||
def cookies(self):
|
||||
return [{"name": "peer-session", "value": "saved"}]
|
||||
|
||||
def storage_state(self, *, path: str) -> None:
|
||||
Path(path).write_text('{"cookies": []}', encoding="utf-8")
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
context = Context()
|
||||
|
||||
class Chromium:
|
||||
def launch_persistent_context(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
return context
|
||||
|
||||
class Playwright:
|
||||
chromium = Chromium()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
cookie_file = tmp_path / "state" / "cookies.json"
|
||||
storage_file = tmp_path / "state" / "storage.json"
|
||||
profile = tmp_path / "profile"
|
||||
monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile))
|
||||
monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106")
|
||||
monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file))
|
||||
monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file))
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright)
|
||||
monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_enter_shangzhi",
|
||||
lambda *_args: page,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_return_old_version",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_goto_vender_ranks",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_collect_competitor_data",
|
||||
lambda *_args: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"save_results",
|
||||
lambda *_args: str(tmp_path / "peer.json"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"step_goto_product_ranks",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"collect_new_products",
|
||||
lambda *_args: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"save_new_product_results",
|
||||
lambda *_args: None,
|
||||
)
|
||||
|
||||
assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0
|
||||
assert observed["user_data_dir"] == str(profile)
|
||||
assert "--remote-debugging-port=22106" in observed["args"]
|
||||
assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session"
|
||||
assert storage_file.is_file()
|
||||
assert context.closed is True
|
||||
|
||||
|
||||
def test_jd_peer_selects_week_once_before_category_loop(monkeypatch) -> None:
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"SHOPS",
|
||||
["peer-shop"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"CATEGORIES",
|
||||
[("男士双肩包", "男包")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"select_last_week_any_day",
|
||||
lambda _page: calls.append("initial-week"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"search_shop",
|
||||
lambda *_args: calls.append("search"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"switch_category",
|
||||
lambda *_args: calls.append("category"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"reselect_last_week",
|
||||
lambda _page: calls.append("reselect-week"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
jd_peer_store_data_collector,
|
||||
"extract_ranking_table",
|
||||
lambda _page: [],
|
||||
)
|
||||
|
||||
page = SimpleNamespace(wait_for_timeout=lambda *_args: None)
|
||||
jd_peer_store_data_collector.step_collect_competitor_data(page)
|
||||
|
||||
assert calls == ["initial-week", "search", "category"]
|
||||
|
||||
|
||||
def test_jd_peer_product_rank_navigation_does_not_wait_for_network_idle() -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
class Page:
|
||||
def goto(self, url: str, **kwargs) -> None:
|
||||
observed["url"] = url
|
||||
observed.update(kwargs)
|
||||
|
||||
def wait_for_timeout(self, timeout_ms: int) -> None:
|
||||
observed["settle_ms"] = timeout_ms
|
||||
|
||||
jd_peer_store_data_collector.step_goto_product_ranks(Page())
|
||||
|
||||
assert observed == {
|
||||
"url": "https://sz.jd.com/szweb/sz/view/industryMarket/productRanks.html",
|
||||
"wait_until": "domcontentloaded",
|
||||
"timeout": 60000,
|
||||
"settle_ms": 8000,
|
||||
}
|
||||
|
||||
|
||||
def test_jd_self_operated_reports_date_not_yet_available() -> None:
|
||||
page = MagicMock()
|
||||
mode = MagicMock()
|
||||
mode.count.return_value = 0
|
||||
date_picker = MagicMock()
|
||||
dropdown = MagicMock()
|
||||
enabled_target = MagicMock()
|
||||
disabled_target = MagicMock()
|
||||
selectable = MagicMock()
|
||||
enabled_target.count.return_value = 0
|
||||
disabled_target.count.return_value = 1
|
||||
selectable.evaluate_all.return_value = [
|
||||
"2026-08-01",
|
||||
"2026-08-02",
|
||||
None,
|
||||
]
|
||||
|
||||
def locate(selector: str):
|
||||
if selector == ".jd-select-selection-item":
|
||||
return SimpleNamespace(first=mode)
|
||||
if selector == ".jd-picker-range":
|
||||
return SimpleNamespace(first=date_picker)
|
||||
if selector == ".jd-picker-dropdown:visible":
|
||||
return SimpleNamespace(first=dropdown)
|
||||
raise AssertionError(selector)
|
||||
|
||||
def locate_dropdown(selector: str):
|
||||
if selector == (
|
||||
"td[title].jd-picker-cell-in-view:not(.jd-picker-cell-disabled)"
|
||||
):
|
||||
return selectable
|
||||
if selector.endswith(":not(.jd-picker-cell-disabled)"):
|
||||
return enabled_target
|
||||
if selector.endswith(".jd-picker-cell-disabled"):
|
||||
return disabled_target
|
||||
raise AssertionError(selector)
|
||||
|
||||
page.locator.side_effect = locate
|
||||
dropdown.locator.side_effect = locate_dropdown
|
||||
|
||||
with pytest.raises(RuntimeError, match="当前最晚可选日期为 2026-08-02"):
|
||||
jd_self_operated_brand_daily.set_single_day(
|
||||
page,
|
||||
date(2026, 8, 3),
|
||||
)
|
||||
|
||||
|
||||
def test_tmall_returns_failure_when_browser_cannot_start(monkeypatch) -> None:
|
||||
monkeypatch.setattr(taobao_sycm, "sync_cookies_from_daily_profile", lambda *_args: 0)
|
||||
monkeypatch.setattr(taobao_sycm, "connect_browser", lambda: (None, None))
|
||||
|
||||
assert taobao_sycm.main() == 1
|
||||
|
||||
|
||||
def test_tmall_profile_seed_preserves_cookie_encryption_state(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
target = tmp_path / "target"
|
||||
(source / "Default" / "Network").mkdir(parents=True)
|
||||
(source / "Local State").write_text("encryption-state", encoding="utf-8")
|
||||
(source / "Default" / "Network" / "Cookies").write_bytes(b"cookie-db")
|
||||
|
||||
copied = taobao_sycm.sync_cookies_from_daily_profile(source, target)
|
||||
|
||||
assert copied >= 2
|
||||
assert (target / "Local State").read_text(encoding="utf-8") == "encryption-state"
|
||||
assert (target / "Default" / "Network" / "Cookies").read_bytes() == b"cookie-db"
|
||||
|
||||
|
||||
def test_tmall_close_always_releases_bound_cdp(monkeypatch) -> None:
|
||||
closed = []
|
||||
killed = []
|
||||
monkeypatch.setattr(taobao_sycm, "DEBUG_PORT", 22107)
|
||||
monkeypatch.setattr(taobao_sycm, "_kill_port_listeners", killed.append)
|
||||
|
||||
taobao_sycm._close_session(SimpleNamespace(close=lambda: closed.append(True)))
|
||||
|
||||
assert closed == [True]
|
||||
assert killed == [22107]
|
||||
@@ -0,0 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE
|
||||
from gyxx_flow.modules.shop_intelligence.runners import utils
|
||||
|
||||
|
||||
def test_runner_rebinds_browser_environment_after_caller_overrides(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
observed["target"] = Path(target)
|
||||
observed["base"] = dict(base_environment)
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py",
|
||||
"GYXX_BROWSER_CDP_PORT": "child-port",
|
||||
"GYXX_BROWSER_PROFILE_DIR": "child-profile",
|
||||
"GYXX_BROWSER_COOKIE_FILE": "child-cookie",
|
||||
"GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage",
|
||||
}
|
||||
|
||||
def fake_run(command, *, environment, timeout):
|
||||
observed["command"] = command
|
||||
observed["environment"] = environment
|
||||
observed["timeout"] = timeout
|
||||
return subprocess.CompletedProcess(command, 0, stdout=b"ok", stderr=b"")
|
||||
|
||||
monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind)
|
||||
monkeypatch.setattr(utils, "_run_managed_command", fake_run)
|
||||
target = str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py")
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"example",
|
||||
target,
|
||||
[],
|
||||
env={"KEEP": "yes", "GYXX_BROWSER_CDP_PORT": "caller-port"},
|
||||
)
|
||||
|
||||
environment = observed["environment"]
|
||||
assert code == 0
|
||||
assert output == "ok"
|
||||
assert observed["target"] == Path(target)
|
||||
assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "caller-port"
|
||||
assert environment["KEEP"] == "yes"
|
||||
assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port"
|
||||
assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile"
|
||||
assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie"
|
||||
assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage"
|
||||
assert observed["timeout"] == 3600
|
||||
|
||||
|
||||
def test_runner_skips_invalid_cookie_before_starting_child(monkeypatch) -> None:
|
||||
observed: dict[str, object] = {}
|
||||
|
||||
def fake_rebind(target, base_environment):
|
||||
return {
|
||||
**base_environment,
|
||||
"GYXX_WORKFLOW_ACCEPTANCE": "1",
|
||||
"GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py",
|
||||
}
|
||||
|
||||
class FakePolicy:
|
||||
enabled = True
|
||||
|
||||
def preflight_cookie(self, binding, *, environment):
|
||||
observed["binding"] = binding
|
||||
observed["environment"] = environment
|
||||
return SimpleNamespace(should_skip=True, reason="browser state is missing")
|
||||
|
||||
monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind)
|
||||
monkeypatch.setattr(utils, "binding_from_environment", lambda env: "binding")
|
||||
monkeypatch.setattr(utils, "current_acceptance_policy", lambda env: FakePolicy())
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"_run_managed_command",
|
||||
lambda *args, **kwargs: (_ for _ in ()).throw(
|
||||
AssertionError("child process must not start")
|
||||
),
|
||||
)
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"example",
|
||||
str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py"),
|
||||
[],
|
||||
)
|
||||
|
||||
assert code == COOKIE_SKIP_EXIT_CODE
|
||||
assert output == "browser state is missing"
|
||||
assert observed["binding"] == "binding"
|
||||
assert observed["environment"]["GYXX_WORKFLOW_ACCEPTANCE"] == "1"
|
||||
|
||||
|
||||
def test_runner_timeout_terminates_collector_grandchild_before_return(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
identity_file = tmp_path / "grandchild.json"
|
||||
marker_file = tmp_path / "grandchild-finished.txt"
|
||||
grandchild_code = (
|
||||
"import pathlib,time; "
|
||||
"time.sleep(2.8); "
|
||||
f"pathlib.Path({str(marker_file)!r}).write_text('alive', encoding='utf-8')"
|
||||
)
|
||||
collector = tmp_path / "collector.py"
|
||||
collector.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
"import json, pathlib, psutil, subprocess, sys, time",
|
||||
f"child = subprocess.Popen([sys.executable, '-c', {grandchild_code!r}], close_fds=False)",
|
||||
f"pathlib.Path({str(identity_file)!r}).write_text(json.dumps({{'pid': child.pid, 'created': psutil.Process(child.pid).create_time()}}), encoding='utf-8')",
|
||||
"time.sleep(10)",
|
||||
)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"environment_for_child_script",
|
||||
lambda _target, environment: environment,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"current_acceptance_policy",
|
||||
lambda _environment: SimpleNamespace(enabled=False),
|
||||
)
|
||||
|
||||
code, output = utils.run_collector(
|
||||
"timeout-probe",
|
||||
str(collector),
|
||||
[],
|
||||
env=dict(os.environ),
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
assert code == 1
|
||||
assert "timeout-probe" in output
|
||||
assert identity_file.is_file()
|
||||
identity = json.loads(identity_file.read_text(encoding="utf-8"))
|
||||
try:
|
||||
descendant = psutil.Process(identity["pid"])
|
||||
same_process_is_live = (
|
||||
descendant.create_time()
|
||||
== pytest.approx(identity["created"], abs=0.01)
|
||||
and descendant.is_running()
|
||||
and descendant.status() != psutil.STATUS_ZOMBIE
|
||||
)
|
||||
except psutil.Error:
|
||||
same_process_is_live = False
|
||||
assert same_process_is_live is False
|
||||
time.sleep(1.0)
|
||||
assert marker_file.exists() is False
|
||||
|
||||
|
||||
def test_runner_redacts_sensitive_command_arguments() -> None:
|
||||
command = [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--password",
|
||||
"placeholder",
|
||||
"--api-key=placeholder",
|
||||
"--output",
|
||||
"safe-path",
|
||||
]
|
||||
|
||||
assert utils._redact_command(command) == [
|
||||
"python",
|
||||
"collector.py",
|
||||
"--password",
|
||||
"[REDACTED]",
|
||||
"--api-key=[REDACTED]",
|
||||
"--output",
|
||||
"safe-path",
|
||||
]
|
||||
|
||||
|
||||
def test_acceptance_skips_entire_feishu_writer_path(monkeypatch) -> None:
|
||||
observed: list[str] = []
|
||||
|
||||
class FakePolicy:
|
||||
def skip_feishu_write(self, operation):
|
||||
observed.append(operation)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(utils, "current_acceptance_policy", lambda: FakePolicy())
|
||||
|
||||
assert utils.skip_feishu_for_acceptance("shop.writer", requested=False) is True
|
||||
assert observed == ["shop.writer"]
|
||||
|
||||
|
||||
def test_explicit_feishu_skip_does_not_need_policy(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
utils,
|
||||
"current_acceptance_policy",
|
||||
lambda: (_ for _ in ()).throw(AssertionError("policy must not be loaded")),
|
||||
)
|
||||
|
||||
assert utils.skip_feishu_for_acceptance("shop.writer", requested=True) is True
|
||||
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
TARGET_COLLECTOR_ROOT = (
|
||||
PROJECT_ROOT
|
||||
/ "src"
|
||||
/ "gyxx_flow"
|
||||
/ "modules"
|
||||
/ "shop_intelligence"
|
||||
/ "collectors"
|
||||
)
|
||||
SOURCE_COLLECTOR_ROOT = Path(
|
||||
os.getenv("GYXX_SHOP_SOURCE_ROOT", r"D:\shop-data-flow")
|
||||
) / "collectors"
|
||||
|
||||
# These functions implement page navigation, date/category selection, extraction,
|
||||
# and iteration order. Portability adaptations belong in imports, configuration,
|
||||
# persistence, browser-session setup, and main/runner boundaries instead.
|
||||
CORE_BUSINESS_FUNCTIONS = {
|
||||
"jd_data_collector.py": (
|
||||
"select_last_week_any_day",
|
||||
"select_last_week_in_old_datepicker",
|
||||
"step_enter_shangzhi",
|
||||
"step_jzt_full_site",
|
||||
"step_jzt_non_full_site",
|
||||
"step_merchant_ranking",
|
||||
"step_shop_star_and_trade",
|
||||
"switch_category",
|
||||
),
|
||||
"jd_peer_store_data_collector.py": (
|
||||
"clear_and_search_brand",
|
||||
"collect_new_products",
|
||||
"reselect_last_week",
|
||||
"search_product_by_name",
|
||||
"select_last_week_any_day",
|
||||
"select_product_category",
|
||||
"step_enter_shangzhi",
|
||||
),
|
||||
"dy_store_competitor_store_scraping.py": (
|
||||
"_check_fxg_login",
|
||||
"_collect_competitor_rankings",
|
||||
"_collect_experience_score",
|
||||
"_ensure_fxg_login",
|
||||
"_extract_experience_score",
|
||||
"_open_market_rank_shop_page",
|
||||
"_select_menu_and_date",
|
||||
),
|
||||
}
|
||||
|
||||
# SHA-256 of ast.dump(function, include_attributes=False) from the reviewed
|
||||
# D:\shop-data-flow source. A source change must be reviewed before updating this
|
||||
# snapshot; copying a new hash merely to make the test pass defeats the guard.
|
||||
SOURCE_AST_SHA256 = {
|
||||
("jd_data_collector.py", "select_last_week_any_day"): (
|
||||
"0c85d64e6d47b08a5971caec3afe391271fd6bc94d86bab65f6ebae9655b8a6f"
|
||||
),
|
||||
("jd_data_collector.py", "select_last_week_in_old_datepicker"): (
|
||||
"ff542b968c43b92304ccb15a2282acf3d64ab7258dee020bae51f1c7f5f5f973"
|
||||
),
|
||||
("jd_data_collector.py", "step_enter_shangzhi"): (
|
||||
"6ba9e2df11ecbf0aae88fb2ed3cab8abe5157121418a0426384f8c43368b9362"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_full_site"): (
|
||||
"0cf4276039e96650879ba6fff8822322245dc581ad3020785ed6cb8a84e11e8f"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_non_full_site"): (
|
||||
"2a911597995c4d9a2cc47b106d6a8c5a811932eea0d83e928b0dcafcdb6864a5"
|
||||
),
|
||||
("jd_data_collector.py", "step_merchant_ranking"): (
|
||||
"027626587724aa224fca6146dae61ea505374e78ca36603c96769b36725dfac7"
|
||||
),
|
||||
("jd_data_collector.py", "step_shop_star_and_trade"): (
|
||||
"2e54324b6091e961eb6fbc704aca2867a91ce829709c2824e8a6ee8fa2260790"
|
||||
),
|
||||
("jd_data_collector.py", "switch_category"): (
|
||||
"f6dfdc7025958cbb85476c31b4930f2b57a8f67316c3000a8e38f5d49acb6aef"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "clear_and_search_brand"): (
|
||||
"32327c849483a44aa8c2b8cab8210f8e80767d3e2b86fd6789d855669062dc00"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "collect_new_products"): (
|
||||
"c2cbf6528a756d4fb26f5b8794177cbcb37790584f85660eff0a851ecffc8f58"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "reselect_last_week"): (
|
||||
"5897bd8afb8bf0604daace2a13628ea002f753afd8de7956246a4ee18e89cbd9"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "search_product_by_name"): (
|
||||
"2fd758156e716d2c666b8c1c1d9edb0c72cad83c56c4c44b09900bd7eb0bd940"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "select_last_week_any_day"): (
|
||||
"0a9b5fad4ea111e7b2afe39241508b5d036b84006091188b95ac85b652e7e3f9"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "select_product_category"): (
|
||||
"ff947df3e6e455176d723c93fb2af06ab92a72ac9174f4a5ae2db306201d5182"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "step_enter_shangzhi"): (
|
||||
"d6bfb283c00c842a129d8f81f0da61cae752e421e2dafaffdb87fd20603f0f2d"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_collect_competitor_rankings"): (
|
||||
"de55dded8a9341b955e000e4fafb8b4914533b25e06dc2aca930535836cfc0b5"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_check_fxg_login"): (
|
||||
"1928b38f6c167971139b147565b51e727e65a74e49ba21fe40d47711a7641e6a"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_collect_experience_score"): (
|
||||
"4609852591005c5b3ba8f9704b4db49918173f846176d9e50da6d4c8ee07914a"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_ensure_fxg_login"): (
|
||||
"ae69c5c58563c6cdb97fd6af380a24e2d275ec6623038a58b5e15a7bda2777f4"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_extract_experience_score"): (
|
||||
"5a28dbc7fa627fea384dae3d5f9b85fb0542b585c7312918ef355810209089df"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_open_market_rank_shop_page"): (
|
||||
"b60acaeaf3b1073144f3f570eb539ad87945fd92f2577cf338b3b496ec542d35"
|
||||
),
|
||||
("dy_store_competitor_store_scraping.py", "_select_menu_and_date"): (
|
||||
"4ffb5e89e7a7af3d0af126adf420399a0017fba8e49b9185266baf7307e3e549"
|
||||
),
|
||||
}
|
||||
|
||||
# Deliberately reviewed target-only compatibility fixes. The source project stays
|
||||
# read-only; these hashes ensure the migrated collector cannot drift beyond the
|
||||
# current JD DOM adaptation without another explicit review.
|
||||
TARGET_AST_SHA256_OVERRIDES = {
|
||||
("jd_data_collector.py", "select_last_week_any_day"): (
|
||||
"8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847"
|
||||
),
|
||||
("jd_data_collector.py", "step_shop_star_and_trade"): (
|
||||
"77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161"
|
||||
),
|
||||
("jd_data_collector.py", "step_merchant_ranking"): (
|
||||
"187cd3cfffcb97a5bf9f1ed4dde0d117ee65f82c1cf431df4c7c9c9ee2a7b282"
|
||||
),
|
||||
("jd_data_collector.py", "step_jzt_non_full_site"): (
|
||||
"1bd53eba5d0123e63d2e628d9772c0efa57fc8d2f87d79a4f3c41a311132952d"
|
||||
),
|
||||
}
|
||||
|
||||
# These two source functions were reviewed after a production run proved that
|
||||
# a failed category click silently copied the previous category's table. They
|
||||
# intentionally diverge from D:\shop-data-flow and retain their own reviewed
|
||||
# target snapshots so the defect cannot be reintroduced by a future sync.
|
||||
REVIEWED_DEFECT_FIX_AST_SHA256 = {
|
||||
("jd_peer_store_data_collector.py", "step_collect_competitor_data"): (
|
||||
"bf6836516dc3e70d00c145acbdfa00b6ddeee480f9927d7236e82ba8b1430ff2"
|
||||
),
|
||||
("jd_peer_store_data_collector.py", "switch_category"): (
|
||||
"bb4dc034447666682e19dc8cce05903c92d53852669ac93cc6810b18ad310d13"
|
||||
),
|
||||
}
|
||||
|
||||
FUNCTION_CASES = tuple(
|
||||
(filename, function_name)
|
||||
for filename, function_names in CORE_BUSINESS_FUNCTIONS.items()
|
||||
for function_name in function_names
|
||||
)
|
||||
|
||||
|
||||
def _call_name(node: ast.AST) -> str | None:
|
||||
if not isinstance(node, ast.Call):
|
||||
return None
|
||||
if isinstance(node.func, ast.Name):
|
||||
return node.func.id
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
return node.func.attr
|
||||
return None
|
||||
|
||||
|
||||
def _is_session_expiry_guard(node: ast.stmt) -> bool:
|
||||
if not isinstance(node, ast.If) or node.orelse:
|
||||
return False
|
||||
detector_names = {"_is_login_page_url", "_has_interactive_login_form"}
|
||||
detectors = {
|
||||
name
|
||||
for child in ast.walk(node.test)
|
||||
if (name := _call_name(child)) is not None
|
||||
}
|
||||
raises_session_expired = any(
|
||||
isinstance(child, ast.Raise)
|
||||
and child.exc is not None
|
||||
and _call_name(child.exc) == "BrowserSessionExpiredError"
|
||||
for statement in node.body
|
||||
for child in ast.walk(statement)
|
||||
)
|
||||
return bool(detectors & detector_names) and raises_session_expired
|
||||
|
||||
|
||||
def _is_navigation_session_wrapper(node: ast.Try) -> bool:
|
||||
if len(node.body) != 1 or node.orelse or node.finalbody or len(node.handlers) != 1:
|
||||
return False
|
||||
navigation = node.body[0]
|
||||
if not (
|
||||
isinstance(navigation, ast.Expr)
|
||||
and isinstance(navigation.value, ast.Call)
|
||||
and _call_name(navigation.value) == "goto"
|
||||
):
|
||||
return False
|
||||
handler_body = node.handlers[0].body
|
||||
return (
|
||||
len(handler_body) == 2
|
||||
and _is_session_expiry_guard(handler_body[0])
|
||||
and isinstance(handler_body[1], ast.Raise)
|
||||
and handler_body[1].exc is None
|
||||
)
|
||||
|
||||
|
||||
class _ApprovedBoundaryNormalizer(ast.NodeTransformer):
|
||||
"""Remove only the approved cookie/session-expiry adapter statements."""
|
||||
|
||||
def visit_If(self, node: ast.If) -> ast.AST | None: # noqa: N802
|
||||
if _is_session_expiry_guard(node):
|
||||
return None
|
||||
return self.generic_visit(node)
|
||||
|
||||
def visit_Try(self, node: ast.Try) -> ast.AST | list[ast.AST]: # noqa: N802
|
||||
if _is_navigation_session_wrapper(node):
|
||||
return [self.visit(statement) for statement in node.body]
|
||||
return self.generic_visit(node)
|
||||
|
||||
|
||||
def _function_ast(path: Path, function_name: str) -> str:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
|
||||
matches = [
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
and node.name == function_name
|
||||
]
|
||||
assert len(matches) == 1, f"expected exactly one top-level {function_name} in {path}"
|
||||
normalized = _ApprovedBoundaryNormalizer().visit(matches[0])
|
||||
assert isinstance(normalized, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
return ast.dump(normalized, annotate_fields=True, include_attributes=False)
|
||||
|
||||
|
||||
def _ast_sha256(function_ast: str) -> str:
|
||||
return hashlib.sha256(function_ast.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def test_ast_snapshot_covers_exactly_the_reviewed_business_functions() -> None:
|
||||
assert set(SOURCE_AST_SHA256) == set(FUNCTION_CASES)
|
||||
assert set(SOURCE_AST_SHA256).isdisjoint(REVIEWED_DEFECT_FIX_AST_SHA256)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
REVIEWED_DEFECT_FIX_AST_SHA256,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_reviewed_local_defect_fixes_match_target_snapshot(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
|
||||
assert _ast_sha256(target_ast) == REVIEWED_DEFECT_FIX_AST_SHA256[
|
||||
(filename, function_name)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
FUNCTION_CASES,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_migrated_core_business_logic_matches_reviewed_ast_snapshot(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
|
||||
expected = TARGET_AST_SHA256_OVERRIDES.get(
|
||||
(filename, function_name),
|
||||
SOURCE_AST_SHA256[(filename, function_name)],
|
||||
)
|
||||
assert _ast_sha256(target_ast) == expected, (
|
||||
f"{filename}:{function_name} changed the reviewed source business logic; "
|
||||
"only explicitly reviewed target compatibility adapters may differ"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not SOURCE_COLLECTOR_ROOT.is_dir(),
|
||||
reason="local D:\\shop-data-flow source project is unavailable",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "function_name"),
|
||||
FUNCTION_CASES,
|
||||
ids=lambda value: value,
|
||||
)
|
||||
def test_reviewed_snapshot_and_migration_match_current_local_source(
|
||||
filename: str,
|
||||
function_name: str,
|
||||
) -> None:
|
||||
source_ast = _function_ast(SOURCE_COLLECTOR_ROOT / filename, function_name)
|
||||
target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name)
|
||||
expected = SOURCE_AST_SHA256[(filename, function_name)]
|
||||
|
||||
assert _ast_sha256(source_ast) == expected, (
|
||||
f"{filename}:{function_name} changed in D:\\shop-data-flow; review the "
|
||||
"source change before deliberately refreshing this snapshot"
|
||||
)
|
||||
override = TARGET_AST_SHA256_OVERRIDES.get((filename, function_name))
|
||||
if override:
|
||||
assert _ast_sha256(target_ast) == override
|
||||
else:
|
||||
assert target_ast == source_ast, (
|
||||
f"{filename}:{function_name} no longer matches the current source project"
|
||||
)
|
||||
Reference in New Issue
Block a user