400 lines
13 KiB
Python
400 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import date
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import collect_dy_persona_to_bitable as dy_worker
|
|
import collect_jd_persona_to_bitable as jd_worker
|
|
import collect_persona_to_bitable as tm_worker
|
|
import dy_audience_profile_collect as dy_collector
|
|
import pytest
|
|
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
|
|
|
from gyxx_flow.adapters import acceptance_policy
|
|
|
|
WORKERS = (tm_worker, dy_worker, jd_worker)
|
|
BUSINESS_DATE = date(2026, 8, 4)
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_missing_time_select_option_is_treated_as_no_existing_record(monkeypatch, worker) -> None:
|
|
monkeypatch.setattr(
|
|
worker.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: SimpleNamespace(
|
|
returncode=1,
|
|
stdout="",
|
|
stderr=json.dumps(
|
|
{
|
|
"ok": False,
|
|
"error": {"code": 800030005, "message": "not_found"},
|
|
}
|
|
),
|
|
),
|
|
)
|
|
|
|
assert worker.find_record_id_by_time("bas_test", "tbl_test", "8.13") is None
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_platform_collector_failure_forces_nonzero(worker) -> None:
|
|
assert worker.combined_result_exit_code(
|
|
[{"status": "ok"}],
|
|
collection_ok=False,
|
|
) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_formal_base_upsert_requires_response_record_id(monkeypatch, worker) -> None:
|
|
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
|
monkeypatch.setattr(
|
|
worker.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps({"ok": True, "data": {}}),
|
|
stderr="",
|
|
),
|
|
)
|
|
|
|
ok, message = worker.upsert_record(
|
|
"bas_test",
|
|
"tbl_test",
|
|
{"时间": "8.4", "男性比例": 0.5},
|
|
)
|
|
|
|
assert ok is False
|
|
assert "record_id" in message
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, worker) -> None:
|
|
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
|
monkeypatch.setattr(
|
|
worker.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps(
|
|
{
|
|
"ok": True,
|
|
"data": {"record": {"record_id_list": ["rec12345678"]}},
|
|
}
|
|
),
|
|
stderr="",
|
|
),
|
|
)
|
|
|
|
ok, message = worker.upsert_record(
|
|
"bas_test",
|
|
"tbl_test",
|
|
{"时间": "8.4", "男性比例": 0.5},
|
|
)
|
|
|
|
assert ok is True
|
|
assert message == "record_id=rec12345678"
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_formal_base_update_reuses_valid_requested_record_id(monkeypatch, worker) -> None:
|
|
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
|
monkeypatch.setattr(
|
|
worker.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps({"ok": True, "data": {"updated": True}}),
|
|
stderr="",
|
|
),
|
|
)
|
|
|
|
ok, message = worker.upsert_record(
|
|
"bas_test",
|
|
"tbl_test",
|
|
{"时间": "8.4", "男性比例": 0.5},
|
|
record_id="rec_existing123",
|
|
)
|
|
|
|
assert ok is True
|
|
assert message == "record_id=rec_existing123"
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
|
|
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
|
|
monkeypatch.setattr(
|
|
worker.subprocess,
|
|
"run",
|
|
lambda *_a, **_k: (_ for _ in ()).throw(
|
|
AssertionError("lark subprocess must not run after acceptance skip")
|
|
),
|
|
)
|
|
|
|
ok, message = worker.upsert_record(
|
|
"bas_test",
|
|
"tbl_test",
|
|
{"时间": "8.4", "男性比例": 0.5},
|
|
)
|
|
|
|
assert ok is False
|
|
assert "acceptance-skipped" in message
|
|
|
|
|
|
def test_tm_payload_must_match_requested_business_date(monkeypatch, tmp_path: Path) -> None:
|
|
style = "款A"
|
|
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
|
style_dir.mkdir(parents=True)
|
|
(style_dir / f"{style}_123_chart_values.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"style_name": style,
|
|
"item_id": "123",
|
|
"business_date": "2026-08-03",
|
|
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(tm_worker, "DMP_OUTPUT_ROOT", tmp_path)
|
|
|
|
assert tm_worker.load_latest_chart_values(style, BUSINESS_DATE) is None
|
|
|
|
|
|
@pytest.mark.parametrize("worker", (dy_worker, jd_worker))
|
|
def test_profile_payload_must_match_requested_business_date(
|
|
monkeypatch,
|
|
tmp_path: Path,
|
|
worker,
|
|
) -> None:
|
|
style = "款A"
|
|
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
|
style_dir.mkdir(parents=True)
|
|
(style_dir / f"{style}_123_profile.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"style_name": style,
|
|
"product_id": "123",
|
|
"business_date": "2026-08-03",
|
|
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
|
|
|
assert worker.load_latest_profile(style, BUSINESS_DATE) is None
|
|
|
|
|
|
@pytest.mark.parametrize("worker", (tm_worker, dy_worker))
|
|
def test_current_run_does_not_reuse_old_artifact(monkeypatch, tmp_path: Path, worker) -> None:
|
|
style = "款A"
|
|
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
|
style_dir.mkdir(parents=True)
|
|
if worker is tm_worker:
|
|
path = style_dir / f"{style}_123_chart_values.json"
|
|
payload = {
|
|
"style_name": style,
|
|
"item_id": "123",
|
|
"business_date": BUSINESS_DATE.isoformat(),
|
|
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
|
}
|
|
monkeypatch.setattr(worker, "DMP_OUTPUT_ROOT", tmp_path)
|
|
load = worker.load_latest_chart_values
|
|
else:
|
|
path = style_dir / f"{style}_123_profile.json"
|
|
payload = {
|
|
"style_name": style,
|
|
"product_id": "123",
|
|
"business_date": BUSINESS_DATE.isoformat(),
|
|
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
|
}
|
|
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
|
load = worker.load_latest_profile
|
|
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
|
|
|
assert load(
|
|
style,
|
|
BUSINESS_DATE,
|
|
artifact_not_before=path.stat().st_mtime + 10,
|
|
) is None
|
|
|
|
|
|
@pytest.mark.parametrize("worker", WORKERS)
|
|
def test_database_failure_stops_before_base_write(monkeypatch, worker) -> None:
|
|
if worker is tm_worker:
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"load_latest_chart_values",
|
|
lambda *_a, **_k: {
|
|
"style_name": "款A",
|
|
"item_id": "123",
|
|
"business_date": BUSINESS_DATE.isoformat(),
|
|
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"chart_values_to_fields",
|
|
lambda *_a, **_k: {"时间": "8.4", "男性比例": 0.5},
|
|
)
|
|
normalize_name = "normalize_tm_payload"
|
|
elif worker is dy_worker:
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"load_latest_profile",
|
|
lambda *_a, **_k: {
|
|
"style_name": "款A",
|
|
"product_id": "123",
|
|
"business_date": BUSINESS_DATE.isoformat(),
|
|
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
|
},
|
|
)
|
|
monkeypatch.setattr(worker, "profile_has_positive_distribution", lambda _profile: True)
|
|
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
|
monkeypatch.setattr(worker, "profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
|
normalize_name = "normalize_dy_payload"
|
|
else:
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"load_latest_profile",
|
|
lambda *_a, **_k: {
|
|
"style_name": "款A",
|
|
"product_id": "123",
|
|
"business_date": BUSINESS_DATE.isoformat(),
|
|
"profile": {"性别": [{"name": "男", "value": 50}]},
|
|
},
|
|
)
|
|
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
|
monkeypatch.setattr(worker, "map_profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
|
normalize_name = "normalize_jd_payload"
|
|
|
|
fake_db = SimpleNamespace(
|
|
get_conn=lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")),
|
|
upsert_persona_metrics=lambda *_a, **_k: 1,
|
|
**{normalize_name: lambda _payload: {"valid": True}},
|
|
)
|
|
monkeypatch.setitem(sys.modules, "db", fake_db)
|
|
monkeypatch.setattr(
|
|
worker,
|
|
"upsert_record",
|
|
lambda *_a, **_k: (_ for _ in ()).throw(
|
|
AssertionError("Base write must not run after DB failure")
|
|
),
|
|
)
|
|
|
|
result = worker.upsert_one_style(
|
|
"款A",
|
|
{"base_token": "bas_test", "table_id": "tbl_test"},
|
|
"8.4",
|
|
"user",
|
|
False,
|
|
BUSINESS_DATE,
|
|
)
|
|
|
|
assert result["status"] == "db_failed"
|
|
|
|
|
|
def test_zero_distributions_are_not_valid_persona() -> None:
|
|
assert dy_worker.profile_has_positive_distribution(
|
|
{
|
|
"gender_distribution": [{"name": "男", "value": 0}],
|
|
"age_distribution": [],
|
|
"strategy_crowd_distribution": [{"name": "Z世代", "value": "0%"}],
|
|
}
|
|
) is False
|
|
|
|
|
|
def test_dy_persistent_profile_is_not_overwritten_by_exported_cookies() -> None:
|
|
assert dy_collector.persistent_profile_launch_options("bound-profile") == {
|
|
"user_data_dir": "bound-profile"
|
|
}
|
|
|
|
|
|
def test_dy_profileless_fallback_can_restore_exported_cookies() -> None:
|
|
cookies = [{"name": "session", "value": "new", "domain": ".jinritemai.com"}]
|
|
|
|
assert dy_collector.cookie_fallback_launch_options(cookies) == {
|
|
"cookies": cookies
|
|
}
|
|
assert dy_collector.cookie_fallback_launch_options(None) == {}
|
|
|
|
|
|
def test_dmp_skips_are_not_valid_style_results() -> None:
|
|
styles = {"款A": ["123"]}
|
|
assert dmp.records_exit_code(
|
|
styles,
|
|
[{"style_name": "款A", "status": "skipped", "skip_reason": "无图表"}],
|
|
) == 2
|
|
assert dmp.records_exit_code(
|
|
styles,
|
|
[
|
|
{
|
|
"style_name": "款A",
|
|
"status": "ok",
|
|
"chart_values": {
|
|
"用户性别": [{"category": "男性用户", "分析人群占比": 50}]
|
|
},
|
|
}
|
|
],
|
|
) == 0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("detector", "url", "body", "expected"),
|
|
(
|
|
(dmp.blocking_page_reason, "https://dmp.taobao.com/", "暂无权限", "权限"),
|
|
(dy_collector.blocking_page_reason, "https://compass.test/", "请登录", "登录"),
|
|
(jd_worker.blocking_page_reason, "https://passport.jd.com/login", "", "登录"),
|
|
),
|
|
)
|
|
def test_login_and_permission_pages_have_explicit_diagnostics(
|
|
detector,
|
|
url: str,
|
|
body: str,
|
|
expected: str,
|
|
) -> None:
|
|
assert expected in detector(url, body)
|
|
|
|
|
|
def test_persona_search_selectors_are_finite_and_evidence_based() -> None:
|
|
assert 1 <= len(dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS) <= 6
|
|
assert 1 <= len(jd_worker.JD_SEARCH_INPUT_SELECTORS) <= 6
|
|
assert all("input" in selector for selector in dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS)
|
|
assert all("input" in selector for selector in jd_worker.JD_SEARCH_INPUT_SELECTORS)
|
|
|
|
|
|
def test_tm_collector_command_redacts_credentials() -> None:
|
|
rendered = tm_worker.redacted_command(
|
|
["python", "collector.py", "--account", "secret-user", "--password", "secret-pass"]
|
|
)
|
|
|
|
assert "secret-user" not in rendered
|
|
assert "secret-pass" not in rendered
|
|
assert rendered.count("***") == 2
|
|
|
|
|
|
def test_jd_persona_login_home_navigation_does_not_wait_for_networkidle() -> None:
|
|
project_root = Path(__file__).resolve().parents[3]
|
|
source_path = (
|
|
project_root
|
|
/ "src"
|
|
/ "gyxx_flow"
|
|
/ "modules"
|
|
/ "product_commerce"
|
|
/ "vendors"
|
|
/ "jd-data-flow"
|
|
/ "jd_data_collector.py"
|
|
)
|
|
source = source_path.read_text(encoding="utf-8")
|
|
|
|
step_login = source[source.index("def step_login") :]
|
|
home_block = step_login[step_login.index("shop.jd.com/jdm/home") :]
|
|
assert 'wait_until="networkidle"' not in home_block[:400]
|
|
assert 'wait_until="domcontentloaded"' in home_block[:400]
|
|
assert "timeout=60_000" in home_block[:400]
|