feat: complete production workflow migration
This commit is contained in:
@@ -0,0 +1,319 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import collect_dy_persona_to_bitable as dy_worker
|
||||
import collect_jd_persona_to_bitable as jd_worker
|
||||
import collect_persona_to_bitable as tm_worker
|
||||
import dy_audience_profile_collect as dy_collector
|
||||
import pytest
|
||||
import taobao_dmp_item_crowd_insight_screenshots as dmp
|
||||
|
||||
from gyxx_flow.adapters import acceptance_policy
|
||||
|
||||
WORKERS = (tm_worker, dy_worker, jd_worker)
|
||||
BUSINESS_DATE = date(2026, 8, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_platform_collector_failure_forces_nonzero(worker) -> None:
|
||||
assert worker.combined_result_exit_code(
|
||||
[{"status": "ok"}],
|
||||
collection_ok=False,
|
||||
) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_upsert_requires_response_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"ok": True, "data": {}}),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert "record_id" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"data": {"record": {"record_id_list": ["rec12345678"]}},
|
||||
}
|
||||
),
|
||||
stderr="",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is True
|
||||
assert message == "record_id=rec12345678"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None:
|
||||
monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True)
|
||||
monkeypatch.setattr(
|
||||
worker.subprocess,
|
||||
"run",
|
||||
lambda *_a, **_k: (_ for _ in ()).throw(
|
||||
AssertionError("lark subprocess must not run after acceptance skip")
|
||||
),
|
||||
)
|
||||
|
||||
ok, message = worker.upsert_record(
|
||||
"bas_test",
|
||||
"tbl_test",
|
||||
{"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert "acceptance-skipped" in message
|
||||
|
||||
|
||||
def test_tm_payload_must_match_requested_business_date(monkeypatch, tmp_path: Path) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
(style_dir / f"{style}_123_chart_values.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"style_name": style,
|
||||
"item_id": "123",
|
||||
"business_date": "2026-08-03",
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(tm_worker, "DMP_OUTPUT_ROOT", tmp_path)
|
||||
|
||||
assert tm_worker.load_latest_chart_values(style, BUSINESS_DATE) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", (dy_worker, jd_worker))
|
||||
def test_profile_payload_must_match_requested_business_date(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
worker,
|
||||
) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
(style_dir / f"{style}_123_profile.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"style_name": style,
|
||||
"product_id": "123",
|
||||
"business_date": "2026-08-03",
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
||||
|
||||
assert worker.load_latest_profile(style, BUSINESS_DATE) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", (tm_worker, dy_worker))
|
||||
def test_current_run_does_not_reuse_old_artifact(monkeypatch, tmp_path: Path, worker) -> None:
|
||||
style = "款A"
|
||||
style_dir = tmp_path / BUSINESS_DATE.isoformat() / style
|
||||
style_dir.mkdir(parents=True)
|
||||
if worker is tm_worker:
|
||||
path = style_dir / f"{style}_123_chart_values.json"
|
||||
payload = {
|
||||
"style_name": style,
|
||||
"item_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
}
|
||||
monkeypatch.setattr(worker, "DMP_OUTPUT_ROOT", tmp_path)
|
||||
load = worker.load_latest_chart_values
|
||||
else:
|
||||
path = style_dir / f"{style}_123_profile.json"
|
||||
payload = {
|
||||
"style_name": style,
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
}
|
||||
monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path)
|
||||
load = worker.load_latest_profile
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
assert load(
|
||||
style,
|
||||
BUSINESS_DATE,
|
||||
artifact_not_before=path.stat().st_mtime + 10,
|
||||
) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker", WORKERS)
|
||||
def test_database_failure_stops_before_base_write(monkeypatch, worker) -> None:
|
||||
if worker is tm_worker:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_chart_values",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"item_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"chart_values_to_fields",
|
||||
lambda *_a, **_k: {"时间": "8.4", "男性比例": 0.5},
|
||||
)
|
||||
normalize_name = "normalize_tm_payload"
|
||||
elif worker is dy_worker:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_profile",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"gender_distribution": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(worker, "profile_has_positive_distribution", lambda _profile: True)
|
||||
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
||||
monkeypatch.setattr(worker, "profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
||||
normalize_name = "normalize_dy_payload"
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"load_latest_profile",
|
||||
lambda *_a, **_k: {
|
||||
"style_name": "款A",
|
||||
"product_id": "123",
|
||||
"business_date": BUSINESS_DATE.isoformat(),
|
||||
"profile": {"性别": [{"name": "男", "value": 50}]},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: [])
|
||||
monkeypatch.setattr(worker, "map_profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5})
|
||||
normalize_name = "normalize_jd_payload"
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
get_conn=lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")),
|
||||
upsert_persona_metrics=lambda *_a, **_k: 1,
|
||||
**{normalize_name: lambda _payload: {"valid": True}},
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "db", fake_db)
|
||||
monkeypatch.setattr(
|
||||
worker,
|
||||
"upsert_record",
|
||||
lambda *_a, **_k: (_ for _ in ()).throw(
|
||||
AssertionError("Base write must not run after DB failure")
|
||||
),
|
||||
)
|
||||
|
||||
result = worker.upsert_one_style(
|
||||
"款A",
|
||||
{"base_token": "bas_test", "table_id": "tbl_test"},
|
||||
"8.4",
|
||||
"user",
|
||||
False,
|
||||
BUSINESS_DATE,
|
||||
)
|
||||
|
||||
assert result["status"] == "db_failed"
|
||||
|
||||
|
||||
def test_zero_distributions_are_not_valid_persona() -> None:
|
||||
assert dy_worker.profile_has_positive_distribution(
|
||||
{
|
||||
"gender_distribution": [{"name": "男", "value": 0}],
|
||||
"age_distribution": [],
|
||||
"strategy_crowd_distribution": [{"name": "Z世代", "value": "0%"}],
|
||||
}
|
||||
) is False
|
||||
|
||||
|
||||
def test_dmp_skips_are_not_valid_style_results() -> None:
|
||||
styles = {"款A": ["123"]}
|
||||
assert dmp.records_exit_code(
|
||||
styles,
|
||||
[{"style_name": "款A", "status": "skipped", "skip_reason": "无图表"}],
|
||||
) == 2
|
||||
assert dmp.records_exit_code(
|
||||
styles,
|
||||
[
|
||||
{
|
||||
"style_name": "款A",
|
||||
"status": "ok",
|
||||
"chart_values": {
|
||||
"用户性别": [{"category": "男性用户", "分析人群占比": 50}]
|
||||
},
|
||||
}
|
||||
],
|
||||
) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("detector", "url", "body", "expected"),
|
||||
(
|
||||
(dmp.blocking_page_reason, "https://dmp.taobao.com/", "暂无权限", "权限"),
|
||||
(dy_collector.blocking_page_reason, "https://compass.test/", "请登录", "登录"),
|
||||
(jd_worker.blocking_page_reason, "https://passport.jd.com/login", "", "登录"),
|
||||
),
|
||||
)
|
||||
def test_login_and_permission_pages_have_explicit_diagnostics(
|
||||
detector,
|
||||
url: str,
|
||||
body: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert expected in detector(url, body)
|
||||
|
||||
|
||||
def test_persona_search_selectors_are_finite_and_evidence_based() -> None:
|
||||
assert 1 <= len(dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS) <= 6
|
||||
assert 1 <= len(jd_worker.JD_SEARCH_INPUT_SELECTORS) <= 6
|
||||
assert all("input" in selector for selector in dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS)
|
||||
assert all("input" in selector for selector in jd_worker.JD_SEARCH_INPUT_SELECTORS)
|
||||
|
||||
|
||||
def test_tm_collector_command_redacts_credentials() -> None:
|
||||
rendered = tm_worker.redacted_command(
|
||||
["python", "collector.py", "--account", "secret-user", "--password", "secret-pass"]
|
||||
)
|
||||
|
||||
assert "secret-user" not in rendered
|
||||
assert "secret-pass" not in rendered
|
||||
assert rendered.count("***") == 2
|
||||
Reference in New Issue
Block a user