477 lines
16 KiB
Python
477 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import http.client
|
|
import json
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from gyxx_flow.catalog import ScheduleEntry
|
|
from gyxx_flow.console import create_console_server
|
|
from gyxx_flow.core.config import Settings
|
|
from gyxx_flow.core.context import RunContext
|
|
from gyxx_flow.core.layout import DataLayout
|
|
from gyxx_flow.core.records import RunJournal
|
|
from gyxx_flow.daily_summary import DailySummaryBuilder, _schedule_slots
|
|
from gyxx_flow.ops import RunIndex
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class FakeAnalyzer:
|
|
def __init__(self) -> None:
|
|
self.calls: list[dict[str, object]] = []
|
|
|
|
@property
|
|
def identity(self) -> str:
|
|
return "fake:v1"
|
|
|
|
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
|
self.calls.append(payload)
|
|
workflows = []
|
|
for item in payload["workflows"]:
|
|
workflow_id = item["workflow_id"]
|
|
status = item["status"]
|
|
if workflow_id == "content.success.daily":
|
|
execution_result = "当天的数据采集已顺利完成，预定内容已经产出。"
|
|
anomalies = []
|
|
repair_actions = []
|
|
elif workflow_id == "content.missed.daily":
|
|
execution_result = "计划执行时间已过，但当天没有发现任务启动记录。"
|
|
anomalies = ["常驻调度服务没有按计划启动这个工作流，日报内容因此没有生成。"]
|
|
repair_actions = ["检查常驻调度服务和计划设置，恢复后按原业务日期补跑。"]
|
|
else:
|
|
execution_result = "当天的工作流执行情况已经完成检查。"
|
|
anomalies = (
|
|
[]
|
|
if status in {"normal", "recovered"}
|
|
else ["该工作流没有按预期完成，相关业务结果可能缺失。"]
|
|
)
|
|
repair_actions = (
|
|
[]
|
|
if status in {"normal", "recovered"}
|
|
else ["检查运行环境和计划设置，排除原因后按原业务日期重新执行。"]
|
|
)
|
|
workflows.append(
|
|
{
|
|
"workflow_id": workflow_id,
|
|
"execution_result": execution_result,
|
|
"anomalies": anomalies,
|
|
"repair_actions": repair_actions,
|
|
}
|
|
)
|
|
return {
|
|
"overview": "模型已逐个检查运行证据。",
|
|
"workflows": workflows,
|
|
}
|
|
|
|
|
|
def _write_catalog(project_root: Path) -> None:
|
|
config = project_root / "config"
|
|
config.mkdir(parents=True)
|
|
workflows = {
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": workflow_id,
|
|
"module": "content_marketing",
|
|
"trigger": "scheduled",
|
|
"execution": {"entry": "daily.py"},
|
|
"provenance": {
|
|
"source_project": "content",
|
|
"task_name": name,
|
|
},
|
|
}
|
|
for workflow_id, name in (
|
|
("content.success.daily", "成功日报"),
|
|
("content.missed.daily", "漏跑日报"),
|
|
)
|
|
],
|
|
}
|
|
schedules = {
|
|
"schema_version": 1,
|
|
"timezone": "Asia/Shanghai",
|
|
"schedules": [
|
|
{
|
|
"workflow_id": workflow_id,
|
|
"kind": "daily",
|
|
"at": at,
|
|
"enabled": True,
|
|
}
|
|
for workflow_id, at in (
|
|
("content.success.daily", "09:00"),
|
|
("content.missed.daily", "10:00"),
|
|
)
|
|
],
|
|
}
|
|
(config / "workflows.json").write_text(
|
|
json.dumps(workflows, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
(config / "schedules.json").write_text(
|
|
json.dumps(schedules, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _write_multi_slot_catalog(project_root: Path) -> None:
|
|
config = project_root / "config"
|
|
config.mkdir(parents=True)
|
|
workflows = {
|
|
"schema_version": 3,
|
|
"workflows": [
|
|
{
|
|
"id": "content.success.daily",
|
|
"module": "content_marketing",
|
|
"trigger": "scheduled",
|
|
"execution": {"entry": "daily.py"},
|
|
"provenance": {
|
|
"source_project": "content",
|
|
"task_name": "成功日报",
|
|
},
|
|
}
|
|
],
|
|
}
|
|
schedules = {
|
|
"schema_version": 1,
|
|
"timezone": "Asia/Shanghai",
|
|
"schedules": [
|
|
{
|
|
"workflow_id": "content.success.daily",
|
|
"kind": "daily",
|
|
"at": ["08:00", "16:00", "22:00"],
|
|
"enabled": True,
|
|
}
|
|
],
|
|
}
|
|
(config / "workflows.json").write_text(
|
|
json.dumps(workflows, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
(config / "schedules.json").write_text(
|
|
json.dumps(schedules, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _successful_run(data_root: Path) -> None:
|
|
context = RunContext.create(
|
|
"content.success.daily",
|
|
"2026-08-12",
|
|
now=datetime(2026, 8, 13, 1, 0, tzinfo=timezone.utc),
|
|
random_suffix="abc123",
|
|
)
|
|
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
|
journal.start_step("collect.attempt-1", attempt=1)
|
|
journal.finish_step("collect.attempt-1", status="success", exit_code=0)
|
|
journal.finalize("success")
|
|
RunIndex(data_root).index_journal(journal)
|
|
|
|
|
|
def test_daily_summary_keeps_status_deterministic_and_analyzes_each_workflow(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
project_root = tmp_path / "project"
|
|
data_root = tmp_path / "data"
|
|
_write_catalog(project_root)
|
|
_successful_run(data_root)
|
|
log_root = data_root / "logs" / "scheduler"
|
|
log_root.mkdir(parents=True)
|
|
(log_root / "content.success.daily-20260813T090000+0800.log").write_text(
|
|
"completed token=top-secret-value",
|
|
encoding="utf-8",
|
|
)
|
|
analyzer = FakeAnalyzer()
|
|
builder = DailySummaryBuilder(
|
|
project_root=project_root,
|
|
data_root=data_root,
|
|
analyzer=analyzer,
|
|
workflow_name=lambda workflow_id: {
|
|
"content.success.daily": "成功日报",
|
|
"content.missed.daily": "漏跑日报",
|
|
}[workflow_id],
|
|
)
|
|
|
|
report = builder.build(datetime(2026, 8, 13).date())
|
|
|
|
by_id = {item["workflow_id"]: item for item in report["workflows"]}
|
|
assert report["overview"] == "模型已逐个检查运行证据。"
|
|
assert report["summary"] == {
|
|
"total": 2,
|
|
"normal": 1,
|
|
"recovered": 0,
|
|
"abnormal": 0,
|
|
"running": 0,
|
|
"missed": 1,
|
|
}
|
|
assert by_id["content.success.daily"]["status"] == "normal"
|
|
assert by_id["content.success.daily"]["execution_result"] == (
|
|
"当天的数据采集已顺利完成，预定内容已经产出。"
|
|
)
|
|
assert by_id["content.missed.daily"]["status"] == "missed"
|
|
assert by_id["content.missed.daily"]["repair_actions"] == [
|
|
"检查常驻调度服务和计划设置，恢复后按原业务日期补跑。"
|
|
]
|
|
assert "top-secret-value" not in json.dumps(analyzer.calls, ensure_ascii=False)
|
|
assert "[REDACTED]" in json.dumps(analyzer.calls, ensure_ascii=False)
|
|
serialized_report = json.dumps(report, ensure_ascii=False)
|
|
assert "content.success.daily-20260813T090000+0800.log" not in serialized_report
|
|
assert "completed token" not in serialized_report
|
|
assert "[REDACTED]" not in serialized_report
|
|
|
|
cached = builder.build(datetime(2026, 8, 13).date())
|
|
assert cached == report
|
|
assert len(analyzer.calls) == 1
|
|
|
|
|
|
def test_daily_summary_detects_failed_run_and_returns_rule_based_repair(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
project_root = tmp_path / "project"
|
|
data_root = tmp_path / "data"
|
|
_write_catalog(project_root)
|
|
context = RunContext.create(
|
|
"content.success.daily",
|
|
"2026-08-12",
|
|
now=datetime(2026, 8, 13, 1, 0, tzinfo=timezone.utc),
|
|
random_suffix="def456",
|
|
)
|
|
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
|
journal.start_step("notify.attempt-1", attempt=1)
|
|
journal.finish_step(
|
|
"notify.attempt-1",
|
|
status="failed",
|
|
exit_code=1,
|
|
error=(
|
|
'Traceback: File "D:\\secret\\notify.py", line 42; '
|
|
"open_id cross app (99992361) token=do-not-publish"
|
|
),
|
|
)
|
|
journal.finalize(
|
|
"failed",
|
|
error=(
|
|
'Traceback: File "D:\\secret\\notify.py", line 42; '
|
|
"open_id cross app (99992361) token=do-not-publish"
|
|
),
|
|
)
|
|
RunIndex(data_root).index_journal(journal)
|
|
|
|
class UnavailableAnalyzer(FakeAnalyzer):
|
|
@property
|
|
def identity(self) -> str:
|
|
return "unavailable:v1"
|
|
|
|
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
|
from gyxx_flow.daily_summary import DailySummaryAnalysisError
|
|
|
|
raise DailySummaryAnalysisError("Hermes 未启动")
|
|
|
|
report = DailySummaryBuilder(
|
|
project_root=project_root,
|
|
data_root=data_root,
|
|
analyzer=UnavailableAnalyzer(),
|
|
).build(datetime(2026, 8, 13).date())
|
|
|
|
by_id = {item["workflow_id"]: item for item in report["workflows"]}
|
|
failed = by_id["content.success.daily"]
|
|
assert report["analysis"] == {
|
|
"status": "unavailable",
|
|
"source": "rules",
|
|
"message": "大模型分析服务暂时不可用，当前展示系统生成的通俗基础说明。",
|
|
}
|
|
assert failed["status"] == "abnormal"
|
|
assert "身份绑定不一致" in failed["anomalies"][0]
|
|
assert "重新绑定" in failed["repair_actions"][0]
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
assert "Traceback" not in serialized
|
|
assert "notify.py" not in serialized
|
|
assert "open_id cross app" not in serialized
|
|
assert "99992361" not in serialized
|
|
assert "do-not-publish" not in serialized
|
|
assert all("error" not in run and "steps" not in run for run in failed["runs"])
|
|
assert "log_evidence" not in failed
|
|
|
|
|
|
def test_daily_summary_rejects_model_output_that_copies_raw_diagnostics(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
project_root = tmp_path / "project"
|
|
data_root = tmp_path / "data"
|
|
_write_catalog(project_root)
|
|
_successful_run(data_root)
|
|
|
|
class EchoingAnalyzer(FakeAnalyzer):
|
|
@property
|
|
def identity(self) -> str:
|
|
return "echoing:v1"
|
|
|
|
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
|
result = super().analyze(payload)
|
|
result["workflows"][0]["execution_result"] = (
|
|
'Traceback: File "D:\\internal\\worker.py", line 17'
|
|
)
|
|
return result
|
|
|
|
report = DailySummaryBuilder(
|
|
project_root=project_root,
|
|
data_root=data_root,
|
|
analyzer=EchoingAnalyzer(),
|
|
).build(datetime(2026, 8, 13).date())
|
|
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
assert report["analysis"]["status"] == "unavailable"
|
|
assert "Traceback" not in serialized
|
|
assert "worker.py" not in serialized
|
|
assert "正式执行 1 次" in serialized
|
|
|
|
|
|
def test_daily_summary_normalizes_relative_dates_in_model_output(tmp_path: Path) -> None:
|
|
project_root = tmp_path / "project"
|
|
data_root = tmp_path / "data"
|
|
_write_catalog(project_root)
|
|
_successful_run(data_root)
|
|
|
|
class RelativeDateAnalyzer(FakeAnalyzer):
|
|
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
|
result = super().analyze(payload)
|
|
result["overview"] = "今日工作流已完成检查。"
|
|
result["workflows"][0]["execution_result"] = "今天的数据采集已完成。"
|
|
return result
|
|
|
|
report = DailySummaryBuilder(
|
|
project_root=project_root,
|
|
data_root=data_root,
|
|
analyzer=RelativeDateAnalyzer(),
|
|
).build(datetime(2026, 8, 13).date())
|
|
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
assert "今日" not in serialized
|
|
assert "今天" not in serialized
|
|
assert "当天工作流已完成检查" in serialized
|
|
|
|
|
|
def test_daily_summary_http_get_and_forced_refresh(tmp_path: Path) -> None:
|
|
analyzer = FakeAnalyzer()
|
|
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path / "data")
|
|
server = create_console_server(
|
|
settings,
|
|
host="127.0.0.1",
|
|
port=0,
|
|
daily_summary_analyzer=analyzer,
|
|
)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
connection = http.client.HTTPConnection(
|
|
"127.0.0.1",
|
|
server.server_port,
|
|
timeout=10,
|
|
)
|
|
connection.request("GET", "/api/daily-summary?date=2026-08-13")
|
|
response = connection.getresponse()
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
assert response.status == 200
|
|
assert payload["report_date"] == "2026-08-13"
|
|
assert payload["schema_version"] == 2
|
|
assert payload["workflows"]
|
|
assert all(
|
|
{"workflow_id", "status", "execution_result", "repair_actions"}
|
|
<= set(item)
|
|
for item in payload["workflows"]
|
|
)
|
|
|
|
body = json.dumps({"date": "2026-08-13"}).encode("utf-8")
|
|
connection.request(
|
|
"POST",
|
|
"/api/daily-summary/refresh",
|
|
body=body,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"Content-Length": str(len(body)),
|
|
"X-GYXX-Console": "1",
|
|
"Origin": f"http://127.0.0.1:{server.server_port}",
|
|
},
|
|
)
|
|
refreshed = connection.getresponse()
|
|
refreshed.read()
|
|
assert refreshed.status == 200
|
|
assert len(analyzer.calls) == 2
|
|
connection.close()
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join(timeout=5)
|
|
|
|
|
|
def test_schedule_slots_filters_future_times_when_now_is_given() -> None:
|
|
schedule = ScheduleEntry(
|
|
workflow_id="content.success.daily",
|
|
kind="daily",
|
|
at="08:00",
|
|
at_times=("08:00", "16:00", "22:00"),
|
|
)
|
|
timezone_info = ZoneInfo("Asia/Shanghai")
|
|
target = datetime(2026, 8, 18).date()
|
|
|
|
all_slots = _schedule_slots(schedule, target, timezone_info)
|
|
assert [slot.strftime("%H:%M") for slot in all_slots] == [
|
|
"08:00",
|
|
"16:00",
|
|
"22:00",
|
|
]
|
|
|
|
morning = _schedule_slots(
|
|
schedule,
|
|
target,
|
|
timezone_info,
|
|
now=datetime(2026, 8, 18, 9, 55, tzinfo=timezone_info),
|
|
)
|
|
assert [slot.strftime("%H:%M") for slot in morning] == ["08:00"]
|
|
|
|
|
|
def test_daily_summary_does_not_flag_future_multi_slots_as_missing(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
project_root = tmp_path / "project"
|
|
data_root = tmp_path / "data"
|
|
_write_multi_slot_catalog(project_root)
|
|
|
|
context = RunContext.create(
|
|
"content.success.daily",
|
|
"2026-08-18",
|
|
now=datetime(2026, 8, 18, 0, 1, tzinfo=timezone.utc),
|
|
random_suffix="abc123",
|
|
)
|
|
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
|
journal.start_step("collect.attempt-1", attempt=1)
|
|
journal.finish_step("collect.attempt-1", status="success", exit_code=0)
|
|
journal.finalize("success")
|
|
RunIndex(data_root).index_journal(journal)
|
|
|
|
log_root = data_root / "logs" / "scheduler"
|
|
log_root.mkdir(parents=True)
|
|
(log_root / "content.success.daily-20260818T080000+0800.log").write_text(
|
|
"completed",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
analyzer = FakeAnalyzer()
|
|
builder = DailySummaryBuilder(
|
|
project_root=project_root,
|
|
data_root=data_root,
|
|
analyzer=analyzer,
|
|
workflow_name=lambda workflow_id: "成功日报",
|
|
)
|
|
report = builder.build(
|
|
datetime(2026, 8, 18).date(),
|
|
now=datetime(2026, 8, 18, 9, 55, tzinfo=ZoneInfo("Asia/Shanghai")),
|
|
)
|
|
|
|
item = report["workflows"][0]
|
|
assert item["workflow_id"] == "content.success.daily"
|
|
assert item["expected_slots"] == ["2026-08-18T08:00:00+08:00"]
|
|
assert item["missed_slots"] == 0
|
|
assert item["status"] == "normal"
|
|
assert "没有发现调度日志或运行记录" not in item["execution_result"]
|