feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
@@ -0,0 +1,476 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from gyxx_flow.catalog import ScheduleEntry
|
||||
from gyxx_flow.console import create_console_server
|
||||
from gyxx_flow.core.config import Settings
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.core.layout import DataLayout
|
||||
from gyxx_flow.core.records import RunJournal
|
||||
from gyxx_flow.daily_summary import DailySummaryBuilder, _schedule_slots
|
||||
from gyxx_flow.ops import RunIndex
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class FakeAnalyzer:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[dict[str, object]] = []
|
||||
|
||||
@property
|
||||
def identity(self) -> str:
|
||||
return "fake:v1"
|
||||
|
||||
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
self.calls.append(payload)
|
||||
workflows = []
|
||||
for item in payload["workflows"]:
|
||||
workflow_id = item["workflow_id"]
|
||||
status = item["status"]
|
||||
if workflow_id == "content.success.daily":
|
||||
execution_result = "当天的数据采集已顺利完成，预定内容已经产出。"
|
||||
anomalies = []
|
||||
repair_actions = []
|
||||
elif workflow_id == "content.missed.daily":
|
||||
execution_result = "计划执行时间已过，但当天没有发现任务启动记录。"
|
||||
anomalies = ["常驻调度服务没有按计划启动这个工作流，日报内容因此没有生成。"]
|
||||
repair_actions = ["检查常驻调度服务和计划设置，恢复后按原业务日期补跑。"]
|
||||
else:
|
||||
execution_result = "当天的工作流执行情况已经完成检查。"
|
||||
anomalies = (
|
||||
[]
|
||||
if status in {"normal", "recovered"}
|
||||
else ["该工作流没有按预期完成，相关业务结果可能缺失。"]
|
||||
)
|
||||
repair_actions = (
|
||||
[]
|
||||
if status in {"normal", "recovered"}
|
||||
else ["检查运行环境和计划设置，排除原因后按原业务日期重新执行。"]
|
||||
)
|
||||
workflows.append(
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"execution_result": execution_result,
|
||||
"anomalies": anomalies,
|
||||
"repair_actions": repair_actions,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"overview": "模型已逐个检查运行证据。",
|
||||
"workflows": workflows,
|
||||
}
|
||||
|
||||
|
||||
def _write_catalog(project_root: Path) -> None:
|
||||
config = project_root / "config"
|
||||
config.mkdir(parents=True)
|
||||
workflows = {
|
||||
"schema_version": 3,
|
||||
"workflows": [
|
||||
{
|
||||
"id": workflow_id,
|
||||
"module": "content_marketing",
|
||||
"trigger": "scheduled",
|
||||
"execution": {"entry": "daily.py"},
|
||||
"provenance": {
|
||||
"source_project": "content",
|
||||
"task_name": name,
|
||||
},
|
||||
}
|
||||
for workflow_id, name in (
|
||||
("content.success.daily", "成功日报"),
|
||||
("content.missed.daily", "漏跑日报"),
|
||||
)
|
||||
],
|
||||
}
|
||||
schedules = {
|
||||
"schema_version": 1,
|
||||
"timezone": "Asia/Shanghai",
|
||||
"schedules": [
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"kind": "daily",
|
||||
"at": at,
|
||||
"enabled": True,
|
||||
}
|
||||
for workflow_id, at in (
|
||||
("content.success.daily", "09:00"),
|
||||
("content.missed.daily", "10:00"),
|
||||
)
|
||||
],
|
||||
}
|
||||
(config / "workflows.json").write_text(
|
||||
json.dumps(workflows, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(config / "schedules.json").write_text(
|
||||
json.dumps(schedules, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_multi_slot_catalog(project_root: Path) -> None:
|
||||
config = project_root / "config"
|
||||
config.mkdir(parents=True)
|
||||
workflows = {
|
||||
"schema_version": 3,
|
||||
"workflows": [
|
||||
{
|
||||
"id": "content.success.daily",
|
||||
"module": "content_marketing",
|
||||
"trigger": "scheduled",
|
||||
"execution": {"entry": "daily.py"},
|
||||
"provenance": {
|
||||
"source_project": "content",
|
||||
"task_name": "成功日报",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
schedules = {
|
||||
"schema_version": 1,
|
||||
"timezone": "Asia/Shanghai",
|
||||
"schedules": [
|
||||
{
|
||||
"workflow_id": "content.success.daily",
|
||||
"kind": "daily",
|
||||
"at": ["08:00", "16:00", "22:00"],
|
||||
"enabled": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
(config / "workflows.json").write_text(
|
||||
json.dumps(workflows, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(config / "schedules.json").write_text(
|
||||
json.dumps(schedules, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _successful_run(data_root: Path) -> None:
|
||||
context = RunContext.create(
|
||||
"content.success.daily",
|
||||
"2026-08-12",
|
||||
now=datetime(2026, 8, 13, 1, 0, tzinfo=timezone.utc),
|
||||
random_suffix="abc123",
|
||||
)
|
||||
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
||||
journal.start_step("collect.attempt-1", attempt=1)
|
||||
journal.finish_step("collect.attempt-1", status="success", exit_code=0)
|
||||
journal.finalize("success")
|
||||
RunIndex(data_root).index_journal(journal)
|
||||
|
||||
|
||||
def test_daily_summary_keeps_status_deterministic_and_analyzes_each_workflow(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project_root = tmp_path / "project"
|
||||
data_root = tmp_path / "data"
|
||||
_write_catalog(project_root)
|
||||
_successful_run(data_root)
|
||||
log_root = data_root / "logs" / "scheduler"
|
||||
log_root.mkdir(parents=True)
|
||||
(log_root / "content.success.daily-20260813T090000+0800.log").write_text(
|
||||
"completed token=top-secret-value",
|
||||
encoding="utf-8",
|
||||
)
|
||||
analyzer = FakeAnalyzer()
|
||||
builder = DailySummaryBuilder(
|
||||
project_root=project_root,
|
||||
data_root=data_root,
|
||||
analyzer=analyzer,
|
||||
workflow_name=lambda workflow_id: {
|
||||
"content.success.daily": "成功日报",
|
||||
"content.missed.daily": "漏跑日报",
|
||||
}[workflow_id],
|
||||
)
|
||||
|
||||
report = builder.build(datetime(2026, 8, 13).date())
|
||||
|
||||
by_id = {item["workflow_id"]: item for item in report["workflows"]}
|
||||
assert report["overview"] == "模型已逐个检查运行证据。"
|
||||
assert report["summary"] == {
|
||||
"total": 2,
|
||||
"normal": 1,
|
||||
"recovered": 0,
|
||||
"abnormal": 0,
|
||||
"running": 0,
|
||||
"missed": 1,
|
||||
}
|
||||
assert by_id["content.success.daily"]["status"] == "normal"
|
||||
assert by_id["content.success.daily"]["execution_result"] == (
|
||||
"当天的数据采集已顺利完成，预定内容已经产出。"
|
||||
)
|
||||
assert by_id["content.missed.daily"]["status"] == "missed"
|
||||
assert by_id["content.missed.daily"]["repair_actions"] == [
|
||||
"检查常驻调度服务和计划设置，恢复后按原业务日期补跑。"
|
||||
]
|
||||
assert "top-secret-value" not in json.dumps(analyzer.calls, ensure_ascii=False)
|
||||
assert "[REDACTED]" in json.dumps(analyzer.calls, ensure_ascii=False)
|
||||
serialized_report = json.dumps(report, ensure_ascii=False)
|
||||
assert "content.success.daily-20260813T090000+0800.log" not in serialized_report
|
||||
assert "completed token" not in serialized_report
|
||||
assert "[REDACTED]" not in serialized_report
|
||||
|
||||
cached = builder.build(datetime(2026, 8, 13).date())
|
||||
assert cached == report
|
||||
assert len(analyzer.calls) == 1
|
||||
|
||||
|
||||
def test_daily_summary_detects_failed_run_and_returns_rule_based_repair(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project_root = tmp_path / "project"
|
||||
data_root = tmp_path / "data"
|
||||
_write_catalog(project_root)
|
||||
context = RunContext.create(
|
||||
"content.success.daily",
|
||||
"2026-08-12",
|
||||
now=datetime(2026, 8, 13, 1, 0, tzinfo=timezone.utc),
|
||||
random_suffix="def456",
|
||||
)
|
||||
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
||||
journal.start_step("notify.attempt-1", attempt=1)
|
||||
journal.finish_step(
|
||||
"notify.attempt-1",
|
||||
status="failed",
|
||||
exit_code=1,
|
||||
error=(
|
||||
'Traceback: File "D:\\secret\\notify.py", line 42; '
|
||||
"open_id cross app (99992361) token=do-not-publish"
|
||||
),
|
||||
)
|
||||
journal.finalize(
|
||||
"failed",
|
||||
error=(
|
||||
'Traceback: File "D:\\secret\\notify.py", line 42; '
|
||||
"open_id cross app (99992361) token=do-not-publish"
|
||||
),
|
||||
)
|
||||
RunIndex(data_root).index_journal(journal)
|
||||
|
||||
class UnavailableAnalyzer(FakeAnalyzer):
|
||||
@property
|
||||
def identity(self) -> str:
|
||||
return "unavailable:v1"
|
||||
|
||||
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
from gyxx_flow.daily_summary import DailySummaryAnalysisError
|
||||
|
||||
raise DailySummaryAnalysisError("Hermes 未启动")
|
||||
|
||||
report = DailySummaryBuilder(
|
||||
project_root=project_root,
|
||||
data_root=data_root,
|
||||
analyzer=UnavailableAnalyzer(),
|
||||
).build(datetime(2026, 8, 13).date())
|
||||
|
||||
by_id = {item["workflow_id"]: item for item in report["workflows"]}
|
||||
failed = by_id["content.success.daily"]
|
||||
assert report["analysis"] == {
|
||||
"status": "unavailable",
|
||||
"source": "rules",
|
||||
"message": "大模型分析服务暂时不可用，当前展示系统生成的通俗基础说明。",
|
||||
}
|
||||
assert failed["status"] == "abnormal"
|
||||
assert "身份绑定不一致" in failed["anomalies"][0]
|
||||
assert "重新绑定" in failed["repair_actions"][0]
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
assert "Traceback" not in serialized
|
||||
assert "notify.py" not in serialized
|
||||
assert "open_id cross app" not in serialized
|
||||
assert "99992361" not in serialized
|
||||
assert "do-not-publish" not in serialized
|
||||
assert all("error" not in run and "steps" not in run for run in failed["runs"])
|
||||
assert "log_evidence" not in failed
|
||||
|
||||
|
||||
def test_daily_summary_rejects_model_output_that_copies_raw_diagnostics(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project_root = tmp_path / "project"
|
||||
data_root = tmp_path / "data"
|
||||
_write_catalog(project_root)
|
||||
_successful_run(data_root)
|
||||
|
||||
class EchoingAnalyzer(FakeAnalyzer):
|
||||
@property
|
||||
def identity(self) -> str:
|
||||
return "echoing:v1"
|
||||
|
||||
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
result = super().analyze(payload)
|
||||
result["workflows"][0]["execution_result"] = (
|
||||
'Traceback: File "D:\\internal\\worker.py", line 17'
|
||||
)
|
||||
return result
|
||||
|
||||
report = DailySummaryBuilder(
|
||||
project_root=project_root,
|
||||
data_root=data_root,
|
||||
analyzer=EchoingAnalyzer(),
|
||||
).build(datetime(2026, 8, 13).date())
|
||||
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
assert report["analysis"]["status"] == "unavailable"
|
||||
assert "Traceback" not in serialized
|
||||
assert "worker.py" not in serialized
|
||||
assert "正式执行 1 次" in serialized
|
||||
|
||||
|
||||
def test_daily_summary_normalizes_relative_dates_in_model_output(tmp_path: Path) -> None:
|
||||
project_root = tmp_path / "project"
|
||||
data_root = tmp_path / "data"
|
||||
_write_catalog(project_root)
|
||||
_successful_run(data_root)
|
||||
|
||||
class RelativeDateAnalyzer(FakeAnalyzer):
|
||||
def analyze(self, payload: dict[str, object]) -> dict[str, object]:
|
||||
result = super().analyze(payload)
|
||||
result["overview"] = "今日工作流已完成检查。"
|
||||
result["workflows"][0]["execution_result"] = "今天的数据采集已完成。"
|
||||
return result
|
||||
|
||||
report = DailySummaryBuilder(
|
||||
project_root=project_root,
|
||||
data_root=data_root,
|
||||
analyzer=RelativeDateAnalyzer(),
|
||||
).build(datetime(2026, 8, 13).date())
|
||||
|
||||
serialized = json.dumps(report, ensure_ascii=False)
|
||||
assert "今日" not in serialized
|
||||
assert "今天" not in serialized
|
||||
assert "当天工作流已完成检查" in serialized
|
||||
|
||||
|
||||
def test_daily_summary_http_get_and_forced_refresh(tmp_path: Path) -> None:
|
||||
analyzer = FakeAnalyzer()
|
||||
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path / "data")
|
||||
server = create_console_server(
|
||||
settings,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
daily_summary_analyzer=analyzer,
|
||||
)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
connection = http.client.HTTPConnection(
|
||||
"127.0.0.1",
|
||||
server.server_port,
|
||||
timeout=10,
|
||||
)
|
||||
connection.request("GET", "/api/daily-summary?date=2026-08-13")
|
||||
response = connection.getresponse()
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
assert response.status == 200
|
||||
assert payload["report_date"] == "2026-08-13"
|
||||
assert payload["schema_version"] == 2
|
||||
assert payload["workflows"]
|
||||
assert all(
|
||||
{"workflow_id", "status", "execution_result", "repair_actions"}
|
||||
<= set(item)
|
||||
for item in payload["workflows"]
|
||||
)
|
||||
|
||||
body = json.dumps({"date": "2026-08-13"}).encode("utf-8")
|
||||
connection.request(
|
||||
"POST",
|
||||
"/api/daily-summary/refresh",
|
||||
body=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Content-Length": str(len(body)),
|
||||
"X-GYXX-Console": "1",
|
||||
"Origin": f"http://127.0.0.1:{server.server_port}",
|
||||
},
|
||||
)
|
||||
refreshed = connection.getresponse()
|
||||
refreshed.read()
|
||||
assert refreshed.status == 200
|
||||
assert len(analyzer.calls) == 2
|
||||
connection.close()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_schedule_slots_filters_future_times_when_now_is_given() -> None:
|
||||
schedule = ScheduleEntry(
|
||||
workflow_id="content.success.daily",
|
||||
kind="daily",
|
||||
at="08:00",
|
||||
at_times=("08:00", "16:00", "22:00"),
|
||||
)
|
||||
timezone_info = ZoneInfo("Asia/Shanghai")
|
||||
target = datetime(2026, 8, 18).date()
|
||||
|
||||
all_slots = _schedule_slots(schedule, target, timezone_info)
|
||||
assert [slot.strftime("%H:%M") for slot in all_slots] == [
|
||||
"08:00",
|
||||
"16:00",
|
||||
"22:00",
|
||||
]
|
||||
|
||||
morning = _schedule_slots(
|
||||
schedule,
|
||||
target,
|
||||
timezone_info,
|
||||
now=datetime(2026, 8, 18, 9, 55, tzinfo=timezone_info),
|
||||
)
|
||||
assert [slot.strftime("%H:%M") for slot in morning] == ["08:00"]
|
||||
|
||||
|
||||
def test_daily_summary_does_not_flag_future_multi_slots_as_missing(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project_root = tmp_path / "project"
|
||||
data_root = tmp_path / "data"
|
||||
_write_multi_slot_catalog(project_root)
|
||||
|
||||
context = RunContext.create(
|
||||
"content.success.daily",
|
||||
"2026-08-18",
|
||||
now=datetime(2026, 8, 18, 0, 1, tzinfo=timezone.utc),
|
||||
random_suffix="abc123",
|
||||
)
|
||||
journal = RunJournal.create(DataLayout(data_root), context, mode="execute")
|
||||
journal.start_step("collect.attempt-1", attempt=1)
|
||||
journal.finish_step("collect.attempt-1", status="success", exit_code=0)
|
||||
journal.finalize("success")
|
||||
RunIndex(data_root).index_journal(journal)
|
||||
|
||||
log_root = data_root / "logs" / "scheduler"
|
||||
log_root.mkdir(parents=True)
|
||||
(log_root / "content.success.daily-20260818T080000+0800.log").write_text(
|
||||
"completed",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
analyzer = FakeAnalyzer()
|
||||
builder = DailySummaryBuilder(
|
||||
project_root=project_root,
|
||||
data_root=data_root,
|
||||
analyzer=analyzer,
|
||||
workflow_name=lambda workflow_id: "成功日报",
|
||||
)
|
||||
report = builder.build(
|
||||
datetime(2026, 8, 18).date(),
|
||||
now=datetime(2026, 8, 18, 9, 55, tzinfo=ZoneInfo("Asia/Shanghai")),
|
||||
)
|
||||
|
||||
item = report["workflows"][0]
|
||||
assert item["workflow_id"] == "content.success.daily"
|
||||
assert item["expected_slots"] == ["2026-08-18T08:00:00+08:00"]
|
||||
assert item["missed_slots"] == 0
|
||||
assert item["status"] == "normal"
|
||||
assert "没有发现调度日志或运行记录" not in item["execution_result"]
|
||||
Reference in New Issue
Block a user