feat: expand module workflows, dynamic config, notifications and console API

This commit is contained in:
wangyunlong committed 2026-09-04 09:49:37 +08:00
1 parent 8df5266abb
commit b124d757b0
309 files changed
+89357 -6231

No files matched your search

+37 -10
View File
@@ -18,6 +18,9 @@ from zoneinfo import ZoneInfo
from gyxx_flow.catalog import ScheduleEntry, WorkflowCatalog
from gyxx_flow.core.artifacts import atomic_write_json
_TMALL_BAIBU_WORKFLOW_ID = "product.tmall_baibu_apply"
_TMALL_BAIBU_SCHEDULE_IMPORT_MODE = "without_hyperlinks"
def latest_due_slot(schedule: ScheduleEntry, now: datetime) -> datetime:
"""Return the latest scheduled wall-clock slot at or before ``now``."""
@@ -126,22 +129,36 @@ class SubprocessWorkflowLauncher:
**os.environ,
"GYXX_PROJECT_ROOT": str(self.project_root),
"GYXX_DATA_ROOT": str(self.data_root),
# Keep the concrete scheduled slot available to the workflow.
# A workflow may have more than one daily slot; the business date
# alone cannot distinguish the initial collection from a later
# same-day retry.
"GYXX_SCHEDULE_SLOT": slot.isoformat(),
"GYXX_SCHEDULE_TIME": slot.strftime("%H:%M"),
}
kwargs: dict[str, object] = {}
if os.name == "nt":
kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
command = [
str(self.python_executable),
"-m",
"gyxx_flow",
"run",
workflow_id,
"--date",
business_date.isoformat(),
]
if workflow_id == _TMALL_BAIBU_WORKFLOW_ID:
command.extend(
[
"--tmall-baibu-import-mode",
_TMALL_BAIBU_SCHEDULE_IMPORT_MODE,
]
)
command.append("--execute")
try:
process = subprocess.Popen(
[
str(self.python_executable),
"-m",
"gyxx_flow",
"run",
workflow_id,
"--date",
business_date.isoformat(),
"--execute",
],
command,
cwd=self.project_root,
env=environment,
stdin=subprocess.DEVNULL,
@@ -180,6 +197,7 @@ class PythonScheduler:
misfire_grace_seconds: int = 6 * 60 * 60,
dry_run: bool = False,
catalog_loader: Callable[[], WorkflowCatalog] | None = None,
maintenance: Callable[[], object] | None = None,
) -> None:
if misfire_grace_seconds < 0:
raise ValueError("misfire grace must be non-negative")
@@ -191,6 +209,7 @@ class PythonScheduler:
self.misfire_grace_seconds = misfire_grace_seconds
self.dry_run = dry_run
self._catalog_loader = catalog_loader
self._maintenance = maintenance
self._state = self._load_state()
self._active: dict[str, _ActiveRun] = {}
@@ -238,6 +257,14 @@ class PythonScheduler:
self._reload_catalog()
current = (now or datetime.now(self.timezone)).astimezone(self.timezone)
self._reap(current)
if not self.dry_run and self._maintenance is not None:
try:
self._maintenance()
except Exception:
# Maintenance must never prevent scheduled business workflows
# from being evaluated. The maintenance owner records its own
# redacted status where appropriate.
pass
due: list[tuple[str, datetime, date]] = []
state_changed = False
for workflow in self.catalog.scheduled_workflows():