feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
1 parent
8df5266abb
commit
b124d757b0
309 files changed
+89357
-6231
No files matched your search
@@ -18,6 +18,9 @@ from zoneinfo import ZoneInfo
|
||||
from gyxx_flow.catalog import ScheduleEntry, WorkflowCatalog
|
||||
from gyxx_flow.core.artifacts import atomic_write_json
|
||||
|
||||
_TMALL_BAIBU_WORKFLOW_ID = "product.tmall_baibu_apply"
|
||||
_TMALL_BAIBU_SCHEDULE_IMPORT_MODE = "without_hyperlinks"
|
||||
|
||||
|
||||
def latest_due_slot(schedule: ScheduleEntry, now: datetime) -> datetime:
|
||||
"""Return the latest scheduled wall-clock slot at or before ``now``."""
|
||||
@@ -126,22 +129,36 @@ class SubprocessWorkflowLauncher:
|
||||
**os.environ,
|
||||
"GYXX_PROJECT_ROOT": str(self.project_root),
|
||||
"GYXX_DATA_ROOT": str(self.data_root),
|
||||
# Keep the concrete scheduled slot available to the workflow.
|
||||
# A workflow may have more than one daily slot; the business date
|
||||
# alone cannot distinguish the initial collection from a later
|
||||
# same-day retry.
|
||||
"GYXX_SCHEDULE_SLOT": slot.isoformat(),
|
||||
"GYXX_SCHEDULE_TIME": slot.strftime("%H:%M"),
|
||||
}
|
||||
kwargs: dict[str, object] = {}
|
||||
if os.name == "nt":
|
||||
kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
|
||||
command = [
|
||||
str(self.python_executable),
|
||||
"-m",
|
||||
"gyxx_flow",
|
||||
"run",
|
||||
workflow_id,
|
||||
"--date",
|
||||
business_date.isoformat(),
|
||||
]
|
||||
if workflow_id == _TMALL_BAIBU_WORKFLOW_ID:
|
||||
command.extend(
|
||||
[
|
||||
"--tmall-baibu-import-mode",
|
||||
_TMALL_BAIBU_SCHEDULE_IMPORT_MODE,
|
||||
]
|
||||
)
|
||||
command.append("--execute")
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
str(self.python_executable),
|
||||
"-m",
|
||||
"gyxx_flow",
|
||||
"run",
|
||||
workflow_id,
|
||||
"--date",
|
||||
business_date.isoformat(),
|
||||
"--execute",
|
||||
],
|
||||
command,
|
||||
cwd=self.project_root,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
@@ -180,6 +197,7 @@ class PythonScheduler:
|
||||
misfire_grace_seconds: int = 6 * 60 * 60,
|
||||
dry_run: bool = False,
|
||||
catalog_loader: Callable[[], WorkflowCatalog] | None = None,
|
||||
maintenance: Callable[[], object] | None = None,
|
||||
) -> None:
|
||||
if misfire_grace_seconds < 0:
|
||||
raise ValueError("misfire grace must be non-negative")
|
||||
@@ -191,6 +209,7 @@ class PythonScheduler:
|
||||
self.misfire_grace_seconds = misfire_grace_seconds
|
||||
self.dry_run = dry_run
|
||||
self._catalog_loader = catalog_loader
|
||||
self._maintenance = maintenance
|
||||
self._state = self._load_state()
|
||||
self._active: dict[str, _ActiveRun] = {}
|
||||
|
||||
@@ -238,6 +257,14 @@ class PythonScheduler:
|
||||
self._reload_catalog()
|
||||
current = (now or datetime.now(self.timezone)).astimezone(self.timezone)
|
||||
self._reap(current)
|
||||
if not self.dry_run and self._maintenance is not None:
|
||||
try:
|
||||
self._maintenance()
|
||||
except Exception:
|
||||
# Maintenance must never prevent scheduled business workflows
|
||||
# from being evaluated. The maintenance owner records its own
|
||||
# redacted status where appropriate.
|
||||
pass
|
||||
due: list[tuple[str, datetime, date]] = []
|
||||
state_changed = False
|
||||
for workflow in self.catalog.scheduled_workflows():
|
||||
|
||||
Reference in new issue
Block a user