wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete production workflow migration

Browse Source
This commit is contained in:
wangyunlong
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gyxx_flow/cli.py
+282 -90
View File
@@ -5,13 +5,17 @@ from __future__ import annotations
import argparse
import hashlib
import json
import os
import signal
import sys
import threading
from collections.abc import Sequence
from contextlib import suppress
from datetime import date, datetime, timedelta, timezone
from datetime import date, timedelta
from pathlib import Path
from typing import TextIO
from zoneinfo import ZoneInfo
from dotenv import dotenv_values
from gyxx_flow.acceptance import build_acceptance_report
from gyxx_flow.adapters.native import DeferredModuleCommandStep
@@ -24,15 +28,14 @@ from gyxx_flow.core.locks import LockManager
from gyxx_flow.core.records import RunJournal
from gyxx_flow.diagnostics import run_doctor
from gyxx_flow.modules import create_default_registry as create_module_registry
from gyxx_flow.ops import EffectLedger, RunIndex
from gyxx_flow.scheduler import (
PowerShellCurrentTaskProvider,
SchedulerConfig,
build_schedule_plan,
detect_schedule_drift,
write_schedule_plan_bundle,
from gyxx_flow.ops import EffectLedger, EffectStateAmbiguous, RunIndex
from gyxx_flow.scheduler_service import (
PythonScheduler,
SchedulerInstanceLock,
SubprocessWorkflowLauncher,
)
from gyxx_flow.script_catalog import ScriptCatalog, ScriptCatalogError
from gyxx_flow.source_sync.cli import add_sources_parser, run_sources_command
from gyxx_flow.workflow.engine import WorkflowEngine, WorkflowRunResult
from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
from gyxx_flow.workflow.registry import WorkflowRegistry, WorkflowRegistryError
@@ -64,11 +67,13 @@ def build_parser() -> argparse.ArgumentParser:
)
commands = parser.add_subparsers(dest="command", required=True)
list_parser = commands.add_parser("list", help="list catalog workflows")
add_sources_parser(commands)
list_parser = commands.add_parser("list", help="list scheduled workflows")
list_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
scripts_parser = commands.add_parser(
"scripts", help="discover and manually run project-owned module scripts"
"scripts", help="list and manually run explicitly registered commands"
)
script_commands = scripts_parser.add_subparsers(dest="scripts_command", required=True)
scripts_list_parser = script_commands.add_parser("list", help="list runnable scripts")
@@ -76,9 +81,9 @@ def build_parser() -> argparse.ArgumentParser:
scripts_list_parser.add_argument("--json", action="store_true")
scripts_run_parser = script_commands.add_parser("run", help="run one local script")
scripts_run_parser.add_argument("script_id")
scripts_run_date = scripts_run_parser.add_mutually_exclusive_group(required=True)
scripts_run_date.add_argument("--date", help="business date (YYYY-MM-DD)")
scripts_run_date.add_argument("--scheduled", action="store_true")
scripts_run_parser.add_argument(
"--date", required=True, help="business date (YYYY-MM-DD)"
)
scripts_run_parser.add_argument("--execute", action="store_true")
scripts_run_parser.add_argument("--shadow", action="store_true")
scripts_run_parser.add_argument(
@@ -89,14 +94,35 @@ def build_parser() -> argparse.ArgumentParser:
help="one argument passed to the script; repeat for multiple arguments",
)
effects_parser = commands.add_parser(
"effects", help="inspect or reconcile guarded production effects"
)
effect_commands = effects_parser.add_subparsers(
dest="effects_command", required=True
)
reconcile_parser = effect_commands.add_parser(
"reconcile", help="resolve one verified ambiguous effect receipt"
)
reconcile_parser.add_argument("workflow_id")
reconcile_parser.add_argument("--date", required=True, dest="business_date")
reconcile_parser.add_argument("--step", required=True, dest="step_id")
reconcile_parser.add_argument("--expected-run-id", required=True)
reconcile_parser.add_argument(
"--action", required=True, choices=("retry", "applied")
)
reconcile_parser.add_argument("--operator", required=True)
reconcile_parser.add_argument("--reason", required=True)
reconcile_parser.add_argument("--evidence", required=True)
reconcile_parser.add_argument(
"--execute",
action="store_true",
help="confirm the audited receipt mutation",
)
run_parser = commands.add_parser("run", help="run one workflow business date")
_add_execution_arguments(run_parser)
run_date = run_parser.add_mutually_exclusive_group(required=True)
run_date.add_argument("--date", help="business date (YYYY-MM-DD)")
run_date.add_argument(
"--scheduled",
action="store_true",
help="execute for today's Asia/Shanghai business date",
run_parser.add_argument(
"--date", required=True, help="business date (YYYY-MM-DD)"
)
backfill_parser = commands.add_parser(
@@ -110,16 +136,42 @@ def build_parser() -> argparse.ArgumentParser:
"--to", dest="to_date", required=True, help="last business date"
)
schedule_parser = commands.add_parser("schedule", help="plan managed schedules")
schedule_parser = commands.add_parser(
"schedule", help="run the project-owned resident scheduler"
)
schedule_commands = schedule_parser.add_subparsers(
dest="schedule_command", required=True
)
plan_parser = schedule_commands.add_parser(
"plan", help="write an inert Task Scheduler bundle and drift report"
run_schedule_parser = schedule_commands.add_parser(
"run", help="run the cross-platform resident Python scheduler"
)
run_schedule_parser.add_argument("--python-executable", type=Path)
run_schedule_parser.add_argument("--poll-seconds", type=float, default=15.0)
run_schedule_parser.add_argument("--misfire-grace-seconds", type=int, default=21600)
run_schedule_parser.add_argument("--shutdown-timeout-seconds", type=float, default=300.0)
run_schedule_parser.add_argument("--once", action="store_true")
run_schedule_parser.add_argument("--dry-run", action="store_true")
run_schedule_parser.add_argument(
"--env-file",
action="append",
default=[],
type=Path,
help="explicit runtime environment file; repeatable, existing process values win",
)
schedule_commands.add_parser("status", help="show persisted Python scheduler state")
console_parser = commands.add_parser(
"console", help="serve the workflow operator web console"
)
console_parser.add_argument("--host", default="127.0.0.1")
console_parser.add_argument("--port", type=int, default=8765)
console_parser.add_argument(
"--env-file",
action="append",
default=[],
type=Path,
help="explicit runtime environment file; repeatable, existing process values win",
)
plan_parser.add_argument("--output", type=Path)
plan_parser.add_argument("--start-date", help="trigger boundary date (YYYY-MM-DD)")
plan_parser.add_argument("--python-executable", type=Path, required=True)
doctor_parser = commands.add_parser("doctor", help="run environment preflight checks")
doctor_parser.add_argument("--json", action="store_true")
@@ -167,7 +219,6 @@ def main(
settings: Settings | None = None,
stdout: TextIO | None = None,
stderr: TextIO | None = None,
current_task_provider: PowerShellCurrentTaskProvider | None = None,
) -> int:
"""Execute a CLI command and return a stable process exit code."""
@@ -179,10 +230,19 @@ def main(
except SystemExit as exc:
return int(exc.code)
resolved_settings = settings or Settings.from_env()
try:
_load_runtime_environment_files(arguments)
resolved_settings = settings or Settings.from_env()
if arguments.command == "doctor":
return _doctor_command(arguments, resolved_settings, output)
if arguments.command == "console":
return _console_command(arguments, resolved_settings, output)
if arguments.command == "sources":
return run_sources_command(
arguments,
project_root=resolved_settings.project_root,
output=output,
)
if (
arguments.command == "acceptance"
and arguments.acceptance_command == "status"
@@ -192,6 +252,13 @@ def main(
return _list_scripts(arguments, output)
if arguments.command == "scripts" and arguments.scripts_command == "run":
return _run_script(arguments, resolved_settings, output)
if (
arguments.command == "effects"
and arguments.effects_command == "reconcile"
):
return _effects_reconcile_command(
arguments, resolved_settings, output
)
resolved_registry = registry or build_default_registry(resolved_settings)
if arguments.command == "list":
return _list_workflows(arguments, resolved_registry, output)
@@ -199,14 +266,10 @@ def main(
return _run_command(arguments, resolved_registry, resolved_settings, output)
if arguments.command == "backfill":
return _backfill_command(arguments, resolved_registry, resolved_settings, output)
if arguments.command == "schedule" and arguments.schedule_command == "plan":
return _schedule_plan_command(
arguments,
resolved_registry.catalog,
resolved_settings,
output,
current_task_provider=current_task_provider,
)
if arguments.command == "schedule" and arguments.schedule_command == "run":
return _schedule_run_command(arguments, resolved_registry.catalog, resolved_settings, output)
if arguments.command == "schedule" and arguments.schedule_command == "status":
return _schedule_status_command(resolved_registry.catalog, resolved_settings, output)
raise CliConfigurationError(f"unsupported command: {arguments.command}")
except (
CatalogError,
@@ -222,6 +285,26 @@ def main(
return EXIT_RUNTIME
def _load_runtime_environment_files(arguments: argparse.Namespace) -> None:
"""Load only explicitly named service environment files without leaking values."""
raw_paths = getattr(arguments, "env_file", ()) or ()
for raw_path in raw_paths:
path = Path(raw_path).expanduser().resolve()
if not path.is_file():
raise CliConfigurationError(f"runtime environment file not found: {path}")
try:
values = dotenv_values(path, encoding="utf-8")
except (OSError, UnicodeError) as exc:
raise CliConfigurationError(
f"cannot read runtime environment file: {path}"
) from exc
for key, value in values.items():
if not isinstance(key, str) or not key or value is None:
continue
os.environ.setdefault(key, value)
def _doctor_command(
arguments: argparse.Namespace, settings: Settings, output: TextIO
) -> int:
@@ -235,6 +318,48 @@ def _doctor_command(
return EXIT_SUCCESS if report.is_healthy else EXIT_RUNTIME
def _effects_reconcile_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
if not arguments.execute:
raise CliConfigurationError("effect reconciliation requires --execute")
try:
result = EffectLedger(settings.data_root).reconcile(
workflow_id=arguments.workflow_id,
business_date=arguments.business_date,
step_id=arguments.step_id,
expected_run_id=arguments.expected_run_id,
action=arguments.action,
operator=arguments.operator,
reason=arguments.reason,
evidence=arguments.evidence,
)
except EffectStateAmbiguous as exc:
raise CliConfigurationError(str(exc)) from exc
_write_json(output, result)
return EXIT_SUCCESS
def _console_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
from gyxx_flow.console import serve_console
try:
return serve_console(
settings,
host=arguments.host,
port=arguments.port,
output=output,
)
except OSError as exc:
raise CliRuntimeError("cannot start workflow console") from exc
def _acceptance_status_command(
arguments: argparse.Namespace, settings: Settings, output: TextIO
) -> int:
@@ -264,14 +389,19 @@ def _list_workflows(
"module": entry.module,
"registered": registry.is_registered(entry.workflow_id),
"trigger": entry.trigger,
"enabled": registry.catalog.schedule_for(entry.workflow_id).enabled,
}
for entry in registry.catalog.workflows
for entry in registry.catalog.scheduled_workflows()
]
if arguments.json:
_write_json(output, rows)
else:
for row in rows:
state = "ready" if row["registered"] else "not-registered"
state = (
"not-registered"
if not row["registered"]
else "ready" if row["enabled"] else "disabled"
)
output.write(
f"{row['id']}\t{row['module']}\t{row['trigger']}\t{state}\n"
)
@@ -279,13 +409,16 @@ def _list_workflows(
def _list_scripts(arguments: argparse.Namespace, output: TextIO) -> int:
catalog = ScriptCatalog.discover_default()
catalog = ScriptCatalog.load_default()
rows = [
{
"id": script.script_id,
"id": script.command_id,
"command_id": script.command_id,
"legacy_script_id": script.legacy_script_id,
"module": script.module,
"entry": script.entry,
"kind": script.kind,
"default_args": list(script.default_args),
}
for script in catalog.scripts
if arguments.module is None or script.module == arguments.module
@@ -305,10 +438,19 @@ def _run_script(
settings: Settings,
output: TextIO,
) -> int:
script = ScriptCatalog.discover_default().get(arguments.script_id)
suffix = hashlib.sha256(script.script_id.encode("utf-8")).hexdigest()[:12]
script = ScriptCatalog.load_default().get(arguments.script_id)
business_date = _parse_date(arguments.date)
script_args = (
*_render_script_default_args(script.default_args, business_date),
*arguments.script_args,
)
invocation_identity = json.dumps(
[script.command_id, *script_args],
ensure_ascii=False,
separators=(",", ":"),
)
suffix = hashlib.sha256(invocation_identity.encode("utf-8")).hexdigest()[:12]
workflow_id = f"script.{script.module}.{suffix}"
script_args = tuple(arguments.script_args)
entry = WorkflowEntry(
workflow_id=workflow_id,
module=script.module,
@@ -329,19 +471,28 @@ def _run_script(
),
),
)
business_date = _today_shanghai() if arguments.scheduled else _parse_date(arguments.date)
exit_code, payload = _execute_once(
workflow,
business_date=business_date,
shadow=arguments.shadow,
dry_run=not (arguments.execute or arguments.scheduled),
dry_run=not arguments.execute,
settings=settings,
)
payload["script_id"] = script.script_id
payload["command_id"] = script.command_id
payload["script_id"] = script.legacy_script_id
_write_json(output, payload)
return exit_code
def _render_script_default_args(
arguments: tuple[str, ...], business_date: date
) -> tuple[str, ...]:
return tuple(
argument.replace("{business_date}", business_date.isoformat())
for argument in arguments
)
def _run_command(
arguments: argparse.Namespace,
registry: WorkflowRegistry,
@@ -350,8 +501,8 @@ def _run_command(
) -> int:
registered = registry.resolve(arguments.workflow_id)
workflow = _select_workflow(registered.definition, arguments)
business_date = _today_shanghai() if arguments.scheduled else _parse_date(arguments.date)
dry_run = not (arguments.execute or arguments.scheduled)
business_date = _parse_date(arguments.date)
dry_run = not arguments.execute
exit_code, payload = _execute_once(
workflow,
business_date=business_date,
@@ -363,51 +514,91 @@ def _run_command(
return exit_code
def _schedule_plan_command(
def _python_scheduler(
catalog: WorkflowCatalog,
settings: Settings,
*,
python_executable: Path | None = None,
misfire_grace_seconds: int = 21600,
dry_run: bool = False,
) -> PythonScheduler:
launcher = SubprocessWorkflowLauncher(
settings.project_root, settings.data_root, python_executable=python_executable
)
return PythonScheduler(
catalog,
settings.data_root,
launcher=launcher,
misfire_grace_seconds=misfire_grace_seconds,
dry_run=dry_run,
catalog_loader=lambda: WorkflowCatalog.load(
settings.project_root / "config"
),
)
def _schedule_run_command(
arguments: argparse.Namespace,
catalog: WorkflowCatalog,
settings: Settings,
output: TextIO,
*,
current_task_provider: PowerShellCurrentTaskProvider | None,
) -> int:
start_date = (
_parse_date(arguments.start_date)
if arguments.start_date
else _today_shanghai()
)
destination = arguments.output
if destination is None:
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
destination = settings.data_root / "evidence" / "schedule-plans" / timestamp
config = SchedulerConfig(
if arguments.poll_seconds <= 0:
raise CliConfigurationError("poll seconds must be positive")
if arguments.shutdown_timeout_seconds < 0:
raise CliConfigurationError("shutdown timeout must be non-negative")
scheduler = _python_scheduler(
catalog,
settings,
python_executable=arguments.python_executable,
project_root=settings.project_root,
misfire_grace_seconds=arguments.misfire_grace_seconds,
dry_run=arguments.dry_run,
)
plan = build_schedule_plan(catalog, config, start_date=start_date)
lock_path = settings.data_root / "state" / "scheduler" / "service.lock"
try:
bundle = write_schedule_plan_bundle(destination, plan)
except FileExistsError as exc:
raise CliConfigurationError(
f"schedule plan destination already exists: {destination}"
) from exc
provider = current_task_provider or PowerShellCurrentTaskProvider()
try:
current = provider.current_tasks(config.task_path)
except Exception as exc:
raise CliRuntimeError("cannot read current managed scheduled tasks") from exc
drift = detect_schedule_drift(plan, current)
atomic_write_json(bundle / "drift.json", drift.as_dict())
_write_json(
output,
{
"path": str(bundle),
"desired_count": len(plan.tasks),
"drift_count": len(drift.items),
"is_clean": drift.is_clean,
"applied": False,
},
)
with SchedulerInstanceLock(lock_path):
first = scheduler.tick()
if arguments.once or arguments.dry_run:
if not arguments.dry_run:
scheduler.wait_for_active(arguments.shutdown_timeout_seconds)
_write_json(output, {
"mode": "dry-run" if arguments.dry_run else "once",
"due_or_started": first,
"state": scheduler.status(),
})
return EXIT_SUCCESS
stop = threading.Event()
previous_handlers: dict[int, object] = {}
def request_stop(_signum, _frame) -> None: # type: ignore[no-untyped-def]
stop.set()
for signum in (signal.SIGINT, signal.SIGTERM):
previous_handlers[signum] = signal.getsignal(signum)
signal.signal(signum, request_stop)
try:
while not stop.wait(arguments.poll_seconds):
scheduler.tick()
finally:
for signum, handler in previous_handlers.items():
signal.signal(signum, handler)
scheduler.wait_for_active(arguments.shutdown_timeout_seconds)
_write_json(output, {"mode": "service", "state": scheduler.status()})
return EXIT_SUCCESS
except RuntimeError as exc:
raise CliRuntimeError(str(exc)) from exc
def _schedule_status_command(
catalog: WorkflowCatalog, settings: Settings, output: TextIO
) -> int:
scheduler = _python_scheduler(catalog, settings)
_write_json(output, {
"timezone": catalog.timezone,
"scheduled_count": len(catalog.scheduled_workflows()),
"state": scheduler.status(),
})
return EXIT_SUCCESS
@@ -481,7 +672,12 @@ def _execute_once(
layout = DataLayout(settings.data_root)
journal: RunJournal | None = None
try:
journal = RunJournal.create(layout, context)
journal = RunJournal.create(
layout,
context,
mode="dry_run" if dry_run else "execute",
)
RunIndex(settings.data_root).index_journal(journal)
result = WorkflowEngine(
LockManager(settings.data_root / "state" / "locks"),
effect_ledger=EffectLedger(settings.data_root),
@@ -536,10 +732,6 @@ def _parse_date(value: str) -> date:
return parsed
def _today_shanghai() -> date:
return datetime.now(ZoneInfo("Asia/Shanghai")).date()
def _write_json(output: TextIO, payload: object) -> None:
json.dump(payload, output, ensure_ascii=False, sort_keys=True)
output.write("\n")