feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,545 @@
|
||||
"""Operator-safe command-line entry point for GYXX Flow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from contextlib import suppress
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import TextIO
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from gyxx_flow.acceptance import build_acceptance_report
|
||||
from gyxx_flow.adapters.native import DeferredModuleCommandStep
|
||||
from gyxx_flow.catalog import CatalogError, WorkflowCatalog, WorkflowEntry
|
||||
from gyxx_flow.core.artifacts import atomic_write_json
|
||||
from gyxx_flow.core.config import Settings
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.core.layout import DataLayout
|
||||
from gyxx_flow.core.locks import LockManager
|
||||
from gyxx_flow.core.records import RunJournal
|
||||
from gyxx_flow.diagnostics import run_doctor
|
||||
from gyxx_flow.modules import create_default_registry as create_module_registry
|
||||
from gyxx_flow.ops import EffectLedger, RunIndex
|
||||
from gyxx_flow.scheduler import (
|
||||
PowerShellCurrentTaskProvider,
|
||||
SchedulerConfig,
|
||||
build_schedule_plan,
|
||||
detect_schedule_drift,
|
||||
write_schedule_plan_bundle,
|
||||
)
|
||||
from gyxx_flow.script_catalog import ScriptCatalog, ScriptCatalogError
|
||||
from gyxx_flow.workflow.engine import WorkflowEngine, WorkflowRunResult
|
||||
from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
|
||||
from gyxx_flow.workflow.registry import WorkflowRegistry, WorkflowRegistryError
|
||||
from gyxx_flow.workflow.selection import rerun_step, resume_from
|
||||
|
||||
EXIT_SUCCESS = 0
|
||||
EXIT_WORKFLOW_FAILED = 1
|
||||
EXIT_USAGE = 2
|
||||
EXIT_CONFIGURATION = 3
|
||||
EXIT_RUNTIME = 4
|
||||
EXIT_ACCEPTANCE_INCOMPLETE = 1
|
||||
MAX_BACKFILL_DAYS = 366
|
||||
|
||||
|
||||
class CliConfigurationError(ValueError):
|
||||
"""Safe operator-facing configuration or input error."""
|
||||
|
||||
|
||||
class CliRuntimeError(RuntimeError):
|
||||
"""Sanitized runtime failure."""
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Build the CLI parser without initializing runtime resources."""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="gyxx",
|
||||
description="GYXX Flow workflow orchestration",
|
||||
)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
list_parser = commands.add_parser("list", help="list catalog workflows")
|
||||
list_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
|
||||
|
||||
scripts_parser = commands.add_parser(
|
||||
"scripts", help="discover and manually run project-owned module scripts"
|
||||
)
|
||||
script_commands = scripts_parser.add_subparsers(dest="scripts_command", required=True)
|
||||
scripts_list_parser = script_commands.add_parser("list", help="list runnable scripts")
|
||||
scripts_list_parser.add_argument("--module")
|
||||
scripts_list_parser.add_argument("--json", action="store_true")
|
||||
scripts_run_parser = script_commands.add_parser("run", help="run one local script")
|
||||
scripts_run_parser.add_argument("script_id")
|
||||
scripts_run_date = scripts_run_parser.add_mutually_exclusive_group(required=True)
|
||||
scripts_run_date.add_argument("--date", help="business date (YYYY-MM-DD)")
|
||||
scripts_run_date.add_argument("--scheduled", action="store_true")
|
||||
scripts_run_parser.add_argument("--execute", action="store_true")
|
||||
scripts_run_parser.add_argument("--shadow", action="store_true")
|
||||
scripts_run_parser.add_argument(
|
||||
"--arg",
|
||||
dest="script_args",
|
||||
action="append",
|
||||
default=[],
|
||||
help="one argument passed to the script; repeat for multiple arguments",
|
||||
)
|
||||
|
||||
run_parser = commands.add_parser("run", help="run one workflow business date")
|
||||
_add_execution_arguments(run_parser)
|
||||
run_date = run_parser.add_mutually_exclusive_group(required=True)
|
||||
run_date.add_argument("--date", help="business date (YYYY-MM-DD)")
|
||||
run_date.add_argument(
|
||||
"--scheduled",
|
||||
action="store_true",
|
||||
help="execute for today's Asia/Shanghai business date",
|
||||
)
|
||||
|
||||
backfill_parser = commands.add_parser(
|
||||
"backfill", help="run an inclusive business-date range"
|
||||
)
|
||||
_add_execution_arguments(backfill_parser)
|
||||
backfill_parser.add_argument(
|
||||
"--from", dest="from_date", required=True, help="first business date"
|
||||
)
|
||||
backfill_parser.add_argument(
|
||||
"--to", dest="to_date", required=True, help="last business date"
|
||||
)
|
||||
|
||||
schedule_parser = commands.add_parser("schedule", help="plan managed schedules")
|
||||
schedule_commands = schedule_parser.add_subparsers(
|
||||
dest="schedule_command", required=True
|
||||
)
|
||||
plan_parser = schedule_commands.add_parser(
|
||||
"plan", help="write an inert Task Scheduler bundle and drift report"
|
||||
)
|
||||
plan_parser.add_argument("--output", type=Path)
|
||||
plan_parser.add_argument("--start-date", help="trigger boundary date (YYYY-MM-DD)")
|
||||
plan_parser.add_argument("--python-executable", type=Path, required=True)
|
||||
|
||||
doctor_parser = commands.add_parser("doctor", help="run environment preflight checks")
|
||||
doctor_parser.add_argument("--json", action="store_true")
|
||||
|
||||
acceptance_parser = commands.add_parser(
|
||||
"acceptance", help="inspect acceptance evidence"
|
||||
)
|
||||
acceptance_commands = acceptance_parser.add_subparsers(
|
||||
dest="acceptance_command", required=True
|
||||
)
|
||||
status_parser = acceptance_commands.add_parser("status")
|
||||
status_parser.add_argument("--json", action="store_true")
|
||||
status_parser.add_argument("--output", type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
def _add_execution_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("workflow_id")
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="execute registered steps; omitted means no-side-effect dry-run",
|
||||
)
|
||||
parser.add_argument("--shadow", action="store_true")
|
||||
recovery = parser.add_mutually_exclusive_group()
|
||||
recovery.add_argument("--resume-from", metavar="STEP_ID")
|
||||
recovery.add_argument("--rerun-step", metavar="STEP_ID")
|
||||
|
||||
|
||||
def build_default_registry(settings: Settings) -> WorkflowRegistry:
|
||||
"""Compose only workflows explicitly implemented by migrated modules."""
|
||||
|
||||
catalog = WorkflowCatalog.load(settings.project_root / "config")
|
||||
registry = WorkflowRegistry(catalog)
|
||||
modules = create_module_registry(catalog=catalog)
|
||||
for workflow_id in modules.workflow_ids:
|
||||
registry.register(modules.workflow(workflow_id))
|
||||
return registry
|
||||
|
||||
|
||||
def main(
|
||||
argv: Sequence[str] | None = None,
|
||||
*,
|
||||
registry: WorkflowRegistry | None = None,
|
||||
settings: Settings | None = None,
|
||||
stdout: TextIO | None = None,
|
||||
stderr: TextIO | None = None,
|
||||
current_task_provider: PowerShellCurrentTaskProvider | None = None,
|
||||
) -> int:
|
||||
"""Execute a CLI command and return a stable process exit code."""
|
||||
|
||||
output = stdout or sys.stdout
|
||||
errors = stderr or sys.stderr
|
||||
parser = build_parser()
|
||||
try:
|
||||
arguments = parser.parse_args(argv)
|
||||
except SystemExit as exc:
|
||||
return int(exc.code)
|
||||
|
||||
resolved_settings = settings or Settings.from_env()
|
||||
try:
|
||||
if arguments.command == "doctor":
|
||||
return _doctor_command(arguments, resolved_settings, output)
|
||||
if (
|
||||
arguments.command == "acceptance"
|
||||
and arguments.acceptance_command == "status"
|
||||
):
|
||||
return _acceptance_status_command(arguments, resolved_settings, output)
|
||||
if arguments.command == "scripts" and arguments.scripts_command == "list":
|
||||
return _list_scripts(arguments, output)
|
||||
if arguments.command == "scripts" and arguments.scripts_command == "run":
|
||||
return _run_script(arguments, resolved_settings, output)
|
||||
resolved_registry = registry or build_default_registry(resolved_settings)
|
||||
if arguments.command == "list":
|
||||
return _list_workflows(arguments, resolved_registry, output)
|
||||
if arguments.command == "run":
|
||||
return _run_command(arguments, resolved_registry, resolved_settings, output)
|
||||
if arguments.command == "backfill":
|
||||
return _backfill_command(arguments, resolved_registry, resolved_settings, output)
|
||||
if arguments.command == "schedule" and arguments.schedule_command == "plan":
|
||||
return _schedule_plan_command(
|
||||
arguments,
|
||||
resolved_registry.catalog,
|
||||
resolved_settings,
|
||||
output,
|
||||
current_task_provider=current_task_provider,
|
||||
)
|
||||
raise CliConfigurationError(f"unsupported command: {arguments.command}")
|
||||
except (
|
||||
CatalogError,
|
||||
ScriptCatalogError,
|
||||
WorkflowRegistryError,
|
||||
CliConfigurationError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
errors.write(f"error: {exc}\n")
|
||||
return EXIT_CONFIGURATION
|
||||
except CliRuntimeError as exc:
|
||||
errors.write(f"error: {exc}\n")
|
||||
return EXIT_RUNTIME
|
||||
|
||||
|
||||
def _doctor_command(
|
||||
arguments: argparse.Namespace, settings: Settings, output: TextIO
|
||||
) -> int:
|
||||
report = run_doctor(settings)
|
||||
if arguments.json:
|
||||
_write_json(output, report.as_dict())
|
||||
else:
|
||||
for check in report.checks:
|
||||
marker = "PASS" if check.passed else "FAIL"
|
||||
output.write(f"{marker}\t{check.name}\t{check.message}\n")
|
||||
return EXIT_SUCCESS if report.is_healthy else EXIT_RUNTIME
|
||||
|
||||
|
||||
def _acceptance_status_command(
|
||||
arguments: argparse.Namespace, settings: Settings, output: TextIO
|
||||
) -> int:
|
||||
report = build_acceptance_report(settings)
|
||||
payload = report.as_dict()
|
||||
if arguments.output is not None:
|
||||
atomic_write_json(arguments.output, payload)
|
||||
if arguments.json:
|
||||
_write_json(output, payload)
|
||||
else:
|
||||
summary = payload["summary"]
|
||||
output.write(
|
||||
f"acceptance: {summary['completed']}/{summary['total']} complete; "
|
||||
f"pending={summary['pending']}\n"
|
||||
)
|
||||
return EXIT_SUCCESS if report.is_complete else EXIT_ACCEPTANCE_INCOMPLETE
|
||||
|
||||
|
||||
def _list_workflows(
|
||||
arguments: argparse.Namespace,
|
||||
registry: WorkflowRegistry,
|
||||
output: TextIO,
|
||||
) -> int:
|
||||
rows = [
|
||||
{
|
||||
"id": entry.workflow_id,
|
||||
"module": entry.module,
|
||||
"registered": registry.is_registered(entry.workflow_id),
|
||||
"trigger": entry.trigger,
|
||||
}
|
||||
for entry in registry.catalog.workflows
|
||||
]
|
||||
if arguments.json:
|
||||
_write_json(output, rows)
|
||||
else:
|
||||
for row in rows:
|
||||
state = "ready" if row["registered"] else "not-registered"
|
||||
output.write(
|
||||
f"{row['id']}\t{row['module']}\t{row['trigger']}\t{state}\n"
|
||||
)
|
||||
return EXIT_SUCCESS
|
||||
|
||||
|
||||
def _list_scripts(arguments: argparse.Namespace, output: TextIO) -> int:
|
||||
catalog = ScriptCatalog.discover_default()
|
||||
rows = [
|
||||
{
|
||||
"id": script.script_id,
|
||||
"module": script.module,
|
||||
"entry": script.entry,
|
||||
"kind": script.kind,
|
||||
}
|
||||
for script in catalog.scripts
|
||||
if arguments.module is None or script.module == arguments.module
|
||||
]
|
||||
if arguments.module is not None and not rows:
|
||||
raise CliConfigurationError(f"unknown module or no runnable scripts: {arguments.module}")
|
||||
if arguments.json:
|
||||
_write_json(output, rows)
|
||||
else:
|
||||
for row in rows:
|
||||
output.write(f"{row['id']}\t{row['kind']}\n")
|
||||
return EXIT_SUCCESS
|
||||
|
||||
|
||||
def _run_script(
|
||||
arguments: argparse.Namespace,
|
||||
settings: Settings,
|
||||
output: TextIO,
|
||||
) -> int:
|
||||
script = ScriptCatalog.discover_default().get(arguments.script_id)
|
||||
suffix = hashlib.sha256(script.script_id.encode("utf-8")).hexdigest()[:12]
|
||||
workflow_id = f"script.{script.module}.{suffix}"
|
||||
script_args = tuple(arguments.script_args)
|
||||
entry = WorkflowEntry(
|
||||
workflow_id=workflow_id,
|
||||
module=script.module,
|
||||
trigger="manual",
|
||||
entry=script.entry,
|
||||
args=script_args,
|
||||
)
|
||||
workflow = WorkflowDefinition(
|
||||
workflow_id,
|
||||
(
|
||||
StepDefinition(
|
||||
"module_script",
|
||||
DeferredModuleCommandStep(entry),
|
||||
timeout_seconds=6 * 60 * 60,
|
||||
max_attempts=1,
|
||||
resources=(f"module:{script.module}",),
|
||||
production_sink=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
business_date = _today_shanghai() if arguments.scheduled else _parse_date(arguments.date)
|
||||
exit_code, payload = _execute_once(
|
||||
workflow,
|
||||
business_date=business_date,
|
||||
shadow=arguments.shadow,
|
||||
dry_run=not (arguments.execute or arguments.scheduled),
|
||||
settings=settings,
|
||||
)
|
||||
payload["script_id"] = script.script_id
|
||||
_write_json(output, payload)
|
||||
return exit_code
|
||||
|
||||
|
||||
def _run_command(
|
||||
arguments: argparse.Namespace,
|
||||
registry: WorkflowRegistry,
|
||||
settings: Settings,
|
||||
output: TextIO,
|
||||
) -> int:
|
||||
registered = registry.resolve(arguments.workflow_id)
|
||||
workflow = _select_workflow(registered.definition, arguments)
|
||||
business_date = _today_shanghai() if arguments.scheduled else _parse_date(arguments.date)
|
||||
dry_run = not (arguments.execute or arguments.scheduled)
|
||||
exit_code, payload = _execute_once(
|
||||
workflow,
|
||||
business_date=business_date,
|
||||
shadow=arguments.shadow,
|
||||
dry_run=dry_run,
|
||||
settings=settings,
|
||||
)
|
||||
_write_json(output, payload)
|
||||
return exit_code
|
||||
|
||||
|
||||
def _schedule_plan_command(
|
||||
arguments: argparse.Namespace,
|
||||
catalog: WorkflowCatalog,
|
||||
settings: Settings,
|
||||
output: TextIO,
|
||||
*,
|
||||
current_task_provider: PowerShellCurrentTaskProvider | None,
|
||||
) -> int:
|
||||
start_date = (
|
||||
_parse_date(arguments.start_date)
|
||||
if arguments.start_date
|
||||
else _today_shanghai()
|
||||
)
|
||||
destination = arguments.output
|
||||
if destination is None:
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
destination = settings.data_root / "evidence" / "schedule-plans" / timestamp
|
||||
config = SchedulerConfig(
|
||||
python_executable=arguments.python_executable,
|
||||
project_root=settings.project_root,
|
||||
)
|
||||
plan = build_schedule_plan(catalog, config, start_date=start_date)
|
||||
try:
|
||||
bundle = write_schedule_plan_bundle(destination, plan)
|
||||
except FileExistsError as exc:
|
||||
raise CliConfigurationError(
|
||||
f"schedule plan destination already exists: {destination}"
|
||||
) from exc
|
||||
provider = current_task_provider or PowerShellCurrentTaskProvider()
|
||||
try:
|
||||
current = provider.current_tasks(config.task_path)
|
||||
except Exception as exc:
|
||||
raise CliRuntimeError("cannot read current managed scheduled tasks") from exc
|
||||
drift = detect_schedule_drift(plan, current)
|
||||
atomic_write_json(bundle / "drift.json", drift.as_dict())
|
||||
_write_json(
|
||||
output,
|
||||
{
|
||||
"path": str(bundle),
|
||||
"desired_count": len(plan.tasks),
|
||||
"drift_count": len(drift.items),
|
||||
"is_clean": drift.is_clean,
|
||||
"applied": False,
|
||||
},
|
||||
)
|
||||
return EXIT_SUCCESS
|
||||
|
||||
|
||||
def _backfill_command(
|
||||
arguments: argparse.Namespace,
|
||||
registry: WorkflowRegistry,
|
||||
settings: Settings,
|
||||
output: TextIO,
|
||||
) -> int:
|
||||
registered = registry.resolve(arguments.workflow_id)
|
||||
workflow = _select_workflow(registered.definition, arguments)
|
||||
start = _parse_date(arguments.from_date)
|
||||
end = _parse_date(arguments.to_date)
|
||||
if end < start:
|
||||
raise CliConfigurationError("backfill end date must not precede start date")
|
||||
day_count = (end - start).days + 1
|
||||
if day_count > MAX_BACKFILL_DAYS:
|
||||
raise CliConfigurationError(
|
||||
f"backfill range exceeds safety limit of {MAX_BACKFILL_DAYS} days"
|
||||
)
|
||||
|
||||
runs: list[dict[str, object]] = []
|
||||
final_exit = EXIT_SUCCESS
|
||||
for offset in range(day_count):
|
||||
business_date = start + timedelta(days=offset)
|
||||
exit_code, payload = _execute_once(
|
||||
workflow,
|
||||
business_date=business_date,
|
||||
shadow=arguments.shadow,
|
||||
dry_run=not arguments.execute,
|
||||
settings=settings,
|
||||
)
|
||||
runs.append(payload)
|
||||
if exit_code != EXIT_SUCCESS:
|
||||
final_exit = EXIT_WORKFLOW_FAILED
|
||||
_write_json(
|
||||
output,
|
||||
{
|
||||
"workflow_id": workflow.workflow_id,
|
||||
"from": start.isoformat(),
|
||||
"to": end.isoformat(),
|
||||
"dry_run": not arguments.execute,
|
||||
"status": "success" if final_exit == EXIT_SUCCESS else "failed",
|
||||
"runs": runs,
|
||||
},
|
||||
)
|
||||
return final_exit
|
||||
|
||||
|
||||
def _select_workflow(workflow, arguments: argparse.Namespace): # type: ignore[no-untyped-def]
|
||||
if arguments.rerun_step:
|
||||
return rerun_step(workflow, arguments.rerun_step)
|
||||
if arguments.resume_from:
|
||||
return resume_from(workflow, arguments.resume_from)
|
||||
return workflow
|
||||
|
||||
|
||||
def _execute_once(
|
||||
workflow, # type: ignore[no-untyped-def]
|
||||
*,
|
||||
business_date: date,
|
||||
shadow: bool,
|
||||
dry_run: bool,
|
||||
settings: Settings,
|
||||
) -> tuple[int, dict[str, object]]:
|
||||
context = RunContext.create(
|
||||
workflow.workflow_id,
|
||||
business_date.isoformat(),
|
||||
shadow=shadow,
|
||||
)
|
||||
layout = DataLayout(settings.data_root)
|
||||
journal: RunJournal | None = None
|
||||
try:
|
||||
journal = RunJournal.create(layout, context)
|
||||
result = WorkflowEngine(
|
||||
LockManager(settings.data_root / "state" / "locks"),
|
||||
effect_ledger=EffectLedger(settings.data_root),
|
||||
).execute(workflow, context=context, journal=journal, dry_run=dry_run)
|
||||
RunIndex(settings.data_root).index_journal(journal)
|
||||
except Exception as exc:
|
||||
if journal is not None:
|
||||
with suppress(OSError, ValueError, KeyError, TypeError, json.JSONDecodeError):
|
||||
journal.finalize("failed", error="runtime infrastructure failure")
|
||||
RunIndex(settings.data_root).index_journal(journal)
|
||||
raise CliRuntimeError(
|
||||
f"runtime failure while executing {workflow.workflow_id}"
|
||||
) from exc
|
||||
return result.exit_code, _result_payload(
|
||||
context,
|
||||
result,
|
||||
dry_run=dry_run,
|
||||
journal_path=journal.path.relative_to(settings.data_root).as_posix(),
|
||||
)
|
||||
|
||||
|
||||
def _result_payload(
|
||||
context: RunContext,
|
||||
result: WorkflowRunResult,
|
||||
*,
|
||||
dry_run: bool,
|
||||
journal_path: str,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"workflow_id": result.workflow_id,
|
||||
"run_id": context.run_id,
|
||||
"business_date": context.business_date.isoformat(),
|
||||
"shadow": context.shadow,
|
||||
"dry_run": dry_run,
|
||||
"status": result.status,
|
||||
"exit_code": result.exit_code,
|
||||
"journal_path": journal_path,
|
||||
"steps": {
|
||||
step_id: step.status for step_id, step in result.steps.items()
|
||||
},
|
||||
"warnings": list(result.warnings),
|
||||
}
|
||||
|
||||
|
||||
def _parse_date(value: str) -> date:
|
||||
try:
|
||||
parsed = date.fromisoformat(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise CliConfigurationError(f"invalid business date: {value!r}") from exc
|
||||
if parsed.isoformat() != value:
|
||||
raise CliConfigurationError(f"invalid business date: {value!r}")
|
||||
return parsed
|
||||
|
||||
|
||||
def _today_shanghai() -> date:
|
||||
return datetime.now(ZoneInfo("Asia/Shanghai")).date()
|
||||
|
||||
|
||||
def _write_json(output: TextIO, payload: object) -> None:
|
||||
json.dump(payload, output, ensure_ascii=False, sort_keys=True)
|
||||
output.write("\n")
|
||||
Reference in New Issue
Block a user