feat: consolidate legacy workflows into gyxx-flow

src/gyxx_flow/workflow/__init__.py

@@ -0,0 +1,15 @@
+"""Workflow definitions, executable steps, and orchestration engine."""
+
+from .engine import WorkflowEngine, WorkflowRunResult
+from .model import StepDefinition, WorkflowDefinition, WorkflowValidationError
+from .steps import CommandStep, StepExecution
+
+__all__ = [
+    "CommandStep",
+    "StepDefinition",
+    "StepExecution",
+    "WorkflowDefinition",
+    "WorkflowEngine",
+    "WorkflowRunResult",
+    "WorkflowValidationError",
+]

src/gyxx_flow/workflow/engine.py

@@ -0,0 +1,322 @@
+"""Deterministic workflow execution with retries, locks, and policy guards."""
+
+from __future__ import annotations
+
+import time
+from contextlib import ExitStack
+from dataclasses import dataclass
+from typing import Callable, Literal
+
+from gyxx_flow.core.context import RunContext
+from gyxx_flow.core.locks import LockManager
+from gyxx_flow.core.records import RunJournal
+from gyxx_flow.ops.effects import (
+    EffectAlreadyApplied,
+    EffectClaim,
+    EffectLedger,
+    EffectStateAmbiguous,
+)
+
+from .model import StepDefinition, WorkflowDefinition
+from .steps import StepExecution
+
+StepRunStatus = Literal["success", "failed", "skipped"]
+WorkflowRunStatus = Literal["success", "failed"]
+
+
+@dataclass(frozen=True, slots=True)
+class AttemptResult:
+    attempt: int
+    status: StepRunStatus
+    exit_code: int | None
+    error: str | None = None
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class StepRunResult:
+    step_id: str
+    status: StepRunStatus
+    attempts: tuple[AttemptResult, ...]
+    final_exit_code: int | None
+    critical: bool
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class WorkflowRunResult:
+    workflow_id: str
+    status: WorkflowRunStatus
+    exit_code: int
+    steps: dict[str, StepRunResult]
+    warnings: tuple[str, ...] = ()
+
+
+class WorkflowEngine:
+    """Run a validated workflow while recording every action attempt."""
+
+    def __init__(
+        self,
+        lock_manager: LockManager,
+        *,
+        sleep: Callable[[float], None] = time.sleep,
+        effect_ledger: EffectLedger | None = None,
+    ) -> None:
+        self._lock_manager = lock_manager
+        self._sleep = sleep
+        self._effect_ledger = effect_ledger
+
+    def execute(
+        self,
+        workflow: WorkflowDefinition,
+        *,
+        context: RunContext,
+        journal: RunJournal,
+        dry_run: bool = False,
+    ) -> WorkflowRunResult:
+        if context.workflow_id != workflow.workflow_id:
+            raise ValueError(
+                "run context workflow_id does not match workflow definition: "
+                f"{context.workflow_id!r} != {workflow.workflow_id!r}"
+            )
+
+        results: dict[str, StepRunResult] = {}
+        with self._lock_manager.acquire(
+            f"workflow:{workflow.workflow_id}", owner=context.run_id
+        ):
+            for step in workflow.ordered_steps():
+                failed_dependency = any(
+                    _blocks_dependents(results[dependency])
+                    for dependency in step.depends_on
+                )
+                if failed_dependency:
+                    results[step.step_id] = self._record_policy_skip(
+                        step, journal, reason="dependency-failed"
+                    )
+                    continue
+                if dry_run:
+                    results[step.step_id] = self._record_policy_skip(
+                        step, journal, reason="dry-run"
+                    )
+                    continue
+                if context.shadow and (
+                    step.production_sink or step.official_notification
+                ):
+                    results[step.step_id] = self._record_policy_skip(
+                        step, journal, reason="shadow-policy"
+                    )
+                    continue
+                claim: EffectClaim | None = None
+                if step.production_sink and self._effect_ledger is not None:
+                    try:
+                        claim = self._effect_ledger.begin(
+                            workflow_id=context.workflow_id,
+                            business_date=context.business_date.isoformat(),
+                            step_id=step.step_id,
+                            run_id=context.run_id,
+                        )
+                    except EffectAlreadyApplied:
+                        results[step.step_id] = self._record_policy_skip(
+                            step, journal, reason="idempotency-replay"
+                        )
+                        continue
+                    except EffectStateAmbiguous:
+                        results[step.step_id] = self._record_policy_failure(
+                            step, journal, reason="effect-state-ambiguous"
+                        )
+                        continue
+                result = self._execute_step(step, context=context, journal=journal)
+                if claim is not None:
+                    if result.status == "success":
+                        self._effect_ledger.mark_applied(claim)
+                    else:
+                        self._effect_ledger.mark_ambiguous(
+                            claim, reason="step-did-not-succeed"
+                        )
+                results[step.step_id] = result
+
+        critical_failures = tuple(
+            step_id
+            for step_id, result in results.items()
+            if result.critical and _is_failure(result)
+        )
+        warnings = tuple(
+            step_id
+            for step_id, result in results.items()
+            if not result.critical and _is_failure(result)
+        )
+        status: WorkflowRunStatus = "failed" if critical_failures else "success"
+        error = None
+        if critical_failures:
+            error = f"critical steps failed: {', '.join(critical_failures)}"
+        journal.finalize(status, error=error)
+        return WorkflowRunResult(
+            workflow_id=workflow.workflow_id,
+            status=status,
+            exit_code=1 if critical_failures else 0,
+            steps=results,
+            warnings=warnings,
+        )
+
+    def _execute_step(
+        self,
+        step: StepDefinition,
+        *,
+        context: RunContext,
+        journal: RunJournal,
+    ) -> StepRunResult:
+        attempts: list[AttemptResult] = []
+        journal.record_input(f"step:{step.step_id}")
+        for attempt in range(1, step.max_attempts + 1):
+            journal_id = _attempt_journal_id(step.step_id, attempt)
+            journal.start_step(journal_id, attempt=attempt)
+            try:
+                with ExitStack() as stack:
+                    for resource in sorted(step.resources):
+                        stack.enter_context(
+                            self._lock_manager.acquire(resource, owner=context.run_id)
+                        )
+                    outcome = step.action.execute(
+                        context=context,
+                        timeout_seconds=step.timeout_seconds,
+                        dry_run=False,
+                    )
+                    if not isinstance(outcome, StepExecution):
+                        raise TypeError("step action must return StepExecution")
+            except Exception as exc:  # action and resource failures are retryable
+                outcome = StepExecution(
+                    exit_code=1,
+                    error=f"{type(exc).__name__}: {exc}",
+                )
+
+            status = _execution_status(outcome)
+            journal.finish_step(
+                journal_id,
+                status=status,
+                exit_code=outcome.exit_code,
+                error=outcome.error or outcome.reason,
+            )
+            exit_code = "none" if outcome.exit_code is None else str(outcome.exit_code)
+            journal.record_output(
+                f"step:{step.step_id}:attempt:{attempt}:{status}:exit:{exit_code}"
+            )
+            attempts.append(
+                AttemptResult(
+                    attempt=attempt,
+                    status=status,
+                    exit_code=outcome.exit_code,
+                    error=outcome.error,
+                    reason=outcome.reason,
+                )
+            )
+            if status in {"success", "skipped"}:
+                break
+            if attempt < step.max_attempts and step.retry_delay_seconds:
+                self._sleep(step.retry_delay_seconds)
+
+        final = attempts[-1]
+        if final.status == "success":
+            if step.production_sink:
+                journal.record_external_write(
+                    f"step:{step.step_id}:production_sink"
+                )
+            if step.official_notification:
+                journal.record_external_write(
+                    f"step:{step.step_id}:official_notification"
+                )
+        return StepRunResult(
+            step_id=step.step_id,
+            status=final.status,
+            attempts=tuple(attempts),
+            final_exit_code=final.exit_code,
+            critical=step.critical,
+            reason=final.reason,
+        )
+
+    def _record_policy_skip(
+        self,
+        step: StepDefinition,
+        journal: RunJournal,
+        *,
+        reason: str,
+    ) -> StepRunResult:
+        journal.record_input(f"step:{step.step_id}")
+        journal_id = _attempt_journal_id(step.step_id, 1)
+        journal.start_step(journal_id, attempt=1)
+        journal.finish_step(
+            journal_id,
+            status="skipped",
+            exit_code=0,
+            error=reason,
+        )
+        journal.record_output(
+            f"step:{step.step_id}:attempt:1:skipped:exit:0"
+        )
+        attempt = AttemptResult(
+            attempt=1,
+            status="skipped",
+            exit_code=0,
+            reason=reason,
+        )
+        return StepRunResult(
+            step_id=step.step_id,
+            status="skipped",
+            attempts=(attempt,),
+            final_exit_code=0,
+            critical=step.critical,
+            reason=reason,
+        )
+
+    def _record_policy_failure(
+        self,
+        step: StepDefinition,
+        journal: RunJournal,
+        *,
+        reason: str,
+    ) -> StepRunResult:
+        journal.record_input(f"step:{step.step_id}")
+        journal_id = _attempt_journal_id(step.step_id, 1)
+        journal.start_step(journal_id, attempt=1)
+        journal.finish_step(
+            journal_id,
+            status="failed",
+            exit_code=75,
+            error=reason,
+        )
+        journal.record_output(
+            f"step:{step.step_id}:attempt:1:failed:exit:75"
+        )
+        attempt = AttemptResult(
+            attempt=1,
+            status="failed",
+            exit_code=75,
+            error=reason,
+        )
+        return StepRunResult(
+            step_id=step.step_id,
+            status="failed",
+            attempts=(attempt,),
+            final_exit_code=75,
+            critical=step.critical,
+            reason=reason,
+        )
+
+
+def _attempt_journal_id(step_id: str, attempt: int) -> str:
+    return f"{step_id}.attempt-{attempt}"
+
+
+def _execution_status(outcome: StepExecution) -> StepRunStatus:
+    if outcome.skipped:
+        return "skipped"
+    if outcome.exit_code == 0 and outcome.error is None:
+        return "success"
+    return "failed"
+
+
+def _blocks_dependents(result: StepRunResult) -> bool:
+    return result.status == "failed" or result.reason == "dependency-failed"
+
+
+def _is_failure(result: StepRunResult) -> bool:
+    return result.status == "failed" or result.reason == "dependency-failed"

src/gyxx_flow/workflow/model.py

@@ -0,0 +1,153 @@
+"""Validated workflow and step definitions."""
+
+from __future__ import annotations
+
+import heapq
+import re
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from gyxx_flow.core.context import RunContext
+
+    from .steps import StepExecution
+
+
+_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+_RESOURCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9:._-]*$")
+
+
+class WorkflowValidationError(ValueError):
+    """Raised when a workflow definition is unsafe or inconsistent."""
+
+
+class ExecutableStep(Protocol):
+    """Action contract consumed by the workflow engine."""
+
+    def execute(
+        self,
+        *,
+        context: RunContext,
+        timeout_seconds: float | None,
+        dry_run: bool,
+    ) -> StepExecution: ...
+
+
+@dataclass(frozen=True, slots=True)
+class StepDefinition:
+    """Execution policy and dependency metadata for one workflow step."""
+
+    step_id: str
+    action: ExecutableStep
+    depends_on: tuple[str, ...] = ()
+    timeout_seconds: float | None = None
+    max_attempts: int = 1
+    retry_delay_seconds: float = 0.0
+    critical: bool = True
+    resources: tuple[str, ...] = ()
+    production_sink: bool = False
+    official_notification: bool = False
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "depends_on", tuple(self.depends_on))
+        object.__setattr__(self, "resources", tuple(self.resources))
+        if not isinstance(self.step_id, str) or not _ID.fullmatch(self.step_id):
+            raise WorkflowValidationError(f"invalid step id: {self.step_id!r}")
+        if not callable(getattr(self.action, "execute", None)):
+            raise WorkflowValidationError(f"step {self.step_id!r} has no executable action")
+        if len(set(self.depends_on)) != len(self.depends_on):
+            raise WorkflowValidationError(f"duplicate dependency for step {self.step_id!r}")
+        for dependency in self.depends_on:
+            if not isinstance(dependency, str) or not _ID.fullmatch(dependency):
+                raise WorkflowValidationError(
+                    f"invalid dependency for step {self.step_id!r}: {dependency!r}"
+                )
+            if dependency == self.step_id:
+                raise WorkflowValidationError(f"step {self.step_id!r} depends on itself")
+        if self.timeout_seconds is not None and (
+            isinstance(self.timeout_seconds, bool) or self.timeout_seconds <= 0
+        ):
+            raise WorkflowValidationError(
+                f"timeout_seconds must be positive for step {self.step_id!r}"
+            )
+        if (
+            isinstance(self.max_attempts, bool)
+            or not isinstance(self.max_attempts, int)
+            or self.max_attempts < 1
+        ):
+            raise WorkflowValidationError(
+                f"max_attempts must be a positive integer for step {self.step_id!r}"
+            )
+        if isinstance(self.retry_delay_seconds, bool) or self.retry_delay_seconds < 0:
+            raise WorkflowValidationError(
+                f"retry_delay_seconds must be non-negative for step {self.step_id!r}"
+            )
+        if len(set(self.resources)) != len(self.resources):
+            raise WorkflowValidationError(f"duplicate resource for step {self.step_id!r}")
+        for resource in self.resources:
+            if not isinstance(resource, str) or not _RESOURCE.fullmatch(resource):
+                raise WorkflowValidationError(
+                    f"invalid resource for step {self.step_id!r}: {resource!r}"
+                )
+
+
+@dataclass(frozen=True, slots=True)
+class WorkflowDefinition:
+    """A validated DAG with deterministic execution ordering."""
+
+    workflow_id: str
+    steps: tuple[StepDefinition, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "steps", tuple(self.steps))
+        if not isinstance(self.workflow_id, str) or not _ID.fullmatch(self.workflow_id):
+            raise WorkflowValidationError(f"invalid workflow id: {self.workflow_id!r}")
+        ids = [step.step_id for step in self.steps]
+        duplicate = _first_duplicate(ids)
+        if duplicate is not None:
+            raise WorkflowValidationError(f"duplicate step id: {duplicate}")
+        known = set(ids)
+        for step in self.steps:
+            for dependency in step.depends_on:
+                if dependency not in known:
+                    raise WorkflowValidationError(
+                        f"unknown dependency {dependency!r} for step {step.step_id!r}"
+                    )
+        self.ordered_steps()
+
+    def ordered_steps(self) -> tuple[StepDefinition, ...]:
+        """Return a stable topological order or reject dependency cycles."""
+
+        positions = {step.step_id: index for index, step in enumerate(self.steps)}
+        indegrees = {step.step_id: len(step.depends_on) for step in self.steps}
+        dependents: dict[str, list[str]] = {step.step_id: [] for step in self.steps}
+        for step in self.steps:
+            for dependency in step.depends_on:
+                dependents[dependency].append(step.step_id)
+
+        ready = [positions[step.step_id] for step in self.steps if not step.depends_on]
+        heapq.heapify(ready)
+        ordered: list[StepDefinition] = []
+        while ready:
+            position = heapq.heappop(ready)
+            step = self.steps[position]
+            ordered.append(step)
+            for dependent in dependents[step.step_id]:
+                indegrees[dependent] -= 1
+                if indegrees[dependent] == 0:
+                    heapq.heappush(ready, positions[dependent])
+        if len(ordered) != len(self.steps):
+            cyclic = sorted(step_id for step_id, degree in indegrees.items() if degree)
+            raise WorkflowValidationError(
+                f"dependency cycle detected: {', '.join(cyclic)}"
+            )
+        return tuple(ordered)
+
+
+def _first_duplicate(values: list[str]) -> str | None:
+    seen: set[str] = set()
+    for value in values:
+        if value in seen:
+            return value
+        seen.add(value)
+    return None

src/gyxx_flow/workflow/registry.py

@@ -0,0 +1,77 @@
+"""Safe registration boundary between catalog metadata and executable workflows."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from gyxx_flow.catalog import WorkflowCatalog, WorkflowEntry
+
+from .model import WorkflowDefinition
+
+
+class WorkflowRegistryError(LookupError):
+    """Base class for workflow resolution errors safe to show to operators."""
+
+
+class UnknownWorkflowError(WorkflowRegistryError):
+    """Raised when a workflow ID is absent from the validated catalog."""
+
+
+class UnavailableWorkflowError(WorkflowRegistryError):
+    """Raised when the catalog explicitly marks a workflow unavailable."""
+
+
+class UnregisteredWorkflowError(WorkflowRegistryError):
+    """Raised when migration has not registered executable steps yet."""
+
+
+@dataclass(frozen=True, slots=True)
+class RegisteredWorkflow:
+    """Catalog metadata plus its executable definition."""
+
+    catalog_entry: WorkflowEntry
+    definition: WorkflowDefinition
+
+
+class WorkflowRegistry:
+    """Validated, explicit registry; catalog entries are never executable by accident."""
+
+    def __init__(self, catalog: WorkflowCatalog) -> None:
+        self.catalog = catalog
+        self._entries = {entry.workflow_id: entry for entry in catalog.workflows}
+        self._definitions: dict[str, WorkflowDefinition] = {}
+
+    def register(self, definition: WorkflowDefinition) -> None:
+        """Register one migrated workflow after validating its catalog state."""
+
+        entry = self._entries.get(definition.workflow_id)
+        if entry is None:
+            raise UnknownWorkflowError(
+                f"workflow is absent from catalog: {definition.workflow_id}"
+            )
+        if entry.trigger == "unavailable":
+            raise UnavailableWorkflowError(
+                f"workflow is marked unavailable: {definition.workflow_id}"
+            )
+        if definition.workflow_id in self._definitions:
+            raise ValueError(f"workflow is already registered: {definition.workflow_id}")
+        self._definitions[definition.workflow_id] = definition
+
+    def resolve(self, workflow_id: str) -> RegisteredWorkflow:
+        """Resolve an executable workflow without falling back to legacy commands."""
+
+        entry = self._entries.get(workflow_id)
+        if entry is None:
+            raise UnknownWorkflowError(f"unknown workflow: {workflow_id}")
+        if entry.trigger == "unavailable":
+            raise UnavailableWorkflowError(f"workflow is unavailable: {workflow_id}")
+        definition = self._definitions.get(workflow_id)
+        if definition is None:
+            raise UnregisteredWorkflowError(
+                f"workflow is not registered for execution: {workflow_id}"
+            )
+        return RegisteredWorkflow(entry, definition)
+
+    def is_registered(self, workflow_id: str) -> bool:
+        return workflow_id in self._definitions
+

src/gyxx_flow/workflow/selection.py

@@ -0,0 +1,50 @@
+"""Pure workflow slicing helpers used for operator-directed recovery."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from .model import StepDefinition, WorkflowDefinition
+
+
+def rerun_step(workflow: WorkflowDefinition, step_id: str) -> WorkflowDefinition:
+    """Build a one-step rerun, treating prior dependencies as already satisfied."""
+
+    step = _step_by_id(workflow, step_id)
+    return WorkflowDefinition(workflow.workflow_id, (_without_external_dependencies(step, {step_id}),))
+
+
+def resume_from(workflow: WorkflowDefinition, step_id: str) -> WorkflowDefinition:
+    """Build a recovery slice containing a step and all of its descendants."""
+
+    _step_by_id(workflow, step_id)
+    included = {step_id}
+    changed = True
+    while changed:
+        changed = False
+        for step in workflow.steps:
+            if step.step_id not in included and set(step.depends_on) & included:
+                included.add(step.step_id)
+                changed = True
+    steps = tuple(
+        _without_external_dependencies(step, included)
+        for step in workflow.steps
+        if step.step_id in included
+    )
+    return WorkflowDefinition(workflow.workflow_id, steps)
+
+
+def _step_by_id(workflow: WorkflowDefinition, step_id: str) -> StepDefinition:
+    for step in workflow.steps:
+        if step.step_id == step_id:
+            return step
+    raise ValueError(f"unknown step for {workflow.workflow_id}: {step_id}")
+
+
+def _without_external_dependencies(
+    step: StepDefinition, included: set[str]
+) -> StepDefinition:
+    dependencies = tuple(
+        dependency for dependency in step.depends_on if dependency in included
+    )
+    return replace(step, depends_on=dependencies)

src/gyxx_flow/workflow/steps.py

@@ -0,0 +1,101 @@
+"""Built-in executable workflow steps."""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+from gyxx_flow.core.context import RunContext
+
+
+@dataclass(frozen=True, slots=True)
+class StepExecution:
+    """One action invocation outcome before retry aggregation."""
+
+    exit_code: int
+    stdout: str = ""
+    stderr: str = ""
+    error: str | None = None
+    skipped: bool = False
+    reason: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CommandStep:
+    """Execute an argv command without shell parsing or implicit process context."""
+
+    argv: tuple[str, ...]
+    cwd: Path
+    env: Mapping[str, str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "argv", tuple(self.argv))
+        object.__setattr__(self, "cwd", Path(self.cwd))
+        object.__setattr__(self, "env", dict(self.env))
+        if not self.argv or not all(
+            isinstance(argument, str) and argument and "\x00" not in argument
+            for argument in self.argv
+        ):
+            raise ValueError("argv must contain non-empty strings")
+        if not all(
+            isinstance(key, str)
+            and key
+            and "\x00" not in key
+            and isinstance(value, str)
+            and "\x00" not in value
+            for key, value in self.env.items()
+        ):
+            raise ValueError("env must contain valid string keys and values")
+
+    def execute(
+        self,
+        *,
+        context: RunContext,
+        timeout_seconds: float | None,
+        dry_run: bool,
+    ) -> StepExecution:
+        del context
+        if dry_run:
+            return StepExecution(exit_code=0, skipped=True, reason="dry-run")
+        try:
+            completed = subprocess.run(
+                list(self.argv),
+                cwd=self.cwd,
+                env=dict(self.env),
+                shell=False,
+                capture_output=True,
+                text=True,
+                timeout=timeout_seconds,
+                check=False,
+            )
+        except subprocess.TimeoutExpired as exc:
+            stdout = _to_text(exc.stdout)
+            stderr = _to_text(exc.stderr)
+            return StepExecution(
+                exit_code=124,
+                stdout=stdout,
+                stderr=stderr,
+                error=f"command timed out after {exc.timeout} seconds",
+            )
+        except OSError as exc:
+            return StepExecution(exit_code=127, error=f"cannot start command: {exc}")
+
+        error = None
+        if completed.returncode != 0:
+            error = completed.stderr.strip() or f"command exited with code {completed.returncode}"
+        return StepExecution(
+            exit_code=completed.returncode,
+            stdout=completed.stdout,
+            stderr=completed.stderr,
+            error=error,
+        )
+
+
+def _to_text(value: str | bytes | None) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode(errors="replace")
+    return value