feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
"""Resolve workflow commands from source owned by this installed project."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from types import MappingProxyType
|
||||
from typing import Callable, Mapping
|
||||
|
||||
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
||||
from gyxx_flow.catalog import WorkflowEntry
|
||||
from gyxx_flow.core.config import Settings
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.core.layout import DataLayout
|
||||
from gyxx_flow.workflow.model import ExecutableStep
|
||||
from gyxx_flow.workflow.steps import CommandStep, StepExecution
|
||||
|
||||
|
||||
class ModuleSourceError(ValueError):
|
||||
"""Raised when a workflow cannot resolve project-owned module source safely."""
|
||||
|
||||
|
||||
class ModuleSourceRoots:
|
||||
"""Validated runtime roots for independently packaged business modules."""
|
||||
|
||||
def __init__(self, roots: Mapping[str, Path | str]) -> None:
|
||||
resolved: dict[str, Path] = {}
|
||||
for module, configured in roots.items():
|
||||
if not module or not isinstance(module, str):
|
||||
raise ModuleSourceError(f"invalid module id: {module!r}")
|
||||
root = Path(configured).expanduser().resolve(strict=True)
|
||||
if not root.is_dir():
|
||||
raise ModuleSourceError(f"module source root is not a directory: {root}")
|
||||
resolved[module] = root
|
||||
self._roots = MappingProxyType(resolved)
|
||||
|
||||
def get(self, module: str) -> Path:
|
||||
try:
|
||||
return self._roots[module]
|
||||
except KeyError as exc:
|
||||
raise ModuleSourceError(f"unknown module source: {module!r}") from exc
|
||||
|
||||
|
||||
class ModuleCommandAdapter:
|
||||
"""Build a shell-free command rooted in source shipped with this project."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
roots: ModuleSourceRoots,
|
||||
*,
|
||||
base_env: Mapping[str, str] | None = None,
|
||||
python_executable: str | Path | None = None,
|
||||
project_root: str | Path | None = None,
|
||||
data_root: str | Path | None = None,
|
||||
integration_catalog: RuntimeIntegrationCatalog | None = None,
|
||||
) -> None:
|
||||
self._roots = roots
|
||||
self._base_env = dict(os.environ if base_env is None else base_env)
|
||||
self._python = str(python_executable or sys.executable)
|
||||
settings = Settings.from_env(project_root=project_root, env=self._base_env)
|
||||
self._project_root = settings.project_root
|
||||
self._data_root = (
|
||||
Path(data_root).expanduser().resolve()
|
||||
if data_root is not None
|
||||
else settings.data_root
|
||||
)
|
||||
binding_file = self._project_root / "config" / "runtime-bindings.json"
|
||||
self._integration_catalog = integration_catalog
|
||||
if self._integration_catalog is None and binding_file.is_file():
|
||||
self._integration_catalog = RuntimeIntegrationCatalog.load_default(
|
||||
project_root=self._project_root,
|
||||
data_root=self._data_root,
|
||||
)
|
||||
|
||||
def build(self, entry: WorkflowEntry, *, context: RunContext) -> CommandStep:
|
||||
if entry.workflow_id != context.workflow_id:
|
||||
raise ModuleSourceError(
|
||||
f"workflow entry {entry.workflow_id!r} does not match "
|
||||
f"run context {context.workflow_id!r}"
|
||||
)
|
||||
if entry.trigger == "unavailable":
|
||||
raise ModuleSourceError(f"workflow is unavailable: {entry.workflow_id}")
|
||||
root = self._roots.get(entry.module)
|
||||
executable = _resolve_entry(root, entry.entry)
|
||||
suffix = executable.suffix.casefold()
|
||||
if suffix in {".py", ".pyw"}:
|
||||
argv = (self._python, str(executable), *entry.args)
|
||||
elif suffix in {".bat", ".cmd"}:
|
||||
command_line = subprocess.list2cmdline([str(executable), *entry.args])
|
||||
argv = ("cmd.exe", "/d", "/s", "/c", command_line)
|
||||
elif suffix == ".ps1":
|
||||
argv = (
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(executable),
|
||||
*entry.args,
|
||||
)
|
||||
else:
|
||||
argv = (str(executable), *entry.args)
|
||||
env = {
|
||||
**self._base_env,
|
||||
"GYXX_PROJECT_ROOT": str(self._project_root),
|
||||
"GYXX_DATA_ROOT": str(self._data_root),
|
||||
"GYXX_MODULE_ROOT": str(root),
|
||||
"GYXX_MODULE_ID": entry.module,
|
||||
"GYXX_PYTHON": self._python,
|
||||
"GYXX_WORKFLOW_ID": context.workflow_id,
|
||||
"GYXX_RUN_ID": context.run_id,
|
||||
"GYXX_BUSINESS_DATE": context.business_date.isoformat(),
|
||||
"GYXX_SHADOW": "true" if context.shadow else "false",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
}
|
||||
if entry.module == "supply_chain":
|
||||
supply_paths = DataLayout(self._data_root).for_module("supply_chain")
|
||||
env.update(
|
||||
{
|
||||
"GYXX_SUPPLY_RAW_ROOT": str(supply_paths.raw_root),
|
||||
"GYXX_SUPPLY_STATE_ROOT": str(supply_paths.state_root),
|
||||
"GYXX_SUPPLY_EXPORT_ROOT": str(supply_paths.exports_root),
|
||||
"GYXX_SUPPLY_WORK_ROOT": str(supply_paths.tmp_root),
|
||||
}
|
||||
)
|
||||
if self._integration_catalog is not None:
|
||||
env = self._integration_catalog.environment_for(
|
||||
f"{entry.module}:{entry.entry}", env
|
||||
)
|
||||
existing_pythonpath = env.get("PYTHONPATH", "").strip()
|
||||
env["PYTHONPATH"] = (
|
||||
f"{root}{os.pathsep}{existing_pythonpath}" if existing_pythonpath else str(root)
|
||||
)
|
||||
return CommandStep(argv=tuple(argv), cwd=root, env=env)
|
||||
|
||||
|
||||
ModuleCommandFactory = Callable[[WorkflowEntry, RunContext], ExecutableStep]
|
||||
|
||||
|
||||
def _command_from_project(entry: WorkflowEntry, context: RunContext) -> ExecutableStep:
|
||||
package_root = Path(__file__).resolve().parents[1]
|
||||
runtime_root = package_root / "modules" / entry.module / "runtime"
|
||||
settings = Settings.from_env()
|
||||
return ModuleCommandAdapter(
|
||||
ModuleSourceRoots({entry.module: runtime_root}),
|
||||
project_root=settings.project_root,
|
||||
data_root=settings.data_root,
|
||||
).build(entry, context=context)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeferredModuleCommandStep:
|
||||
"""Resolve project-owned module source only when a real execution starts."""
|
||||
|
||||
entry: WorkflowEntry
|
||||
command_factory: ModuleCommandFactory = _command_from_project
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not callable(self.command_factory):
|
||||
raise TypeError("command_factory must be callable")
|
||||
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
context: RunContext,
|
||||
timeout_seconds: float | None,
|
||||
dry_run: bool,
|
||||
) -> StepExecution:
|
||||
if dry_run:
|
||||
return StepExecution(exit_code=0, skipped=True, reason="dry-run")
|
||||
command = self.command_factory(self.entry, context)
|
||||
if not callable(getattr(command, "execute", None)):
|
||||
raise TypeError("command_factory must return an executable step")
|
||||
return command.execute(
|
||||
context=context,
|
||||
timeout_seconds=timeout_seconds,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_entry(root: Path, configured_entry: str) -> Path:
|
||||
if not configured_entry or "\\" in configured_entry or ":" in configured_entry:
|
||||
raise ModuleSourceError("module entry must use a safe relative path")
|
||||
relative = PurePosixPath(configured_entry)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise ModuleSourceError("module entry cannot escape its module root")
|
||||
try:
|
||||
candidate = root.joinpath(*relative.parts).resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ModuleSourceError(
|
||||
f"module entry does not exist or cannot be resolved: {configured_entry}"
|
||||
) from exc
|
||||
if not candidate.is_relative_to(root):
|
||||
raise ModuleSourceError("module entry cannot escape its module root")
|
||||
if not candidate.is_file():
|
||||
raise ModuleSourceError(f"module entry is not a file: {configured_entry}")
|
||||
return candidate
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeferredModuleCommandStep",
|
||||
"ModuleCommandAdapter",
|
||||
"ModuleCommandFactory",
|
||||
"ModuleSourceError",
|
||||
"ModuleSourceRoots",
|
||||
]
|
||||
Reference in New Issue
Block a user