210 lines
8.1 KiB
Python
210 lines
8.1 KiB
Python
"""Resolve workflow commands from source owned by this installed project."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
from types import MappingProxyType
|
|
from typing import Callable, Mapping
|
|
|
|
from gyxx_flow.adapters.integration import RuntimeIntegrationCatalog
|
|
from gyxx_flow.catalog import WorkflowEntry
|
|
from gyxx_flow.core.config import Settings
|
|
from gyxx_flow.core.context import RunContext
|
|
from gyxx_flow.core.layout import DataLayout
|
|
from gyxx_flow.workflow.model import ExecutableStep
|
|
from gyxx_flow.workflow.steps import CommandStep, StepExecution
|
|
|
|
|
|
class ModuleSourceError(ValueError):
|
|
"""Raised when a workflow cannot resolve project-owned module source safely."""
|
|
|
|
|
|
class ModuleSourceRoots:
|
|
"""Validated runtime roots for independently packaged business modules."""
|
|
|
|
def __init__(self, roots: Mapping[str, Path | str]) -> None:
|
|
resolved: dict[str, Path] = {}
|
|
for module, configured in roots.items():
|
|
if not module or not isinstance(module, str):
|
|
raise ModuleSourceError(f"invalid module id: {module!r}")
|
|
root = Path(configured).expanduser().resolve(strict=True)
|
|
if not root.is_dir():
|
|
raise ModuleSourceError(f"module source root is not a directory: {root}")
|
|
resolved[module] = root
|
|
self._roots = MappingProxyType(resolved)
|
|
|
|
def get(self, module: str) -> Path:
|
|
try:
|
|
return self._roots[module]
|
|
except KeyError as exc:
|
|
raise ModuleSourceError(f"unknown module source: {module!r}") from exc
|
|
|
|
|
|
class ModuleCommandAdapter:
|
|
"""Build a shell-free command rooted in source shipped with this project."""
|
|
|
|
def __init__(
|
|
self,
|
|
roots: ModuleSourceRoots,
|
|
*,
|
|
base_env: Mapping[str, str] | None = None,
|
|
python_executable: str | Path | None = None,
|
|
project_root: str | Path | None = None,
|
|
data_root: str | Path | None = None,
|
|
integration_catalog: RuntimeIntegrationCatalog | None = None,
|
|
) -> None:
|
|
self._roots = roots
|
|
self._base_env = dict(os.environ if base_env is None else base_env)
|
|
self._python = str(python_executable or sys.executable)
|
|
settings = Settings.from_env(project_root=project_root, env=self._base_env)
|
|
self._project_root = settings.project_root
|
|
self._data_root = (
|
|
Path(data_root).expanduser().resolve()
|
|
if data_root is not None
|
|
else settings.data_root
|
|
)
|
|
binding_file = self._project_root / "config" / "runtime-bindings.json"
|
|
self._integration_catalog = integration_catalog
|
|
if self._integration_catalog is None and binding_file.is_file():
|
|
self._integration_catalog = RuntimeIntegrationCatalog.load_default(
|
|
project_root=self._project_root,
|
|
data_root=self._data_root,
|
|
)
|
|
|
|
def build(self, entry: WorkflowEntry, *, context: RunContext) -> CommandStep:
|
|
if entry.workflow_id != context.workflow_id:
|
|
raise ModuleSourceError(
|
|
f"workflow entry {entry.workflow_id!r} does not match "
|
|
f"run context {context.workflow_id!r}"
|
|
)
|
|
if entry.trigger == "unavailable":
|
|
raise ModuleSourceError(f"workflow is unavailable: {entry.workflow_id}")
|
|
root = self._roots.get(entry.module)
|
|
executable = _resolve_entry(root, entry.entry)
|
|
suffix = executable.suffix.casefold()
|
|
if suffix in {".py", ".pyw"}:
|
|
argv = (self._python, str(executable), *entry.args)
|
|
elif suffix in {".bat", ".cmd"}:
|
|
command_line = subprocess.list2cmdline([str(executable), *entry.args])
|
|
argv = ("cmd.exe", "/d", "/s", "/c", command_line)
|
|
elif suffix == ".ps1":
|
|
argv = (
|
|
"powershell.exe",
|
|
"-NoProfile",
|
|
"-ExecutionPolicy",
|
|
"Bypass",
|
|
"-File",
|
|
str(executable),
|
|
*entry.args,
|
|
)
|
|
else:
|
|
argv = (str(executable), *entry.args)
|
|
env = {
|
|
**self._base_env,
|
|
"GYXX_PROJECT_ROOT": str(self._project_root),
|
|
"GYXX_DATA_ROOT": str(self._data_root),
|
|
"GYXX_MODULE_ROOT": str(root),
|
|
"GYXX_MODULE_ID": entry.module,
|
|
"GYXX_PYTHON": self._python,
|
|
"GYXX_WORKFLOW_ID": context.workflow_id,
|
|
"GYXX_RUN_ID": context.run_id,
|
|
"GYXX_BUSINESS_DATE": context.business_date.isoformat(),
|
|
"GYXX_SHADOW": "true" if context.shadow else "false",
|
|
"PYTHONUNBUFFERED": "1",
|
|
}
|
|
if entry.module == "supply_chain":
|
|
supply_paths = DataLayout(self._data_root).for_module("supply_chain")
|
|
env.update(
|
|
{
|
|
"GYXX_SUPPLY_RAW_ROOT": str(supply_paths.raw_root),
|
|
"GYXX_SUPPLY_STATE_ROOT": str(supply_paths.state_root),
|
|
"GYXX_SUPPLY_EXPORT_ROOT": str(supply_paths.exports_root),
|
|
"GYXX_SUPPLY_WORK_ROOT": str(supply_paths.tmp_root),
|
|
}
|
|
)
|
|
if self._integration_catalog is not None:
|
|
env = self._integration_catalog.environment_for(
|
|
f"{entry.module}:{entry.entry}", env
|
|
)
|
|
existing_pythonpath = env.get("PYTHONPATH", "").strip()
|
|
env["PYTHONPATH"] = (
|
|
f"{root}{os.pathsep}{existing_pythonpath}" if existing_pythonpath else str(root)
|
|
)
|
|
return CommandStep(argv=tuple(argv), cwd=root, env=env)
|
|
|
|
|
|
ModuleCommandFactory = Callable[[WorkflowEntry, RunContext], ExecutableStep]
|
|
|
|
|
|
def _command_from_project(entry: WorkflowEntry, context: RunContext) -> ExecutableStep:
|
|
package_root = Path(__file__).resolve().parents[1]
|
|
runtime_root = package_root / "modules" / entry.module / "runtime"
|
|
settings = Settings.from_env()
|
|
return ModuleCommandAdapter(
|
|
ModuleSourceRoots({entry.module: runtime_root}),
|
|
project_root=settings.project_root,
|
|
data_root=settings.data_root,
|
|
).build(entry, context=context)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class DeferredModuleCommandStep:
|
|
"""Resolve project-owned module source only when a real execution starts."""
|
|
|
|
entry: WorkflowEntry
|
|
command_factory: ModuleCommandFactory = _command_from_project
|
|
|
|
def __post_init__(self) -> None:
|
|
if not callable(self.command_factory):
|
|
raise TypeError("command_factory must be callable")
|
|
|
|
def execute(
|
|
self,
|
|
*,
|
|
context: RunContext,
|
|
timeout_seconds: float | None,
|
|
dry_run: bool,
|
|
) -> StepExecution:
|
|
if dry_run:
|
|
return StepExecution(exit_code=0, skipped=True, reason="dry-run")
|
|
command = self.command_factory(self.entry, context)
|
|
if not callable(getattr(command, "execute", None)):
|
|
raise TypeError("command_factory must return an executable step")
|
|
return command.execute(
|
|
context=context,
|
|
timeout_seconds=timeout_seconds,
|
|
dry_run=False,
|
|
)
|
|
|
|
|
|
def _resolve_entry(root: Path, configured_entry: str) -> Path:
|
|
if not configured_entry or "\\" in configured_entry or ":" in configured_entry:
|
|
raise ModuleSourceError("module entry must use a safe relative path")
|
|
relative = PurePosixPath(configured_entry)
|
|
if relative.is_absolute() or ".." in relative.parts:
|
|
raise ModuleSourceError("module entry cannot escape its module root")
|
|
try:
|
|
candidate = root.joinpath(*relative.parts).resolve(strict=True)
|
|
except OSError as exc:
|
|
raise ModuleSourceError(
|
|
f"module entry does not exist or cannot be resolved: {configured_entry}"
|
|
) from exc
|
|
if not candidate.is_relative_to(root):
|
|
raise ModuleSourceError("module entry cannot escape its module root")
|
|
if not candidate.is_file():
|
|
raise ModuleSourceError(f"module entry is not a file: {configured_entry}")
|
|
return candidate
|
|
|
|
|
|
__all__ = [
|
|
"DeferredModuleCommandStep",
|
|
"ModuleCommandAdapter",
|
|
"ModuleCommandFactory",
|
|
"ModuleSourceError",
|
|
"ModuleSourceRoots",
|
|
]
|