feat: complete production workflow migration
This commit is contained in:
+310
-18
@@ -3,13 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import time
|
||||
from datetime import date
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Literal
|
||||
|
||||
_WORKFLOW_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
|
||||
_SCHEDULE_TIME = re.compile(r"^(?:[01]\d|2[0-3]):[0-5]\d$")
|
||||
_MODULES = {
|
||||
"content_marketing",
|
||||
"product_commerce",
|
||||
@@ -25,6 +27,35 @@ class CatalogError(ValueError):
|
||||
"""Raised for an invalid workflow catalog."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowDataEndpoint:
|
||||
label: str
|
||||
system: str | None = None
|
||||
detail: str | None = None
|
||||
condition: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowDataFlow:
|
||||
sources: tuple[WorkflowDataEndpoint, ...]
|
||||
processing: tuple[str, ...]
|
||||
destinations: tuple[WorkflowDataEndpoint, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowStepEntry:
|
||||
step_id: str
|
||||
entry: str
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
args: tuple[str, ...] = ()
|
||||
depends_on: tuple[str, ...] = ()
|
||||
run_after_failure: bool = False
|
||||
timeout_seconds: float | None = None
|
||||
replay_policy: Literal["guarded", "idempotent"] | None = None
|
||||
data_flow: WorkflowDataFlow | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowEntry:
|
||||
workflow_id: str
|
||||
@@ -35,6 +66,7 @@ class WorkflowEntry:
|
||||
source_project: str | None = None
|
||||
source_task_name: str | None = None
|
||||
note: str | None = None
|
||||
steps: tuple[WorkflowStepEntry, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -46,6 +78,14 @@ class ScheduleEntry:
|
||||
day_of_month: int | None = None
|
||||
every_days: int | None = None
|
||||
anchor_date: str | None = None
|
||||
enabled: bool = True
|
||||
business_date_offset_days: int = 0
|
||||
at_times: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def effective_times(self) -> tuple[str, ...]:
|
||||
"""Return every configured wall-clock time, including legacy ``at``."""
|
||||
return self.at_times or (self.at,)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -59,7 +99,7 @@ class WorkflowCatalog:
|
||||
root = Path(config_dir)
|
||||
workflow_payload = _load_json(root / "workflows.json")
|
||||
schedule_payload = _load_json(root / "schedules.json")
|
||||
_require_schema(workflow_payload, "workflows.json", expected=2)
|
||||
_require_schema(workflow_payload, "workflows.json", expected=3)
|
||||
_require_schema(schedule_payload, "schedules.json", expected=1)
|
||||
workflows = tuple(_parse_workflow(item) for item in workflow_payload.get("workflows", []))
|
||||
schedules = tuple(_parse_schedule(item) for item in schedule_payload.get("schedules", []))
|
||||
@@ -144,18 +184,27 @@ def _parse_workflow(item: Any) -> WorkflowEntry:
|
||||
raise CatalogError(f"missing execution definition for {workflow_id}")
|
||||
if not isinstance(provenance, dict):
|
||||
raise CatalogError(f"invalid provenance definition for {workflow_id}")
|
||||
entry = execution.get("entry")
|
||||
if not _is_relative_entry(entry):
|
||||
raise CatalogError(f"execution entry must be relative for {workflow_id}")
|
||||
raw_steps = execution.get("steps")
|
||||
if raw_steps is None:
|
||||
entry = execution.get("entry")
|
||||
if not _is_relative_entry(entry):
|
||||
raise CatalogError(f"execution entry must be relative for {workflow_id}")
|
||||
args = _parse_args(execution.get("args", []), workflow_id)
|
||||
steps: tuple[WorkflowStepEntry, ...] = ()
|
||||
else:
|
||||
if "entry" in execution or "args" in execution:
|
||||
raise CatalogError(
|
||||
f"execution cannot mix entry and steps for {workflow_id}"
|
||||
)
|
||||
steps = _parse_workflow_steps(raw_steps, workflow_id)
|
||||
entry = steps[0].entry
|
||||
args = steps[0].args
|
||||
project = provenance.get("source_project")
|
||||
if project is not None and (not isinstance(project, str) or not project):
|
||||
raise CatalogError(f"invalid source project for {workflow_id}")
|
||||
task_name = provenance.get("task_name")
|
||||
if trigger == "scheduled" and (not isinstance(task_name, str) or not task_name):
|
||||
raise CatalogError(f"scheduled workflow requires source task name: {workflow_id}")
|
||||
args = execution.get("args", [])
|
||||
if not isinstance(args, list) or not all(isinstance(value, str) for value in args):
|
||||
raise CatalogError(f"execution args must be strings for {workflow_id}")
|
||||
return WorkflowEntry(
|
||||
workflow_id=workflow_id,
|
||||
module=module,
|
||||
@@ -165,9 +214,195 @@ def _parse_workflow(item: Any) -> WorkflowEntry:
|
||||
source_project=project,
|
||||
source_task_name=task_name,
|
||||
note=item.get("note"),
|
||||
steps=steps,
|
||||
)
|
||||
|
||||
|
||||
def _parse_workflow_steps(
|
||||
raw_steps: Any,
|
||||
workflow_id: str,
|
||||
) -> tuple[WorkflowStepEntry, ...]:
|
||||
if not isinstance(raw_steps, list) or not raw_steps:
|
||||
raise CatalogError(f"execution steps must be a non-empty list for {workflow_id}")
|
||||
steps: list[WorkflowStepEntry] = []
|
||||
for raw_step in raw_steps:
|
||||
if not isinstance(raw_step, dict):
|
||||
raise CatalogError(f"workflow step must be an object for {workflow_id}")
|
||||
step_id = raw_step.get("id")
|
||||
entry = raw_step.get("entry")
|
||||
if not isinstance(step_id, str) or not _WORKFLOW_ID.fullmatch(step_id):
|
||||
raise CatalogError(f"invalid workflow step id for {workflow_id}: {step_id!r}")
|
||||
if not _is_relative_entry(entry):
|
||||
raise CatalogError(
|
||||
f"workflow step entry must be relative for {workflow_id}.{step_id}"
|
||||
)
|
||||
depends_on = raw_step.get("depends_on", [])
|
||||
if not isinstance(depends_on, list) or not all(
|
||||
isinstance(value, str) and _WORKFLOW_ID.fullmatch(value)
|
||||
for value in depends_on
|
||||
):
|
||||
raise CatalogError(
|
||||
f"workflow step dependencies are invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
run_after_failure = raw_step.get("run_after_failure", False)
|
||||
if not isinstance(run_after_failure, bool):
|
||||
raise CatalogError(
|
||||
f"workflow step run_after_failure is invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
timeout_seconds = raw_step.get("timeout_seconds")
|
||||
if timeout_seconds is not None and (
|
||||
isinstance(timeout_seconds, bool)
|
||||
or not isinstance(timeout_seconds, (int, float))
|
||||
or not math.isfinite(timeout_seconds)
|
||||
or timeout_seconds <= 0
|
||||
):
|
||||
raise CatalogError(
|
||||
f"workflow step timeout_seconds is invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
replay_policy = raw_step.get("replay_policy")
|
||||
if replay_policy is not None and replay_policy not in {
|
||||
"guarded",
|
||||
"idempotent",
|
||||
}:
|
||||
raise CatalogError(
|
||||
f"workflow step replay_policy is invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
name = raw_step.get("name")
|
||||
if name is not None and (not isinstance(name, str) or not name.strip()):
|
||||
raise CatalogError(
|
||||
f"workflow step name is invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
description = raw_step.get("description")
|
||||
if description is not None and (
|
||||
not isinstance(description, str) or not description.strip()
|
||||
):
|
||||
raise CatalogError(
|
||||
f"workflow step description is invalid for {workflow_id}.{step_id}"
|
||||
)
|
||||
steps.append(
|
||||
WorkflowStepEntry(
|
||||
step_id=step_id,
|
||||
entry=entry,
|
||||
name=name.strip() if name is not None else None,
|
||||
description=description.strip() if description is not None else None,
|
||||
args=_parse_args(raw_step.get("args", []), workflow_id),
|
||||
depends_on=tuple(depends_on),
|
||||
run_after_failure=run_after_failure,
|
||||
timeout_seconds=(
|
||||
float(timeout_seconds) if timeout_seconds is not None else None
|
||||
),
|
||||
replay_policy=replay_policy,
|
||||
data_flow=_parse_step_data_flow(
|
||||
raw_step.get("data_flow"),
|
||||
workflow_id,
|
||||
step_id,
|
||||
),
|
||||
)
|
||||
)
|
||||
step_ids = [step.step_id for step in steps]
|
||||
duplicates = _duplicates(step_ids)
|
||||
if duplicates:
|
||||
raise CatalogError(
|
||||
f"duplicate workflow step id for {workflow_id}: {sorted(duplicates)[0]}"
|
||||
)
|
||||
known = set(step_ids)
|
||||
for step in steps:
|
||||
unknown = set(step.depends_on) - known
|
||||
if unknown:
|
||||
raise CatalogError(
|
||||
f"unknown workflow step dependency for {workflow_id}.{step.step_id}: "
|
||||
f"{sorted(unknown)[0]}"
|
||||
)
|
||||
_validate_step_graph(steps, workflow_id)
|
||||
return tuple(steps)
|
||||
|
||||
|
||||
def _parse_step_data_flow(
|
||||
value: Any,
|
||||
workflow_id: str,
|
||||
step_id: str,
|
||||
) -> WorkflowDataFlow | None:
|
||||
if value is None:
|
||||
return None
|
||||
label = f"{workflow_id}.{step_id}"
|
||||
if not isinstance(value, dict):
|
||||
raise CatalogError(f"workflow step data_flow must be an object for {label}")
|
||||
allowed = {"sources", "processing", "destinations"}
|
||||
if set(value) != allowed:
|
||||
raise CatalogError(f"workflow step data_flow fields are invalid for {label}")
|
||||
processing = value.get("processing")
|
||||
if not isinstance(processing, list) or not processing or not all(
|
||||
isinstance(item, str) and item.strip() for item in processing
|
||||
):
|
||||
raise CatalogError(f"workflow step processing is invalid for {label}")
|
||||
return WorkflowDataFlow(
|
||||
sources=_parse_data_endpoints(value.get("sources"), label, "sources"),
|
||||
processing=tuple(item.strip() for item in processing),
|
||||
destinations=_parse_data_endpoints(
|
||||
value.get("destinations"),
|
||||
label,
|
||||
"destinations",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _parse_data_endpoints(
|
||||
value: Any,
|
||||
step_label: str,
|
||||
field: str,
|
||||
) -> tuple[WorkflowDataEndpoint, ...]:
|
||||
if not isinstance(value, list) or not value:
|
||||
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
|
||||
endpoints: list[WorkflowDataEndpoint] = []
|
||||
allowed = {"label", "system", "detail", "condition"}
|
||||
for item in value:
|
||||
if not isinstance(item, dict) or set(item) - allowed:
|
||||
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
|
||||
endpoint_label = item.get("label")
|
||||
if not isinstance(endpoint_label, str) or not endpoint_label.strip():
|
||||
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
|
||||
optional: dict[str, str | None] = {}
|
||||
for key in ("system", "detail", "condition"):
|
||||
raw = item.get(key)
|
||||
if raw is not None and (not isinstance(raw, str) or not raw.strip()):
|
||||
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
|
||||
optional[key] = raw.strip() if raw is not None else None
|
||||
endpoints.append(
|
||||
WorkflowDataEndpoint(
|
||||
label=endpoint_label.strip(),
|
||||
system=optional["system"],
|
||||
detail=optional["detail"],
|
||||
condition=optional["condition"],
|
||||
)
|
||||
)
|
||||
return tuple(endpoints)
|
||||
|
||||
|
||||
def _parse_args(value: Any, workflow_id: str) -> tuple[str, ...]:
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
||||
raise CatalogError(f"execution args must be strings for {workflow_id}")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _validate_step_graph(
|
||||
steps: list[WorkflowStepEntry],
|
||||
workflow_id: str,
|
||||
) -> None:
|
||||
unresolved = {step.step_id: set(step.depends_on) for step in steps}
|
||||
completed: set[str] = set()
|
||||
while unresolved:
|
||||
ready = [
|
||||
step_id
|
||||
for step_id, dependencies in unresolved.items()
|
||||
if dependencies <= completed
|
||||
]
|
||||
if not ready:
|
||||
raise CatalogError(f"workflow step dependency cycle for {workflow_id}")
|
||||
completed.update(ready)
|
||||
for step_id in ready:
|
||||
del unresolved[step_id]
|
||||
|
||||
|
||||
def _is_relative_entry(entry: Any) -> bool:
|
||||
if not isinstance(entry, str) or not entry or "\\" in entry or ":" in entry:
|
||||
return False
|
||||
@@ -180,24 +415,78 @@ def _parse_schedule(item: Any) -> ScheduleEntry:
|
||||
raise CatalogError("schedule entry must be an object")
|
||||
workflow_id = item.get("workflow_id")
|
||||
kind = item.get("kind")
|
||||
at = item.get("at")
|
||||
raw_at = item.get("at")
|
||||
if not isinstance(workflow_id, str) or not _WORKFLOW_ID.fullmatch(workflow_id):
|
||||
raise CatalogError(f"invalid schedule workflow id: {workflow_id!r}")
|
||||
if kind not in _KINDS:
|
||||
raise CatalogError(f"invalid schedule kind for {workflow_id}: {kind!r}")
|
||||
try:
|
||||
time.fromisoformat(at)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise CatalogError(f"invalid schedule time for {workflow_id}: {at!r}") from exc
|
||||
days = tuple(item.get("days", []))
|
||||
if kind == "weekly" and (not days or not set(days) <= _WEEKDAYS):
|
||||
if isinstance(raw_at, str):
|
||||
if not _SCHEDULE_TIME.fullmatch(raw_at):
|
||||
raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}")
|
||||
at = raw_at
|
||||
at_times: tuple[str, ...] = ()
|
||||
elif isinstance(raw_at, list):
|
||||
if (
|
||||
not raw_at
|
||||
or not all(
|
||||
isinstance(value, str) and _SCHEDULE_TIME.fullmatch(value)
|
||||
for value in raw_at
|
||||
)
|
||||
or len(raw_at) != len(set(raw_at))
|
||||
):
|
||||
raise CatalogError(f"invalid schedule times for {workflow_id}: {raw_at!r}")
|
||||
at_times = tuple(raw_at)
|
||||
at = at_times[0]
|
||||
else:
|
||||
raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}")
|
||||
raw_days = item.get("days", [])
|
||||
if not isinstance(raw_days, list) or not all(
|
||||
isinstance(day, str) for day in raw_days
|
||||
):
|
||||
raise CatalogError(f"invalid weekly days for {workflow_id}")
|
||||
days = tuple(raw_days)
|
||||
if kind == "weekly" and (
|
||||
not days
|
||||
or len(days) != len(set(days))
|
||||
or not set(days) <= _WEEKDAYS
|
||||
):
|
||||
raise CatalogError(f"invalid weekly days for {workflow_id}")
|
||||
day_of_month = item.get("day_of_month")
|
||||
if kind == "monthly" and (not isinstance(day_of_month, int) or not 1 <= day_of_month <= 31):
|
||||
if kind == "monthly" and (
|
||||
isinstance(day_of_month, bool)
|
||||
or not isinstance(day_of_month, int)
|
||||
or not 1 <= day_of_month <= 31
|
||||
):
|
||||
raise CatalogError(f"invalid day_of_month for {workflow_id}")
|
||||
every_days = item.get("every_days")
|
||||
if kind == "interval_days" and (not isinstance(every_days, int) or every_days < 1):
|
||||
if kind == "interval_days" and (
|
||||
isinstance(every_days, bool)
|
||||
or not isinstance(every_days, int)
|
||||
or every_days < 1
|
||||
):
|
||||
raise CatalogError(f"invalid every_days for {workflow_id}")
|
||||
anchor_date = item.get("anchor_date")
|
||||
if kind == "interval_days":
|
||||
try:
|
||||
parsed_anchor = date.fromisoformat(anchor_date)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise CatalogError(
|
||||
f"invalid anchor_date for {workflow_id}: {anchor_date!r}"
|
||||
) from exc
|
||||
if parsed_anchor.isoformat() != anchor_date:
|
||||
raise CatalogError(
|
||||
f"invalid anchor_date for {workflow_id}: {anchor_date!r}"
|
||||
)
|
||||
enabled = item.get("enabled", True)
|
||||
if not isinstance(enabled, bool):
|
||||
raise CatalogError(f"invalid enabled flag for {workflow_id}")
|
||||
business_date_offset_days = item.get("business_date_offset_days", 0)
|
||||
if (
|
||||
isinstance(business_date_offset_days, bool)
|
||||
or not isinstance(business_date_offset_days, int)
|
||||
or not -31 <= business_date_offset_days <= 31
|
||||
):
|
||||
raise CatalogError(f"invalid business date offset for {workflow_id}")
|
||||
return ScheduleEntry(
|
||||
workflow_id=workflow_id,
|
||||
kind=kind,
|
||||
@@ -205,7 +494,10 @@ def _parse_schedule(item: Any) -> ScheduleEntry:
|
||||
days=days,
|
||||
day_of_month=day_of_month,
|
||||
every_days=every_days,
|
||||
anchor_date=item.get("anchor_date"),
|
||||
anchor_date=anchor_date,
|
||||
enabled=enabled,
|
||||
business_date_offset_days=business_date_offset_days,
|
||||
at_times=at_times,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user