Files
gyxx-flow/src/gyxx_flow/catalog.py
T

563 lines
21 KiB
Python

"""Validated workflow and schedule catalog."""
from __future__ import annotations
import json
import math
import re
from dataclasses import dataclass
from datetime import date
from pathlib import Path, PurePosixPath
from typing import Any, Literal
_WORKFLOW_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
_SCHEDULE_TIME = re.compile(r"^(?:[01]\d|2[0-3]):[0-5]\d$")
_MODULES = {
"content_marketing",
"product_commerce",
"shop_intelligence",
"supply_chain",
}
_TRIGGERS = {"scheduled", "manual", "unavailable"}
_KINDS = {"daily", "weekly", "monthly", "interval_days"}
_WEEKDAYS = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"}
class CatalogError(ValueError):
"""Raised for an invalid workflow catalog."""
@dataclass(frozen=True, slots=True)
class WorkflowDataEndpoint:
label: str
system: str | None = None
detail: str | None = None
condition: str | None = None
@dataclass(frozen=True, slots=True)
class WorkflowDataFlow:
sources: tuple[WorkflowDataEndpoint, ...]
processing: tuple[str, ...]
destinations: tuple[WorkflowDataEndpoint, ...]
@dataclass(frozen=True, slots=True)
class WorkflowTopicConfig:
"""Operator-facing topic input metadata for a workflow."""
default_keyword: str
label: str = "话题关键词"
hint: str = "输入完整话题或关键词,系统会在搜索结果中按包含关系选中。"
@dataclass(frozen=True, slots=True)
class WorkflowStepEntry:
step_id: str
entry: str
name: str | None = None
description: str | None = None
args: tuple[str, ...] = ()
depends_on: tuple[str, ...] = ()
run_after_failure: bool = False
timeout_seconds: float | None = None
replay_policy: Literal["guarded", "idempotent", "repeatable"] | None = None
data_flow: WorkflowDataFlow | None = None
has_hyperlink: bool = False
@dataclass(frozen=True, slots=True)
class WorkflowEntry:
workflow_id: str
module: str
trigger: Literal["scheduled", "manual", "unavailable"]
entry: str
args: tuple[str, ...] = ()
source_project: str | None = None
source_task_name: str | None = None
note: str | None = None
steps: tuple[WorkflowStepEntry, ...] = ()
notification_workflow_id: str | None = None
topic_config: WorkflowTopicConfig | None = None
@dataclass(frozen=True, slots=True)
class ScheduleEntry:
workflow_id: str
kind: Literal["daily", "weekly", "monthly", "interval_days"]
at: str
days: tuple[str, ...] = ()
day_of_month: int | None = None
every_days: int | None = None
anchor_date: str | None = None
enabled: bool = True
business_date_offset_days: int = 0
at_times: tuple[str, ...] = ()
@property
def effective_times(self) -> tuple[str, ...]:
"""Return every configured wall-clock time, including legacy ``at``."""
return self.at_times or (self.at,)
@dataclass(frozen=True, slots=True)
class WorkflowCatalog:
timezone: str
workflows: tuple[WorkflowEntry, ...]
schedules: tuple[ScheduleEntry, ...]
@classmethod
def load(cls, config_dir: Path | str) -> "WorkflowCatalog":
root = Path(config_dir)
workflow_payload = _load_json(root / "workflows.json")
schedule_payload = _load_json(root / "schedules.json")
_require_schema(workflow_payload, "workflows.json", expected=3)
_require_schema(schedule_payload, "schedules.json", expected=1)
workflows = tuple(_parse_workflow(item) for item in workflow_payload.get("workflows", []))
schedules = tuple(_parse_schedule(item) for item in schedule_payload.get("schedules", []))
timezone = schedule_payload.get("timezone")
if not isinstance(timezone, str) or not timezone:
raise CatalogError("schedules.json requires timezone")
catalog = cls(timezone=timezone, workflows=workflows, schedules=schedules)
catalog._validate_relations()
return catalog
def _validate_relations(self) -> None:
workflow_ids = [item.workflow_id for item in self.workflows]
duplicate_ids = _duplicates(workflow_ids)
if duplicate_ids:
raise CatalogError(f"duplicate workflow id: {sorted(duplicate_ids)[0]}")
task_names = [item.source_task_name for item in self.scheduled_workflows()]
duplicates = _duplicates([name for name in task_names if name])
if duplicates:
raise CatalogError(f"duplicate legacy task name: {sorted(duplicates)[0]}")
known = set(workflow_ids)
schedule_ids = [item.workflow_id for item in self.schedules]
duplicate_schedules = _duplicates(schedule_ids)
if duplicate_schedules:
raise CatalogError(f"duplicate schedule: {sorted(duplicate_schedules)[0]}")
unknown = set(schedule_ids) - known
if unknown:
raise CatalogError(f"schedule references unknown workflow: {sorted(unknown)[0]}")
scheduled_ids = {item.workflow_id for item in self.scheduled_workflows()}
if set(schedule_ids) != scheduled_ids:
missing = scheduled_ids - set(schedule_ids)
extra = set(schedule_ids) - scheduled_ids
raise CatalogError(f"schedule relation mismatch; missing={sorted(missing)}, extra={sorted(extra)}")
def scheduled_workflows(self) -> tuple[WorkflowEntry, ...]:
return tuple(item for item in self.workflows if item.trigger == "scheduled")
def manual_workflows(self) -> tuple[WorkflowEntry, ...]:
return tuple(item for item in self.workflows if item.trigger == "manual")
def unavailable_workflows(self) -> tuple[WorkflowEntry, ...]:
return tuple(item for item in self.workflows if item.trigger == "unavailable")
def schedule_for(self, workflow_id: str) -> ScheduleEntry:
for schedule in self.schedules:
if schedule.workflow_id == workflow_id:
return schedule
raise KeyError(workflow_id)
def _load_json(path: Path) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise CatalogError(f"cannot load {path.name}: {exc}") from exc
if not isinstance(payload, dict):
raise CatalogError(f"{path.name} must contain an object")
return payload
def _require_schema(payload: dict[str, Any], filename: str, *, expected: int) -> None:
if payload.get("schema_version") != expected:
raise CatalogError(f"unsupported schema_version in {filename}")
def _parse_workflow(item: Any) -> WorkflowEntry:
if not isinstance(item, dict):
raise CatalogError("workflow entry must be an object")
workflow_id = item.get("id")
module = item.get("module")
trigger = item.get("trigger")
execution = item.get("execution")
provenance = item.get("provenance", {})
if not isinstance(workflow_id, str) or not _WORKFLOW_ID.fullmatch(workflow_id):
raise CatalogError(f"invalid workflow id: {workflow_id!r}")
if module not in _MODULES:
raise CatalogError(f"invalid module for {workflow_id}: {module!r}")
if trigger not in _TRIGGERS:
raise CatalogError(f"invalid trigger for {workflow_id}: {trigger!r}")
if not isinstance(execution, dict):
raise CatalogError(f"missing execution definition for {workflow_id}")
if not isinstance(provenance, dict):
raise CatalogError(f"invalid provenance definition for {workflow_id}")
raw_steps = execution.get("steps")
if raw_steps is None:
entry = execution.get("entry")
if not _is_relative_entry(entry):
raise CatalogError(f"execution entry must be relative for {workflow_id}")
args = _parse_args(execution.get("args", []), workflow_id)
steps: tuple[WorkflowStepEntry, ...] = ()
else:
if "entry" in execution or "args" in execution:
raise CatalogError(
f"execution cannot mix entry and steps for {workflow_id}"
)
steps = _parse_workflow_steps(raw_steps, workflow_id)
entry = steps[0].entry
args = steps[0].args
project = provenance.get("source_project")
if project is not None and (not isinstance(project, str) or not project):
raise CatalogError(f"invalid source project for {workflow_id}")
task_name = provenance.get("task_name")
if trigger == "scheduled" and (not isinstance(task_name, str) or not task_name):
raise CatalogError(f"scheduled workflow requires source task name: {workflow_id}")
topic_config = _parse_topic_config(item.get("topic_config"), workflow_id)
return WorkflowEntry(
workflow_id=workflow_id,
module=module,
trigger=trigger,
entry=entry,
args=tuple(args),
source_project=project,
source_task_name=task_name,
note=item.get("note"),
steps=steps,
topic_config=topic_config,
)
def _parse_topic_config(value: Any, workflow_id: str) -> WorkflowTopicConfig | None:
if value is None:
return None
if not isinstance(value, dict):
raise CatalogError(f"topic_config must be an object for {workflow_id}")
default_keyword = value.get("default_keyword")
if (
not isinstance(default_keyword, str)
or not default_keyword.strip()
or len(default_keyword.strip()) > 120
):
raise CatalogError(
f"topic_config default_keyword is invalid for {workflow_id}"
)
fields: dict[str, str] = {}
for field, fallback in (
("label", "话题关键词"),
("hint", "输入完整话题或关键词,系统会在搜索结果中按包含关系选中。"),
):
raw = value.get(field, fallback)
if not isinstance(raw, str) or not raw.strip() or len(raw.strip()) > 240:
raise CatalogError(f"topic_config {field} is invalid for {workflow_id}")
fields[field] = raw.strip()
return WorkflowTopicConfig(
default_keyword=default_keyword.strip(),
label=fields["label"],
hint=fields["hint"],
)
def _parse_workflow_steps(
raw_steps: Any,
workflow_id: str,
) -> tuple[WorkflowStepEntry, ...]:
if not isinstance(raw_steps, list) or not raw_steps:
raise CatalogError(f"execution steps must be a non-empty list for {workflow_id}")
steps: list[WorkflowStepEntry] = []
for raw_step in raw_steps:
if not isinstance(raw_step, dict):
raise CatalogError(f"workflow step must be an object for {workflow_id}")
step_id = raw_step.get("id")
entry = raw_step.get("entry")
if not isinstance(step_id, str) or not _WORKFLOW_ID.fullmatch(step_id):
raise CatalogError(f"invalid workflow step id for {workflow_id}: {step_id!r}")
if not _is_relative_entry(entry):
raise CatalogError(
f"workflow step entry must be relative for {workflow_id}.{step_id}"
)
depends_on = raw_step.get("depends_on", [])
if not isinstance(depends_on, list) or not all(
isinstance(value, str) and _WORKFLOW_ID.fullmatch(value)
for value in depends_on
):
raise CatalogError(
f"workflow step dependencies are invalid for {workflow_id}.{step_id}"
)
run_after_failure = raw_step.get("run_after_failure", False)
if not isinstance(run_after_failure, bool):
raise CatalogError(
f"workflow step run_after_failure is invalid for {workflow_id}.{step_id}"
)
timeout_seconds = raw_step.get("timeout_seconds")
if timeout_seconds is not None and (
isinstance(timeout_seconds, bool)
or not isinstance(timeout_seconds, (int, float))
or not math.isfinite(timeout_seconds)
or timeout_seconds <= 0
):
raise CatalogError(
f"workflow step timeout_seconds is invalid for {workflow_id}.{step_id}"
)
replay_policy = raw_step.get("replay_policy")
if replay_policy is not None and replay_policy not in {
"guarded",
"idempotent",
"repeatable",
}:
raise CatalogError(
f"workflow step replay_policy is invalid for {workflow_id}.{step_id}"
)
name = raw_step.get("name")
if name is not None and (not isinstance(name, str) or not name.strip()):
raise CatalogError(
f"workflow step name is invalid for {workflow_id}.{step_id}"
)
description = raw_step.get("description")
if description is not None and (
not isinstance(description, str) or not description.strip()
):
raise CatalogError(
f"workflow step description is invalid for {workflow_id}.{step_id}"
)
has_hyperlink = raw_step.get("has_hyperlink", False)
if not isinstance(has_hyperlink, bool):
raise CatalogError(
f"workflow step has_hyperlink is invalid for {workflow_id}.{step_id}"
)
steps.append(
WorkflowStepEntry(
step_id=step_id,
entry=entry,
name=name.strip() if name is not None else None,
description=description.strip() if description is not None else None,
has_hyperlink=has_hyperlink,
args=_parse_args(raw_step.get("args", []), workflow_id),
depends_on=tuple(depends_on),
run_after_failure=run_after_failure,
timeout_seconds=(
float(timeout_seconds) if timeout_seconds is not None else None
),
replay_policy=replay_policy,
data_flow=_parse_step_data_flow(
raw_step.get("data_flow"),
workflow_id,
step_id,
),
)
)
step_ids = [step.step_id for step in steps]
duplicates = _duplicates(step_ids)
if duplicates:
raise CatalogError(
f"duplicate workflow step id for {workflow_id}: {sorted(duplicates)[0]}"
)
known = set(step_ids)
for step in steps:
unknown = set(step.depends_on) - known
if unknown:
raise CatalogError(
f"unknown workflow step dependency for {workflow_id}.{step.step_id}: "
f"{sorted(unknown)[0]}"
)
_validate_step_graph(steps, workflow_id)
return tuple(steps)
def _parse_step_data_flow(
value: Any,
workflow_id: str,
step_id: str,
) -> WorkflowDataFlow | None:
if value is None:
return None
label = f"{workflow_id}.{step_id}"
if not isinstance(value, dict):
raise CatalogError(f"workflow step data_flow must be an object for {label}")
allowed = {"sources", "processing", "destinations"}
if set(value) != allowed:
raise CatalogError(f"workflow step data_flow fields are invalid for {label}")
processing = value.get("processing")
if not isinstance(processing, list) or not processing or not all(
isinstance(item, str) and item.strip() for item in processing
):
raise CatalogError(f"workflow step processing is invalid for {label}")
return WorkflowDataFlow(
sources=_parse_data_endpoints(value.get("sources"), label, "sources"),
processing=tuple(item.strip() for item in processing),
destinations=_parse_data_endpoints(
value.get("destinations"),
label,
"destinations",
),
)
def _parse_data_endpoints(
value: Any,
step_label: str,
field: str,
) -> tuple[WorkflowDataEndpoint, ...]:
if not isinstance(value, list) or not value:
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
endpoints: list[WorkflowDataEndpoint] = []
allowed = {"label", "system", "detail", "condition"}
for item in value:
if not isinstance(item, dict) or set(item) - allowed:
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
endpoint_label = item.get("label")
if not isinstance(endpoint_label, str) or not endpoint_label.strip():
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
optional: dict[str, str | None] = {}
for key in ("system", "detail", "condition"):
raw = item.get(key)
if raw is not None and (not isinstance(raw, str) or not raw.strip()):
raise CatalogError(f"workflow step {field} is invalid for {step_label}")
optional[key] = raw.strip() if raw is not None else None
endpoints.append(
WorkflowDataEndpoint(
label=endpoint_label.strip(),
system=optional["system"],
detail=optional["detail"],
condition=optional["condition"],
)
)
return tuple(endpoints)
def _parse_args(value: Any, workflow_id: str) -> tuple[str, ...]:
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise CatalogError(f"execution args must be strings for {workflow_id}")
return tuple(value)
def _validate_step_graph(
steps: list[WorkflowStepEntry],
workflow_id: str,
) -> None:
unresolved = {step.step_id: set(step.depends_on) for step in steps}
completed: set[str] = set()
while unresolved:
ready = [
step_id
for step_id, dependencies in unresolved.items()
if dependencies <= completed
]
if not ready:
raise CatalogError(f"workflow step dependency cycle for {workflow_id}")
completed.update(ready)
for step_id in ready:
del unresolved[step_id]
def _is_relative_entry(entry: Any) -> bool:
if not isinstance(entry, str) or not entry or "\\" in entry or ":" in entry:
return False
path = PurePosixPath(entry)
return not path.is_absolute() and ".." not in path.parts
def _parse_schedule(item: Any) -> ScheduleEntry:
if not isinstance(item, dict):
raise CatalogError("schedule entry must be an object")
workflow_id = item.get("workflow_id")
kind = item.get("kind")
raw_at = item.get("at")
if not isinstance(workflow_id, str) or not _WORKFLOW_ID.fullmatch(workflow_id):
raise CatalogError(f"invalid schedule workflow id: {workflow_id!r}")
if kind not in _KINDS:
raise CatalogError(f"invalid schedule kind for {workflow_id}: {kind!r}")
if isinstance(raw_at, str):
if not _SCHEDULE_TIME.fullmatch(raw_at):
raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}")
at = raw_at
at_times: tuple[str, ...] = ()
elif isinstance(raw_at, list):
if (
not raw_at
or not all(
isinstance(value, str) and _SCHEDULE_TIME.fullmatch(value)
for value in raw_at
)
or len(raw_at) != len(set(raw_at))
):
raise CatalogError(f"invalid schedule times for {workflow_id}: {raw_at!r}")
at_times = tuple(raw_at)
at = at_times[0]
else:
raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}")
raw_days = item.get("days", [])
if not isinstance(raw_days, list) or not all(
isinstance(day, str) for day in raw_days
):
raise CatalogError(f"invalid weekly days for {workflow_id}")
days = tuple(raw_days)
if kind == "weekly" and (
not days
or len(days) != len(set(days))
or not set(days) <= _WEEKDAYS
):
raise CatalogError(f"invalid weekly days for {workflow_id}")
day_of_month = item.get("day_of_month")
if kind == "monthly" and (
isinstance(day_of_month, bool)
or not isinstance(day_of_month, int)
or not 1 <= day_of_month <= 31
):
raise CatalogError(f"invalid day_of_month for {workflow_id}")
every_days = item.get("every_days")
if kind == "interval_days" and (
isinstance(every_days, bool)
or not isinstance(every_days, int)
or every_days < 1
):
raise CatalogError(f"invalid every_days for {workflow_id}")
anchor_date = item.get("anchor_date")
if kind == "interval_days":
try:
parsed_anchor = date.fromisoformat(anchor_date)
except (TypeError, ValueError) as exc:
raise CatalogError(
f"invalid anchor_date for {workflow_id}: {anchor_date!r}"
) from exc
if parsed_anchor.isoformat() != anchor_date:
raise CatalogError(
f"invalid anchor_date for {workflow_id}: {anchor_date!r}"
)
enabled = item.get("enabled", True)
if not isinstance(enabled, bool):
raise CatalogError(f"invalid enabled flag for {workflow_id}")
business_date_offset_days = item.get("business_date_offset_days", 0)
if (
isinstance(business_date_offset_days, bool)
or not isinstance(business_date_offset_days, int)
or not -31 <= business_date_offset_days <= 31
):
raise CatalogError(f"invalid business date offset for {workflow_id}")
return ScheduleEntry(
workflow_id=workflow_id,
kind=kind,
at=at,
days=days,
day_of_month=day_of_month,
every_days=every_days,
anchor_date=anchor_date,
enabled=enabled,
business_date_offset_days=business_date_offset_days,
at_times=at_times,
)
def _duplicates(values: list[str]) -> set[str]:
seen: set[str] = set()
duplicates: set[str] = set()
for value in values:
if value in seen:
duplicates.add(value)
seen.add(value)
return duplicates