"""Validated workflow and schedule catalog.""" from __future__ import annotations import json import math import re from dataclasses import dataclass from datetime import date from pathlib import Path, PurePosixPath from typing import Any, Literal _WORKFLOW_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$") _SCHEDULE_TIME = re.compile(r"^(?:[01]\d|2[0-3]):[0-5]\d$") _MODULES = { "content_marketing", "product_commerce", "shop_intelligence", "supply_chain", } _TRIGGERS = {"scheduled", "manual", "unavailable"} _KINDS = {"daily", "weekly", "monthly", "interval_days"} _WEEKDAYS = {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"} class CatalogError(ValueError): """Raised for an invalid workflow catalog.""" @dataclass(frozen=True, slots=True) class WorkflowDataEndpoint: label: str system: str | None = None detail: str | None = None condition: str | None = None @dataclass(frozen=True, slots=True) class WorkflowDataFlow: sources: tuple[WorkflowDataEndpoint, ...] processing: tuple[str, ...] destinations: tuple[WorkflowDataEndpoint, ...] @dataclass(frozen=True, slots=True) class WorkflowTopicConfig: """Operator-facing topic input metadata for a workflow.""" default_keyword: str label: str = "话题关键词" hint: str = "输入完整话题或关键词,系统会在搜索结果中按包含关系选中。" @dataclass(frozen=True, slots=True) class WorkflowStepEntry: step_id: str entry: str name: str | None = None description: str | None = None args: tuple[str, ...] = () depends_on: tuple[str, ...] = () run_after_failure: bool = False timeout_seconds: float | None = None replay_policy: Literal["guarded", "idempotent", "repeatable"] | None = None data_flow: WorkflowDataFlow | None = None has_hyperlink: bool = False @dataclass(frozen=True, slots=True) class WorkflowEntry: workflow_id: str module: str trigger: Literal["scheduled", "manual", "unavailable"] entry: str args: tuple[str, ...] = () source_project: str | None = None source_task_name: str | None = None note: str | None = None steps: tuple[WorkflowStepEntry, ...] = () notification_workflow_id: str | None = None topic_config: WorkflowTopicConfig | None = None @dataclass(frozen=True, slots=True) class ScheduleEntry: workflow_id: str kind: Literal["daily", "weekly", "monthly", "interval_days"] at: str days: tuple[str, ...] = () day_of_month: int | None = None every_days: int | None = None anchor_date: str | None = None enabled: bool = True business_date_offset_days: int = 0 at_times: tuple[str, ...] = () @property def effective_times(self) -> tuple[str, ...]: """Return every configured wall-clock time, including legacy ``at``.""" return self.at_times or (self.at,) @dataclass(frozen=True, slots=True) class WorkflowCatalog: timezone: str workflows: tuple[WorkflowEntry, ...] schedules: tuple[ScheduleEntry, ...] @classmethod def load(cls, config_dir: Path | str) -> "WorkflowCatalog": root = Path(config_dir) workflow_payload = _load_json(root / "workflows.json") schedule_payload = _load_json(root / "schedules.json") _require_schema(workflow_payload, "workflows.json", expected=3) _require_schema(schedule_payload, "schedules.json", expected=1) workflows = tuple(_parse_workflow(item) for item in workflow_payload.get("workflows", [])) schedules = tuple(_parse_schedule(item) for item in schedule_payload.get("schedules", [])) timezone = schedule_payload.get("timezone") if not isinstance(timezone, str) or not timezone: raise CatalogError("schedules.json requires timezone") catalog = cls(timezone=timezone, workflows=workflows, schedules=schedules) catalog._validate_relations() return catalog def _validate_relations(self) -> None: workflow_ids = [item.workflow_id for item in self.workflows] duplicate_ids = _duplicates(workflow_ids) if duplicate_ids: raise CatalogError(f"duplicate workflow id: {sorted(duplicate_ids)[0]}") task_names = [item.source_task_name for item in self.scheduled_workflows()] duplicates = _duplicates([name for name in task_names if name]) if duplicates: raise CatalogError(f"duplicate legacy task name: {sorted(duplicates)[0]}") known = set(workflow_ids) schedule_ids = [item.workflow_id for item in self.schedules] duplicate_schedules = _duplicates(schedule_ids) if duplicate_schedules: raise CatalogError(f"duplicate schedule: {sorted(duplicate_schedules)[0]}") unknown = set(schedule_ids) - known if unknown: raise CatalogError(f"schedule references unknown workflow: {sorted(unknown)[0]}") scheduled_ids = {item.workflow_id for item in self.scheduled_workflows()} if set(schedule_ids) != scheduled_ids: missing = scheduled_ids - set(schedule_ids) extra = set(schedule_ids) - scheduled_ids raise CatalogError(f"schedule relation mismatch; missing={sorted(missing)}, extra={sorted(extra)}") def scheduled_workflows(self) -> tuple[WorkflowEntry, ...]: return tuple(item for item in self.workflows if item.trigger == "scheduled") def manual_workflows(self) -> tuple[WorkflowEntry, ...]: return tuple(item for item in self.workflows if item.trigger == "manual") def unavailable_workflows(self) -> tuple[WorkflowEntry, ...]: return tuple(item for item in self.workflows if item.trigger == "unavailable") def schedule_for(self, workflow_id: str) -> ScheduleEntry: for schedule in self.schedules: if schedule.workflow_id == workflow_id: return schedule raise KeyError(workflow_id) def _load_json(path: Path) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise CatalogError(f"cannot load {path.name}: {exc}") from exc if not isinstance(payload, dict): raise CatalogError(f"{path.name} must contain an object") return payload def _require_schema(payload: dict[str, Any], filename: str, *, expected: int) -> None: if payload.get("schema_version") != expected: raise CatalogError(f"unsupported schema_version in {filename}") def _parse_workflow(item: Any) -> WorkflowEntry: if not isinstance(item, dict): raise CatalogError("workflow entry must be an object") workflow_id = item.get("id") module = item.get("module") trigger = item.get("trigger") execution = item.get("execution") provenance = item.get("provenance", {}) if not isinstance(workflow_id, str) or not _WORKFLOW_ID.fullmatch(workflow_id): raise CatalogError(f"invalid workflow id: {workflow_id!r}") if module not in _MODULES: raise CatalogError(f"invalid module for {workflow_id}: {module!r}") if trigger not in _TRIGGERS: raise CatalogError(f"invalid trigger for {workflow_id}: {trigger!r}") if not isinstance(execution, dict): raise CatalogError(f"missing execution definition for {workflow_id}") if not isinstance(provenance, dict): raise CatalogError(f"invalid provenance definition for {workflow_id}") raw_steps = execution.get("steps") if raw_steps is None: entry = execution.get("entry") if not _is_relative_entry(entry): raise CatalogError(f"execution entry must be relative for {workflow_id}") args = _parse_args(execution.get("args", []), workflow_id) steps: tuple[WorkflowStepEntry, ...] = () else: if "entry" in execution or "args" in execution: raise CatalogError( f"execution cannot mix entry and steps for {workflow_id}" ) steps = _parse_workflow_steps(raw_steps, workflow_id) entry = steps[0].entry args = steps[0].args project = provenance.get("source_project") if project is not None and (not isinstance(project, str) or not project): raise CatalogError(f"invalid source project for {workflow_id}") task_name = provenance.get("task_name") if trigger == "scheduled" and (not isinstance(task_name, str) or not task_name): raise CatalogError(f"scheduled workflow requires source task name: {workflow_id}") topic_config = _parse_topic_config(item.get("topic_config"), workflow_id) return WorkflowEntry( workflow_id=workflow_id, module=module, trigger=trigger, entry=entry, args=tuple(args), source_project=project, source_task_name=task_name, note=item.get("note"), steps=steps, topic_config=topic_config, ) def _parse_topic_config(value: Any, workflow_id: str) -> WorkflowTopicConfig | None: if value is None: return None if not isinstance(value, dict): raise CatalogError(f"topic_config must be an object for {workflow_id}") default_keyword = value.get("default_keyword") if ( not isinstance(default_keyword, str) or not default_keyword.strip() or len(default_keyword.strip()) > 120 ): raise CatalogError( f"topic_config default_keyword is invalid for {workflow_id}" ) fields: dict[str, str] = {} for field, fallback in ( ("label", "话题关键词"), ("hint", "输入完整话题或关键词,系统会在搜索结果中按包含关系选中。"), ): raw = value.get(field, fallback) if not isinstance(raw, str) or not raw.strip() or len(raw.strip()) > 240: raise CatalogError(f"topic_config {field} is invalid for {workflow_id}") fields[field] = raw.strip() return WorkflowTopicConfig( default_keyword=default_keyword.strip(), label=fields["label"], hint=fields["hint"], ) def _parse_workflow_steps( raw_steps: Any, workflow_id: str, ) -> tuple[WorkflowStepEntry, ...]: if not isinstance(raw_steps, list) or not raw_steps: raise CatalogError(f"execution steps must be a non-empty list for {workflow_id}") steps: list[WorkflowStepEntry] = [] for raw_step in raw_steps: if not isinstance(raw_step, dict): raise CatalogError(f"workflow step must be an object for {workflow_id}") step_id = raw_step.get("id") entry = raw_step.get("entry") if not isinstance(step_id, str) or not _WORKFLOW_ID.fullmatch(step_id): raise CatalogError(f"invalid workflow step id for {workflow_id}: {step_id!r}") if not _is_relative_entry(entry): raise CatalogError( f"workflow step entry must be relative for {workflow_id}.{step_id}" ) depends_on = raw_step.get("depends_on", []) if not isinstance(depends_on, list) or not all( isinstance(value, str) and _WORKFLOW_ID.fullmatch(value) for value in depends_on ): raise CatalogError( f"workflow step dependencies are invalid for {workflow_id}.{step_id}" ) run_after_failure = raw_step.get("run_after_failure", False) if not isinstance(run_after_failure, bool): raise CatalogError( f"workflow step run_after_failure is invalid for {workflow_id}.{step_id}" ) timeout_seconds = raw_step.get("timeout_seconds") if timeout_seconds is not None and ( isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, (int, float)) or not math.isfinite(timeout_seconds) or timeout_seconds <= 0 ): raise CatalogError( f"workflow step timeout_seconds is invalid for {workflow_id}.{step_id}" ) replay_policy = raw_step.get("replay_policy") if replay_policy is not None and replay_policy not in { "guarded", "idempotent", "repeatable", }: raise CatalogError( f"workflow step replay_policy is invalid for {workflow_id}.{step_id}" ) name = raw_step.get("name") if name is not None and (not isinstance(name, str) or not name.strip()): raise CatalogError( f"workflow step name is invalid for {workflow_id}.{step_id}" ) description = raw_step.get("description") if description is not None and ( not isinstance(description, str) or not description.strip() ): raise CatalogError( f"workflow step description is invalid for {workflow_id}.{step_id}" ) has_hyperlink = raw_step.get("has_hyperlink", False) if not isinstance(has_hyperlink, bool): raise CatalogError( f"workflow step has_hyperlink is invalid for {workflow_id}.{step_id}" ) steps.append( WorkflowStepEntry( step_id=step_id, entry=entry, name=name.strip() if name is not None else None, description=description.strip() if description is not None else None, has_hyperlink=has_hyperlink, args=_parse_args(raw_step.get("args", []), workflow_id), depends_on=tuple(depends_on), run_after_failure=run_after_failure, timeout_seconds=( float(timeout_seconds) if timeout_seconds is not None else None ), replay_policy=replay_policy, data_flow=_parse_step_data_flow( raw_step.get("data_flow"), workflow_id, step_id, ), ) ) step_ids = [step.step_id for step in steps] duplicates = _duplicates(step_ids) if duplicates: raise CatalogError( f"duplicate workflow step id for {workflow_id}: {sorted(duplicates)[0]}" ) known = set(step_ids) for step in steps: unknown = set(step.depends_on) - known if unknown: raise CatalogError( f"unknown workflow step dependency for {workflow_id}.{step.step_id}: " f"{sorted(unknown)[0]}" ) _validate_step_graph(steps, workflow_id) return tuple(steps) def _parse_step_data_flow( value: Any, workflow_id: str, step_id: str, ) -> WorkflowDataFlow | None: if value is None: return None label = f"{workflow_id}.{step_id}" if not isinstance(value, dict): raise CatalogError(f"workflow step data_flow must be an object for {label}") allowed = {"sources", "processing", "destinations"} if set(value) != allowed: raise CatalogError(f"workflow step data_flow fields are invalid for {label}") processing = value.get("processing") if not isinstance(processing, list) or not processing or not all( isinstance(item, str) and item.strip() for item in processing ): raise CatalogError(f"workflow step processing is invalid for {label}") return WorkflowDataFlow( sources=_parse_data_endpoints(value.get("sources"), label, "sources"), processing=tuple(item.strip() for item in processing), destinations=_parse_data_endpoints( value.get("destinations"), label, "destinations", ), ) def _parse_data_endpoints( value: Any, step_label: str, field: str, ) -> tuple[WorkflowDataEndpoint, ...]: if not isinstance(value, list) or not value: raise CatalogError(f"workflow step {field} is invalid for {step_label}") endpoints: list[WorkflowDataEndpoint] = [] allowed = {"label", "system", "detail", "condition"} for item in value: if not isinstance(item, dict) or set(item) - allowed: raise CatalogError(f"workflow step {field} is invalid for {step_label}") endpoint_label = item.get("label") if not isinstance(endpoint_label, str) or not endpoint_label.strip(): raise CatalogError(f"workflow step {field} is invalid for {step_label}") optional: dict[str, str | None] = {} for key in ("system", "detail", "condition"): raw = item.get(key) if raw is not None and (not isinstance(raw, str) or not raw.strip()): raise CatalogError(f"workflow step {field} is invalid for {step_label}") optional[key] = raw.strip() if raw is not None else None endpoints.append( WorkflowDataEndpoint( label=endpoint_label.strip(), system=optional["system"], detail=optional["detail"], condition=optional["condition"], ) ) return tuple(endpoints) def _parse_args(value: Any, workflow_id: str) -> tuple[str, ...]: if not isinstance(value, list) or not all(isinstance(item, str) for item in value): raise CatalogError(f"execution args must be strings for {workflow_id}") return tuple(value) def _validate_step_graph( steps: list[WorkflowStepEntry], workflow_id: str, ) -> None: unresolved = {step.step_id: set(step.depends_on) for step in steps} completed: set[str] = set() while unresolved: ready = [ step_id for step_id, dependencies in unresolved.items() if dependencies <= completed ] if not ready: raise CatalogError(f"workflow step dependency cycle for {workflow_id}") completed.update(ready) for step_id in ready: del unresolved[step_id] def _is_relative_entry(entry: Any) -> bool: if not isinstance(entry, str) or not entry or "\\" in entry or ":" in entry: return False path = PurePosixPath(entry) return not path.is_absolute() and ".." not in path.parts def _parse_schedule(item: Any) -> ScheduleEntry: if not isinstance(item, dict): raise CatalogError("schedule entry must be an object") workflow_id = item.get("workflow_id") kind = item.get("kind") raw_at = item.get("at") if not isinstance(workflow_id, str) or not _WORKFLOW_ID.fullmatch(workflow_id): raise CatalogError(f"invalid schedule workflow id: {workflow_id!r}") if kind not in _KINDS: raise CatalogError(f"invalid schedule kind for {workflow_id}: {kind!r}") if isinstance(raw_at, str): if not _SCHEDULE_TIME.fullmatch(raw_at): raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}") at = raw_at at_times: tuple[str, ...] = () elif isinstance(raw_at, list): if ( not raw_at or not all( isinstance(value, str) and _SCHEDULE_TIME.fullmatch(value) for value in raw_at ) or len(raw_at) != len(set(raw_at)) ): raise CatalogError(f"invalid schedule times for {workflow_id}: {raw_at!r}") at_times = tuple(raw_at) at = at_times[0] else: raise CatalogError(f"invalid schedule time for {workflow_id}: {raw_at!r}") raw_days = item.get("days", []) if not isinstance(raw_days, list) or not all( isinstance(day, str) for day in raw_days ): raise CatalogError(f"invalid weekly days for {workflow_id}") days = tuple(raw_days) if kind == "weekly" and ( not days or len(days) != len(set(days)) or not set(days) <= _WEEKDAYS ): raise CatalogError(f"invalid weekly days for {workflow_id}") day_of_month = item.get("day_of_month") if kind == "monthly" and ( isinstance(day_of_month, bool) or not isinstance(day_of_month, int) or not 1 <= day_of_month <= 31 ): raise CatalogError(f"invalid day_of_month for {workflow_id}") every_days = item.get("every_days") if kind == "interval_days" and ( isinstance(every_days, bool) or not isinstance(every_days, int) or every_days < 1 ): raise CatalogError(f"invalid every_days for {workflow_id}") anchor_date = item.get("anchor_date") if kind == "interval_days": try: parsed_anchor = date.fromisoformat(anchor_date) except (TypeError, ValueError) as exc: raise CatalogError( f"invalid anchor_date for {workflow_id}: {anchor_date!r}" ) from exc if parsed_anchor.isoformat() != anchor_date: raise CatalogError( f"invalid anchor_date for {workflow_id}: {anchor_date!r}" ) enabled = item.get("enabled", True) if not isinstance(enabled, bool): raise CatalogError(f"invalid enabled flag for {workflow_id}") business_date_offset_days = item.get("business_date_offset_days", 0) if ( isinstance(business_date_offset_days, bool) or not isinstance(business_date_offset_days, int) or not -31 <= business_date_offset_days <= 31 ): raise CatalogError(f"invalid business date offset for {workflow_id}") return ScheduleEntry( workflow_id=workflow_id, kind=kind, at=at, days=days, day_of_month=day_of_month, every_days=every_days, anchor_date=anchor_date, enabled=enabled, business_date_offset_days=business_date_offset_days, at_times=at_times, ) def _duplicates(values: list[str]) -> set[str]: seen: set[str] = set() duplicates: set[str] = set() for value in values: if value in seen: duplicates.add(value) seen.add(value) return duplicates