feat: consolidate legacy workflows into gyxx-flow

src/gyxx_flow/ops/__init__.py

@@ -0,0 +1,22 @@
+"""Operational run indexing and durable outbox services."""
+
+from .effects import (
+    EffectAlreadyApplied,
+    EffectClaim,
+    EffectLedger,
+    EffectStateAmbiguous,
+)
+from .index import RunIndex, RunRecord
+from .outbox import IdempotencyConflict, Outbox, OutboxMessage
+
+__all__ = [
+    "IdempotencyConflict",
+    "EffectAlreadyApplied",
+    "EffectClaim",
+    "EffectLedger",
+    "EffectStateAmbiguous",
+    "Outbox",
+    "OutboxMessage",
+    "RunIndex",
+    "RunRecord",
+]

src/gyxx_flow/ops/effects.py

@@ -0,0 +1,150 @@
+"""Crash-safe idempotency receipts for opaque production side effects."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+from gyxx_flow.core.artifacts import atomic_write_json
+from gyxx_flow.core.locks import LockManager
+
+_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+_RUN_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
+
+
+class EffectAlreadyApplied(RuntimeError):
+    """The logical production effect already has a success receipt."""
+
+
+class EffectStateAmbiguous(RuntimeError):
+    """A prior attempt may have produced an external effect."""
+
+
+@dataclass(frozen=True, slots=True)
+class EffectClaim:
+    effect_id: str
+    workflow_id: str
+    business_date: str
+    step_id: str
+    run_id: str
+
+
+class EffectLedger:
+    """Prevent automatic replay when an opaque sink may already have written."""
+
+    def __init__(self, data_root: Path | str) -> None:
+        self._data_root = Path(data_root).expanduser().resolve()
+        self._root = self._data_root / "state" / "ops" / "effects"
+        self._locks = LockManager(self._data_root / "state" / "locks")
+
+    def begin(
+        self,
+        *,
+        workflow_id: str,
+        business_date: str,
+        step_id: str,
+        run_id: str,
+    ) -> EffectClaim:
+        _validate(workflow_id, business_date, step_id, run_id)
+        effect_id = _effect_id(workflow_id, business_date, step_id)
+        claim = EffectClaim(effect_id, workflow_id, business_date, step_id, run_id)
+        with self._lock(effect_id):
+            path = self._path(effect_id)
+            if path.exists():
+                payload = self._read(path)
+                if payload["status"] == "applied":
+                    raise EffectAlreadyApplied(
+                        f"effect already applied: {workflow_id}/{business_date}/{step_id}"
+                    )
+                raise EffectStateAmbiguous(
+                    f"effect requires reconciliation: {workflow_id}/{business_date}/{step_id}"
+                )
+            now = _now()
+            atomic_write_json(
+                path,
+                {
+                    "schema_version": 1,
+                    "effect_id": effect_id,
+                    "workflow_id": workflow_id,
+                    "business_date": business_date,
+                    "step_id": step_id,
+                    "run_id": run_id,
+                    "status": "in_progress",
+                    "claimed_at": now,
+                    "updated_at": now,
+                    "reason": None,
+                },
+            )
+        return claim
+
+    def mark_applied(self, claim: EffectClaim) -> None:
+        self._finish(claim, status="applied", reason=None)
+
+    def mark_ambiguous(self, claim: EffectClaim, *, reason: str) -> None:
+        if not isinstance(reason, str) or not reason or len(reason) > 200:
+            raise ValueError("invalid effect ambiguity reason")
+        self._finish(claim, status="ambiguous", reason=reason)
+
+    def _finish(self, claim: EffectClaim, *, status: str, reason: str | None) -> None:
+        with self._lock(claim.effect_id):
+            path = self._path(claim.effect_id)
+            payload = self._read(path)
+            if (
+                payload["status"] != "in_progress"
+                or payload["run_id"] != claim.run_id
+            ):
+                raise EffectStateAmbiguous("effect claim no longer owns the receipt")
+            payload.update(status=status, reason=reason, updated_at=_now())
+            atomic_write_json(path, payload)
+
+    def _path(self, effect_id: str) -> Path:
+        return self._root / f"{effect_id}.json"
+
+    def _lock(self, effect_id: str):  # type: ignore[no-untyped-def]
+        return self._locks.acquire(
+            f"ops:effect:{effect_id}",
+            owner=f"effect-{uuid.uuid4().hex}",
+            timeout_seconds=30,
+        )
+
+    @staticmethod
+    def _read(path: Path) -> dict[str, object]:
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise EffectStateAmbiguous("effect receipt is unreadable") from exc
+        if (
+            not isinstance(payload, dict)
+            or payload.get("schema_version") != 1
+            or payload.get("status") not in {"in_progress", "applied", "ambiguous"}
+        ):
+            raise EffectStateAmbiguous("effect receipt is invalid")
+        return payload
+
+
+def _effect_id(workflow_id: str, business_date: str, step_id: str) -> str:
+    key = f"{workflow_id}\0{business_date}\0{step_id}"
+    return "effect_" + hashlib.sha256(key.encode("utf-8")).hexdigest()
+
+
+def _validate(
+    workflow_id: str, business_date: str, step_id: str, run_id: str
+) -> None:
+    if not _ID.fullmatch(workflow_id) or not _ID.fullmatch(step_id):
+        raise ValueError("invalid effect identity")
+    try:
+        parsed = date.fromisoformat(business_date)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("invalid effect business date") from exc
+    if parsed.isoformat() != business_date or not _RUN_ID.fullmatch(run_id):
+        raise ValueError("invalid effect identity")
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+

src/gyxx_flow/ops/index.py

@@ -0,0 +1,241 @@
+"""Durable, queryable snapshots of workflow run journals."""
+
+from __future__ import annotations
+
+import json
+import re
+import uuid
+from dataclasses import asdict, dataclass
+from datetime import date, datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from gyxx_flow.core.artifacts import atomic_write_json
+from gyxx_flow.core.locks import LockManager
+from gyxx_flow.core.records import RunJournal
+
+_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+_RUN_STATUSES = frozenset({"running", "success", "failed", "cancelled"})
+_STEP_STATUSES = frozenset({"running", "success", "failed", "skipped"})
+
+
+@dataclass(frozen=True, slots=True)
+class RunRecord:
+    """A compact, non-secret operational projection of a run journal."""
+
+    workflow_id: str
+    run_id: str
+    business_date: str
+    shadow: bool
+    status: str
+    started_at: str
+    ended_at: str | None
+    error: str | None
+    step_counts: dict[str, int]
+    journal_path: str
+    indexed_at: str
+    schema_version: int = 1
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> "RunRecord":
+        try:
+            record = cls(
+                workflow_id=payload["workflow_id"],
+                run_id=payload["run_id"],
+                business_date=payload["business_date"],
+                shadow=payload["shadow"],
+                status=payload["status"],
+                started_at=payload["started_at"],
+                ended_at=payload["ended_at"],
+                error=payload["error"],
+                step_counts=dict(payload["step_counts"]),
+                journal_path=payload["journal_path"],
+                indexed_at=payload["indexed_at"],
+                schema_version=payload["schema_version"],
+            )
+        except (KeyError, TypeError, ValueError) as exc:
+            raise ValueError("invalid run index record") from exc
+        _validate_record(record)
+        return record
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class RunIndex:
+    """Per-run sharded index built from :class:`RunJournal` JSON files."""
+
+    def __init__(self, data_root: Path | str, *, lock_manager: LockManager | None = None) -> None:
+        self._data_root = Path(data_root).expanduser().resolve()
+        self._root = self._data_root / "state" / "ops" / "run-index"
+        self._locks = lock_manager or LockManager(self._data_root / "state" / "locks")
+
+    def index_journal(self, journal: RunJournal | Path | str) -> RunRecord:
+        journal_path = journal.path if isinstance(journal, RunJournal) else Path(journal)
+        journal_path = journal_path.expanduser().resolve()
+        try:
+            raw = json.loads(journal_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ValueError("invalid run journal") from exc
+        record = _record_from_journal(raw, journal_path)
+        destination = self._record_path(record.run_id)
+        resource = f"ops:run-index:{_digest(record.run_id)}"
+        with self._locks.acquire(resource, owner=_owner(), timeout_seconds=30):
+            atomic_write_json(destination, record.to_dict())
+        return record
+
+    def get(self, run_id: str) -> RunRecord | None:
+        path = self._record_path(run_id)
+        if not path.exists():
+            return None
+        return _read_record(path)
+
+    def query(
+        self,
+        *,
+        workflow_id: str | None = None,
+        status: str | None = None,
+        business_date: str | None = None,
+        run_id: str | None = None,
+    ) -> tuple[RunRecord, ...]:
+        if workflow_id is not None:
+            _require_safe_id(workflow_id, "workflow_id")
+        if run_id is not None:
+            _require_safe_id(run_id, "run_id")
+        if status is not None and status not in _RUN_STATUSES:
+            raise ValueError(f"invalid run status: {status!r}")
+        if business_date is not None:
+            _require_business_date(business_date)
+
+        records = []
+        for path in sorted(self._root.glob("*.json")):
+            record = _read_record(path)
+            if workflow_id is not None and record.workflow_id != workflow_id:
+                continue
+            if status is not None and record.status != status:
+                continue
+            if business_date is not None and record.business_date != business_date:
+                continue
+            if run_id is not None and record.run_id != run_id:
+                continue
+            records.append(record)
+        records.sort(key=lambda item: (item.started_at, item.run_id), reverse=True)
+        return tuple(records)
+
+    def _record_path(self, run_id: str) -> Path:
+        _require_safe_id(run_id, "run_id")
+        return self._root / f"{run_id}.json"
+
+
+def _record_from_journal(payload: Any, journal_path: Path) -> RunRecord:
+    if not isinstance(payload, dict):
+        raise ValueError("invalid run journal")
+    required = {
+        "workflow_id",
+        "run_id",
+        "business_date",
+        "shadow",
+        "started_at",
+        "ended_at",
+        "status",
+        "error",
+        "steps",
+    }
+    if not required.issubset(payload):
+        raise ValueError("invalid run journal")
+    try:
+        workflow_id = payload["workflow_id"]
+        run_id = payload["run_id"]
+        business_date = payload["business_date"]
+        status = payload["status"]
+        shadow = payload["shadow"]
+        started_at = payload["started_at"]
+        ended_at = payload["ended_at"]
+        error = payload["error"]
+        steps = payload["steps"]
+        _require_safe_id(workflow_id, "workflow_id")
+        _require_safe_id(run_id, "run_id")
+        _require_business_date(business_date)
+        if status not in _RUN_STATUSES:
+            raise ValueError("invalid status")
+        if not isinstance(shadow, bool) or not isinstance(started_at, str):
+            raise ValueError("invalid journal fields")
+        if ended_at is not None and not isinstance(ended_at, str):
+            raise ValueError("invalid ended_at")
+        if error is not None and not isinstance(error, str):
+            raise ValueError("invalid error")
+        if not isinstance(steps, dict):
+            raise ValueError("invalid steps")
+        counts = {name: 0 for name in ("success", "failed", "skipped", "running")}
+        for step in steps.values():
+            if not isinstance(step, dict) or step.get("status") not in _STEP_STATUSES:
+                raise ValueError("invalid step")
+            counts[step["status"]] += 1
+    except (TypeError, ValueError) as exc:
+        raise ValueError("invalid run journal") from exc
+
+    record = RunRecord(
+        workflow_id=workflow_id,
+        run_id=run_id,
+        business_date=business_date,
+        shadow=shadow,
+        status=status,
+        started_at=started_at,
+        ended_at=ended_at,
+        error=error,
+        step_counts=counts,
+        journal_path=str(journal_path),
+        indexed_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+    )
+    _validate_record(record)
+    return record
+
+
+def _read_record(path: Path) -> RunRecord:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"invalid run index record: {path.name}") from exc
+    if not isinstance(payload, dict):
+        raise ValueError(f"invalid run index record: {path.name}")
+    return RunRecord.from_dict(payload)
+
+
+def _validate_record(record: RunRecord) -> None:
+    _require_safe_id(record.workflow_id, "workflow_id")
+    _require_safe_id(record.run_id, "run_id")
+    _require_business_date(record.business_date)
+    if record.status not in _RUN_STATUSES or record.schema_version != 1:
+        raise ValueError("invalid run index record")
+    if not isinstance(record.shadow, bool) or not isinstance(record.started_at, str):
+        raise ValueError("invalid run index record")
+    if set(record.step_counts) != _STEP_STATUSES:
+        raise ValueError("invalid run index record")
+    if any(not isinstance(value, int) or value < 0 for value in record.step_counts.values()):
+        raise ValueError("invalid run index record")
+
+
+def _require_safe_id(value: Any, name: str) -> None:
+    if not isinstance(value, str) or not _SAFE_ID.fullmatch(value):
+        raise ValueError(f"invalid {name}")
+
+
+def _require_business_date(value: Any) -> None:
+    if not isinstance(value, str):
+        raise ValueError("invalid business_date")
+    try:
+        parsed = date.fromisoformat(value)
+    except ValueError as exc:
+        raise ValueError("invalid business_date") from exc
+    if parsed.isoformat() != value:
+        raise ValueError("invalid business_date")
+
+
+def _owner() -> str:
+    return f"ops-{uuid.uuid4().hex}"
+
+
+def _digest(value: str) -> str:
+    import hashlib
+
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()[:32]

src/gyxx_flow/ops/outbox.py

@@ -0,0 +1,287 @@
+"""Atomic local-file transactional outbox with idempotent replay semantics."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import asdict, dataclass, replace
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from gyxx_flow.core.artifacts import atomic_write_json
+from gyxx_flow.core.locks import LockManager
+
+_TOPIC = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+_MESSAGE_ID = re.compile(r"^msg_[0-9a-f]{64}$")
+_STATUSES = frozenset({"pending", "sent", "failed"})
+
+
+class IdempotencyConflict(ValueError):
+    """Raised when a key is reused for a different logical message."""
+
+
+@dataclass(frozen=True, slots=True)
+class OutboxMessage:
+    schema_version: int
+    message_id: str
+    idempotency_key: str
+    topic: str
+    payload: Any
+    run_id: str
+    status: str
+    attempts: int
+    created_at: str
+    updated_at: str
+    sent_at: str | None
+    error: str | None
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> "OutboxMessage":
+        try:
+            message = cls(**payload)
+        except (TypeError, ValueError) as exc:
+            raise ValueError("invalid outbox message") from exc
+        _validate_message(message)
+        return message
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class Outbox:
+    """A durable, at-least-once outbox.
+
+    Dispatchers must forward ``idempotency_key`` to the destination. A sent
+    message is never dispatched again; a failed message is replayed under the
+    same message ID and idempotency key.
+    """
+
+    def __init__(self, data_root: Path | str, *, lock_manager: LockManager | None = None) -> None:
+        self._data_root = Path(data_root).expanduser().resolve()
+        self._root = self._data_root / "state" / "ops" / "outbox" / "messages"
+        self._locks = lock_manager or LockManager(self._data_root / "state" / "locks")
+
+    def enqueue(
+        self,
+        *,
+        idempotency_key: str,
+        topic: str,
+        payload: Any,
+        run_id: str,
+    ) -> OutboxMessage:
+        _validate_new_message(idempotency_key, topic, payload, run_id)
+        normalized_payload = _json_copy(payload)
+        message_id = _message_id(idempotency_key)
+        path = self.message_path(message_id)
+        with self._message_lock(message_id):
+            if path.exists():
+                existing = self._read(path)
+                if (
+                    existing.idempotency_key != idempotency_key
+                    or existing.topic != topic
+                    or existing.payload != normalized_payload
+                    or existing.run_id != run_id
+                ):
+                    raise IdempotencyConflict(
+                        "idempotency key is already bound to a different message"
+                    )
+                return existing
+            now = _now()
+            message = OutboxMessage(
+                schema_version=1,
+                message_id=message_id,
+                idempotency_key=idempotency_key,
+                topic=topic,
+                payload=normalized_payload,
+                run_id=run_id,
+                status="pending",
+                attempts=0,
+                created_at=now,
+                updated_at=now,
+                sent_at=None,
+                error=None,
+            )
+            atomic_write_json(path, message.to_dict())
+            return message
+
+    def get(self, message_id: str) -> OutboxMessage | None:
+        path = self.message_path(message_id)
+        if not path.exists():
+            return None
+        return self._read(path)
+
+    def list(self, *, status: str | None = None) -> tuple[OutboxMessage, ...]:
+        if status is not None and status not in _STATUSES:
+            raise ValueError(f"invalid outbox status: {status!r}")
+        messages = []
+        for path in sorted(self._root.glob("*.json")):
+            message = self._read(path)
+            if status is None or message.status == status:
+                messages.append(message)
+        messages.sort(key=lambda item: (item.created_at, item.message_id))
+        return tuple(messages)
+
+    def mark_sent(self, message_id: str) -> OutboxMessage:
+        with self._message_lock(message_id):
+            message = self._require(message_id)
+            if message.status == "sent":
+                return message
+            if message.status != "pending":
+                raise ValueError(f"cannot mark {message.status} message as sent; replay it first")
+            return self._write_sent(message)
+
+    def mark_failed(self, message_id: str, *, error: str) -> OutboxMessage:
+        if not isinstance(error, str) or not error.strip():
+            raise ValueError("error must be a non-empty string")
+        with self._message_lock(message_id):
+            message = self._require(message_id)
+            if message.status != "pending":
+                raise ValueError(f"cannot mark {message.status} message as failed")
+            return self._write_failed(message, error=error)
+
+    def replay(self, message_id: str) -> OutboxMessage:
+        with self._message_lock(message_id):
+            message = self._require(message_id)
+            if message.status in {"pending", "sent"}:
+                return message
+            replayed = replace(
+                message,
+                status="pending",
+                updated_at=_now(),
+                sent_at=None,
+                error=None,
+            )
+            atomic_write_json(self.message_path(message_id), replayed.to_dict())
+            return replayed
+
+    def dispatch(
+        self,
+        message_id: str,
+        sender: Callable[[OutboxMessage], object],
+    ) -> OutboxMessage:
+        """Dispatch one pending message while holding its per-message lock.
+
+        ``sender`` must make its downstream operation idempotent using the key
+        on the supplied message. Exceptions are persisted as ``failed``.
+        """
+
+        if not callable(sender):
+            raise TypeError("sender must be callable")
+        with self._message_lock(message_id):
+            message = self._require(message_id)
+            if message.status == "sent":
+                return message
+            if message.status != "pending":
+                raise ValueError(f"cannot dispatch {message.status} message; replay it first")
+            try:
+                sender(message)
+            except Exception as exc:
+                return self._write_failed(message, error=f"{type(exc).__name__}: {exc}")
+            return self._write_sent(message)
+
+    def message_path(self, message_id: str) -> Path:
+        if not isinstance(message_id, str) or not _MESSAGE_ID.fullmatch(message_id):
+            raise ValueError("invalid message_id")
+        return self._root / f"{message_id}.json"
+
+    def _message_lock(self, message_id: str):  # type: ignore[no-untyped-def]
+        if not isinstance(message_id, str) or not _MESSAGE_ID.fullmatch(message_id):
+            raise ValueError("invalid message_id")
+        return self._locks.acquire(
+            f"ops:outbox:{message_id}", owner=f"ops-{uuid.uuid4().hex}", timeout_seconds=30
+        )
+
+    def _require(self, message_id: str) -> OutboxMessage:
+        path = self.message_path(message_id)
+        if not path.exists():
+            raise KeyError(f"unknown outbox message: {message_id}")
+        return self._read(path)
+
+    def _read(self, path: Path) -> OutboxMessage:
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise ValueError(f"invalid outbox message: {path.name}") from exc
+        if not isinstance(payload, dict):
+            raise ValueError(f"invalid outbox message: {path.name}")
+        return OutboxMessage.from_dict(payload)
+
+    def _write_sent(self, message: OutboxMessage) -> OutboxMessage:
+        now = _now()
+        sent = replace(
+            message,
+            status="sent",
+            attempts=message.attempts + 1,
+            updated_at=now,
+            sent_at=now,
+            error=None,
+        )
+        atomic_write_json(self.message_path(message.message_id), sent.to_dict())
+        return sent
+
+    def _write_failed(self, message: OutboxMessage, *, error: str) -> OutboxMessage:
+        failed = replace(
+            message,
+            status="failed",
+            attempts=message.attempts + 1,
+            updated_at=_now(),
+            sent_at=None,
+            error=error,
+        )
+        atomic_write_json(self.message_path(message.message_id), failed.to_dict())
+        return failed
+
+
+def _validate_new_message(idempotency_key: str, topic: str, payload: Any, run_id: str) -> None:
+    if (
+        not isinstance(idempotency_key, str)
+        or not idempotency_key
+        or len(idempotency_key) > 512
+        or any(ord(character) < 32 for character in idempotency_key)
+    ):
+        raise ValueError("invalid idempotency_key")
+    if not isinstance(topic, str) or not _TOPIC.fullmatch(topic):
+        raise ValueError("invalid topic")
+    if not isinstance(run_id, str) or not run_id or len(run_id) > 512:
+        raise ValueError("invalid run_id")
+    _json_copy(payload)
+
+
+def _validate_message(message: OutboxMessage) -> None:
+    _validate_new_message(
+        message.idempotency_key, message.topic, message.payload, message.run_id
+    )
+    if message.schema_version != 1 or message.message_id != _message_id(message.idempotency_key):
+        raise ValueError("invalid outbox message")
+    if message.status not in _STATUSES:
+        raise ValueError("invalid outbox message")
+    if not isinstance(message.attempts, int) or isinstance(message.attempts, bool) or message.attempts < 0:
+        raise ValueError("invalid outbox message")
+    if message.sent_at is not None and not isinstance(message.sent_at, str):
+        raise ValueError("invalid outbox message")
+    if message.error is not None and not isinstance(message.error, str):
+        raise ValueError("invalid outbox message")
+    if message.status == "sent" and (message.sent_at is None or message.error is not None):
+        raise ValueError("invalid outbox message")
+    if message.status == "failed" and (message.sent_at is not None or not message.error):
+        raise ValueError("invalid outbox message")
+    if message.status == "pending" and (message.sent_at is not None or message.error is not None):
+        raise ValueError("invalid outbox message")
+
+
+def _message_id(idempotency_key: str) -> str:
+    return f"msg_{hashlib.sha256(idempotency_key.encode('utf-8')).hexdigest()}"
+
+
+def _json_copy(payload: Any) -> Any:
+    try:
+        return json.loads(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    except (TypeError, ValueError) as exc:
+        raise ValueError("payload must be JSON serializable") from exc
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")