feat: complete production workflow migration

src/gyxx_flow/ops/effects.py

@@ -61,9 +61,21 @@ class EffectLedger:
                     raise EffectAlreadyApplied(
                         f"effect already applied: {workflow_id}/{business_date}/{step_id}"
                     )
-                raise EffectStateAmbiguous(
-                    f"effect requires reconciliation: {workflow_id}/{business_date}/{step_id}"
+                if payload["status"] != "retry_allowed":
+                    raise EffectStateAmbiguous(
+                        "effect requires reconciliation: "
+                        f"{workflow_id}/{business_date}/{step_id}"
+                    )
+                now = _now()
+                payload.update(
+                    run_id=run_id,
+                    status="in_progress",
+                    claimed_at=now,
+                    updated_at=now,
+                    reason=None,
                 )
+                atomic_write_json(path, payload)
+                return claim
             now = _now()
             atomic_write_json(
                 path,
@@ -90,6 +102,84 @@ class EffectLedger:
             raise ValueError("invalid effect ambiguity reason")
         self._finish(claim, status="ambiguous", reason=reason)
 
+    def cancel(self, claim: EffectClaim) -> None:
+        """Remove an owned claim after a fail-closed action was skipped."""
+
+        with self._lock(claim.effect_id):
+            path = self._path(claim.effect_id)
+            payload = self._read(path)
+            if (
+                payload["status"] != "in_progress"
+                or payload["run_id"] != claim.run_id
+            ):
+                raise EffectStateAmbiguous("effect claim no longer owns the receipt")
+            path.unlink()
+
+    def reconcile(
+        self,
+        *,
+        workflow_id: str,
+        business_date: str,
+        step_id: str,
+        expected_run_id: str,
+        action: str,
+        operator: str,
+        reason: str,
+        evidence: str,
+    ) -> dict[str, str]:
+        """Resolve one ambiguous receipt with an auditable compare-and-set."""
+
+        _validate(workflow_id, business_date, step_id, expected_run_id)
+        if action not in {"retry", "applied"}:
+            raise ValueError("invalid reconciliation action")
+        _validate_note(operator, "operator", maximum=120)
+        _validate_note(reason, "reason", maximum=500)
+        _validate_note(evidence, "evidence", maximum=1000)
+        effect_id = _effect_id(workflow_id, business_date, step_id)
+        with self._lock(effect_id):
+            path = self._path(effect_id)
+            if not path.is_file():
+                raise EffectStateAmbiguous("effect receipt does not exist")
+            payload = self._read(path)
+            if payload.get("run_id") != expected_run_id:
+                raise EffectStateAmbiguous("effect receipt run_id changed")
+            if payload.get("status") != "ambiguous":
+                raise EffectStateAmbiguous(
+                    "only ambiguous effect receipts can be reconciled"
+                )
+            reconciliations = payload.get("reconciliations", [])
+            if not isinstance(reconciliations, list):
+                raise EffectStateAmbiguous(
+                    "effect receipt reconciliation history is invalid"
+                )
+            reconciled_at = _now()
+            reconciliations.append(
+                {
+                    "action": action,
+                    "operator": operator,
+                    "reason": reason,
+                    "evidence": evidence,
+                    "expected_run_id": expected_run_id,
+                    "reconciled_at": reconciled_at,
+                }
+            )
+            status = "retry_allowed" if action == "retry" else "applied"
+            payload.update(
+                status=status,
+                updated_at=reconciled_at,
+                reconciliations=reconciliations,
+            )
+            atomic_write_json(path, payload)
+        return {
+            "effect_id": effect_id,
+            "workflow_id": workflow_id,
+            "business_date": business_date,
+            "step_id": step_id,
+            "expected_run_id": expected_run_id,
+            "status": status,
+            "reconciled_at": reconciled_at,
+        }
+
     def _finish(self, claim: EffectClaim, *, status: str, reason: str | None) -> None:
         with self._lock(claim.effect_id):
             path = self._path(claim.effect_id)
@@ -121,7 +211,8 @@ class EffectLedger:
         if (
             not isinstance(payload, dict)
             or payload.get("schema_version") != 1
-            or payload.get("status") not in {"in_progress", "applied", "ambiguous"}
+            or payload.get("status")
+            not in {"in_progress", "applied", "ambiguous", "retry_allowed"}
         ):
             raise EffectStateAmbiguous("effect receipt is invalid")
         return payload
@@ -148,3 +239,7 @@ def _validate(
 def _now() -> str:
     return datetime.now(timezone.utc).isoformat(timespec="seconds")
 
+
+def _validate_note(value: str, name: str, *, maximum: int) -> None:
+    if not isinstance(value, str) or not value.strip() or len(value) > maximum:
+        raise ValueError(f"invalid reconciliation {name}")

src/gyxx_flow/ops/index.py

@@ -12,11 +12,13 @@ from typing import Any
 
 from gyxx_flow.core.artifacts import atomic_write_json
 from gyxx_flow.core.locks import LockManager
-from gyxx_flow.core.records import RunJournal
+from gyxx_flow.core.records import RunJournal, RunMode
 
 _SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
 _RUN_STATUSES = frozenset({"running", "success", "failed", "cancelled"})
 _STEP_STATUSES = frozenset({"running", "success", "failed", "skipped"})
+_RUN_MODES = frozenset({"execute", "dry_run", "unknown"})
+_RUN_INDEX_SCHEMA_VERSIONS = frozenset({1, 2})
 
 
 @dataclass(frozen=True, slots=True)
@@ -34,7 +36,8 @@ class RunRecord:
     step_counts: dict[str, int]
     journal_path: str
     indexed_at: str
-    schema_version: int = 1
+    mode: RunMode = "unknown"
+    schema_version: int = 2
 
     @classmethod
     def from_dict(cls, payload: dict[str, Any]) -> "RunRecord":
@@ -51,7 +54,8 @@ class RunRecord:
                 step_counts=dict(payload["step_counts"]),
                 journal_path=payload["journal_path"],
                 indexed_at=payload["indexed_at"],
-                schema_version=payload["schema_version"],
+                mode=payload.get("mode", "unknown"),
+                schema_version=payload.get("schema_version", 1),
             )
         except (KeyError, TypeError, ValueError) as exc:
             raise ValueError("invalid run index record") from exc
@@ -148,6 +152,7 @@ def _record_from_journal(payload: Any, journal_path: Path) -> RunRecord:
         run_id = payload["run_id"]
         business_date = payload["business_date"]
         status = payload["status"]
+        mode = payload.get("mode", "unknown")
         shadow = payload["shadow"]
         started_at = payload["started_at"]
         ended_at = payload["ended_at"]
@@ -158,6 +163,8 @@ def _record_from_journal(payload: Any, journal_path: Path) -> RunRecord:
         _require_business_date(business_date)
         if status not in _RUN_STATUSES:
             raise ValueError("invalid status")
+        if mode not in _RUN_MODES:
+            raise ValueError("invalid mode")
         if not isinstance(shadow, bool) or not isinstance(started_at, str):
             raise ValueError("invalid journal fields")
         if ended_at is not None and not isinstance(ended_at, str):
@@ -178,6 +185,7 @@ def _record_from_journal(payload: Any, journal_path: Path) -> RunRecord:
         workflow_id=workflow_id,
         run_id=run_id,
         business_date=business_date,
+        mode=mode,
         shadow=shadow,
         status=status,
         started_at=started_at,
@@ -205,7 +213,15 @@ def _validate_record(record: RunRecord) -> None:
     _require_safe_id(record.workflow_id, "workflow_id")
     _require_safe_id(record.run_id, "run_id")
     _require_business_date(record.business_date)
-    if record.status not in _RUN_STATUSES or record.schema_version != 1:
+    if (
+        not isinstance(record.status, str)
+        or record.status not in _RUN_STATUSES
+        or not isinstance(record.mode, str)
+        or record.mode not in _RUN_MODES
+        or not isinstance(record.schema_version, int)
+        or isinstance(record.schema_version, bool)
+        or record.schema_version not in _RUN_INDEX_SCHEMA_VERSIONS
+    ):
         raise ValueError("invalid run index record")
     if not isinstance(record.shadow, bool) or not isinstance(record.started_at, str):
         raise ValueError("invalid run index record")