feat: complete production workflow migration

2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
@@ -0,0 +1,418 @@
+from __future__ import annotations
+
+import json
+import threading
+from collections import deque
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from langgraph.graph.state import CompiledStateGraph
+
+from gyxx_flow.catalog import WorkflowCatalog, WorkflowEntry
+from gyxx_flow.cli import build_default_registry
+from gyxx_flow.core.config import Settings
+from gyxx_flow.core.context import RunContext
+from gyxx_flow.core.layout import DataLayout
+from gyxx_flow.core.locks import LockManager
+from gyxx_flow.core.records import RunJournal
+from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
+from gyxx_flow.ops.effects import EffectLedger
+from gyxx_flow.workflow.engine import WorkflowEngine
+from gyxx_flow.workflow.graph import (
+    GraphStepResult,
+    WorkflowGraphState,
+    compile_workflow_graph,
+    initial_workflow_graph_state,
+)
+from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition
+from gyxx_flow.workflow.steps import StepExecution
+
+
+class FakeStep:
+    def __init__(self, *outcomes: StepExecution) -> None:
+        self.outcomes = deque(outcomes or (StepExecution(exit_code=0),))
+        self.calls: list[dict[str, Any]] = []
+
+    def execute(
+        self,
+        *,
+        context: RunContext,
+        timeout_seconds: float | None,
+        dry_run: bool,
+    ) -> StepExecution:
+        self.calls.append(
+            {
+                "context": context,
+                "timeout_seconds": timeout_seconds,
+                "dry_run": dry_run,
+            }
+        )
+        return self.outcomes.popleft()
+
+
+class CoordinatedStep:
+    def __init__(
+        self,
+        barrier: threading.Barrier,
+        *,
+        error: str | None = None,
+    ) -> None:
+        self.barrier = barrier
+        self.error = error
+        self.calls = 0
+
+    def execute(
+        self,
+        *,
+        context: RunContext,
+        timeout_seconds: float | None,
+        dry_run: bool,
+    ) -> StepExecution:
+        del context, timeout_seconds, dry_run
+        self.calls += 1
+        self.barrier.wait(timeout=3)
+        if self.error is not None:
+            raise RuntimeError(self.error)
+        return StepExecution(exit_code=0)
+
+
+def _context(*, suffix: str = "abc123", shadow: bool = False) -> RunContext:
+    return RunContext.create(
+        "shop.weekly",
+        "2026-07-27",
+        shadow=shadow,
+        now=datetime(2026, 7, 27, 4, 0, tzinfo=timezone.utc),
+        random_suffix=suffix,
+    )
+
+
+def _success_state(
+    step: StepDefinition,
+    state: WorkflowGraphState,
+) -> GraphStepResult:
+    del state
+    return {
+        "step_id": step.step_id,
+        "status": "success",
+        "attempts": [
+            {
+                "attempt": 1,
+                "status": "success",
+                "exit_code": 0,
+                "error": None,
+                "reason": None,
+            }
+        ],
+        "final_exit_code": 0,
+        "critical": step.critical,
+        "reason": None,
+    }
+
+
+def test_compiler_creates_one_langgraph_node_per_step_in_stable_order() -> None:
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (
+            StepDefinition("publish", FakeStep(), depends_on=("collect",)),
+            StepDefinition("audit", FakeStep()),
+            StepDefinition("collect", FakeStep()),
+        ),
+    )
+
+    compiled = compile_workflow_graph(workflow, run_step=_success_state)
+    graph = compiled.get_graph()
+
+    assert isinstance(compiled, CompiledStateGraph)
+    assert set(graph.nodes) == {"__start__", "audit", "collect", "publish", "__end__"}
+    assert {(edge.source, edge.target) for edge in graph.edges} == {
+        ("__start__", "audit"),
+        ("__start__", "collect"),
+        ("audit", "__end__"),
+        ("collect", "publish"),
+        ("publish", "__end__"),
+    }
+
+    state = initial_workflow_graph_state(_context(), dry_run=False)
+    final_state = compiled.invoke(state)
+
+    assert set(final_state["step_results"]) == {"audit", "collect", "publish"}
+    json.dumps(final_state)
+
+
+def test_compiler_joins_uneven_dependency_branches_before_running_once() -> None:
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (
+            StepDefinition("source", FakeStep()),
+            StepDefinition("parallel", FakeStep()),
+            StepDefinition("derived", FakeStep(), depends_on=("source",)),
+            StepDefinition(
+                "publish",
+                FakeStep(),
+                depends_on=("parallel", "derived"),
+            ),
+        ),
+    )
+    publish_inputs: list[set[str]] = []
+    calls: list[str] = []
+
+    def run_step(
+        step: StepDefinition,
+        state: WorkflowGraphState,
+    ) -> GraphStepResult:
+        calls.append(step.step_id)
+        if step.step_id == "publish":
+            publish_inputs.append(set(state["step_results"]))
+        return _success_state(step, state)
+
+    compiled = compile_workflow_graph(workflow, run_step=run_step)
+    compiled.invoke(initial_workflow_graph_state(_context(), dry_run=False))
+
+    assert calls.count("publish") == 1
+    assert publish_inputs == [{"source", "parallel", "derived"}]
+
+
+def test_engine_preserves_retry_dependency_and_journal_contract(tmp_path: Path) -> None:
+    flaky = FakeStep(
+        StepExecution(exit_code=7, error="temporary"),
+        StepExecution(exit_code=0),
+    )
+    downstream = FakeStep()
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (
+            StepDefinition("flaky", flaky, max_attempts=2),
+            StepDefinition("required", FakeStep(StepExecution(exit_code=9, error="bad"))),
+            StepDefinition("downstream", downstream, depends_on=("required",)),
+        ),
+    )
+    context = _context()
+    journal = RunJournal.create(DataLayout(tmp_path), context)
+
+    result = WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        workflow,
+        context=context,
+        journal=journal,
+    )
+
+    assert result.status == "failed"
+    assert [attempt.exit_code for attempt in result.steps["flaky"].attempts] == [7, 0]
+    assert result.steps["downstream"].reason == "dependency-failed"
+    assert downstream.calls == []
+    payload = json.loads(journal.path.read_text(encoding="utf-8"))
+    assert payload["steps"]["flaky.attempt-1"]["status"] == "failed"
+    assert payload["steps"]["flaky.attempt-2"]["status"] == "success"
+    assert payload["steps"]["downstream.attempt-1"]["status"] == "skipped"
+
+
+def test_independent_graph_nodes_run_concurrently_and_isolate_action_failure(
+    tmp_path: Path,
+) -> None:
+    barrier = threading.Barrier(2)
+    jd = CoordinatedStep(barrier, error="JD browser session failed")
+    tmall = CoordinatedStep(barrier)
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (
+            StepDefinition("jd", jd, resources=("browser:jd",)),
+            StepDefinition("tmall", tmall, resources=("browser:tmall",)),
+        ),
+    )
+    context = _context(suffix="parallel1")
+    journal = RunJournal.create(DataLayout(tmp_path), context)
+
+    result = WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        workflow,
+        context=context,
+        journal=journal,
+    )
+
+    assert jd.calls == 1
+    assert tmall.calls == 1
+    assert result.status == "failed"
+    assert result.steps["jd"].status == "failed"
+    assert result.steps["tmall"].status == "success"
+    payload = json.loads(journal.path.read_text(encoding="utf-8"))
+    assert payload["error"] == (
+        "critical steps failed: jd: RuntimeError: JD browser session failed"
+    )
+    assert payload["steps"]["jd.attempt-1"]["error"] == (
+        "RuntimeError: JD browser session failed"
+    )
+    assert payload["steps"]["tmall.attempt-1"]["status"] == "success"
+
+
+def test_workflow_error_reports_each_failed_step_with_bounded_details(
+    tmp_path: Path,
+) -> None:
+    jd_error = "JD PG write refused: " + "x" * 5_000 + " JD_ROOT_CAUSE_AT_TAIL"
+    tmall_error = (
+        "TM Feishu insert refused: "
+        + "y" * 5_000
+        + " TM_ROOT_CAUSE_AT_TAIL"
+    )
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (
+            StepDefinition("jd", FakeStep(StepExecution(exit_code=7, error=jd_error))),
+            StepDefinition(
+                "tmall",
+                FakeStep(StepExecution(exit_code=9, error=tmall_error)),
+            ),
+        ),
+    )
+    context = _context(suffix="bounded1")
+    journal = RunJournal.create(DataLayout(tmp_path), context)
+
+    WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        workflow,
+        context=context,
+        journal=journal,
+    )
+
+    error = json.loads(journal.path.read_text(encoding="utf-8"))["error"]
+    assert "jd: JD PG write refused" in error
+    assert "tmall: TM Feishu insert refused" in error
+    assert "JD_ROOT_CAUSE_AT_TAIL" in error
+    assert "TM_ROOT_CAUSE_AT_TAIL" in error
+    assert len(error) <= 4_096
+
+
+def test_engine_preserves_dry_run_and_shadow_policy(tmp_path: Path) -> None:
+    dry_action = FakeStep()
+    dry_workflow = WorkflowDefinition(
+        "shop.weekly", (StepDefinition("collect", dry_action),)
+    )
+    dry_context = _context()
+    dry_result = WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        dry_workflow,
+        context=dry_context,
+        journal=RunJournal.create(DataLayout(tmp_path), dry_context),
+        dry_run=True,
+    )
+
+    sink = FakeStep()
+    shadow_workflow = WorkflowDefinition(
+        "shop.weekly",
+        (StepDefinition("publish", sink, production_sink=True),),
+    )
+    shadow_context = _context(suffix="shadow1", shadow=True)
+    shadow_result = WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        shadow_workflow,
+        context=shadow_context,
+        journal=RunJournal.create(DataLayout(tmp_path), shadow_context),
+    )
+
+    assert dry_result.steps["collect"].reason == "dry-run"
+    assert shadow_result.steps["publish"].reason == "shadow-policy"
+    assert dry_action.calls == []
+    assert sink.calls == []
+
+
+def test_engine_preserves_effect_ledger_replay_policy(tmp_path: Path) -> None:
+    action = FakeStep()
+    workflow = WorkflowDefinition(
+        "shop.weekly",
+        (StepDefinition("publish", action, production_sink=True),),
+    )
+    engine = WorkflowEngine(
+        LockManager(tmp_path / "locks"),
+        effect_ledger=EffectLedger(tmp_path),
+    )
+    first_context = _context(suffix="first1")
+    second_context = _context(suffix="second2")
+
+    first = engine.execute(
+        workflow,
+        context=first_context,
+        journal=RunJournal.create(DataLayout(tmp_path), first_context),
+    )
+    second = engine.execute(
+        workflow,
+        context=second_context,
+        journal=RunJournal.create(DataLayout(tmp_path), second_context),
+    )
+
+    assert first.steps["publish"].status == "success"
+    assert second.steps["publish"].reason == "idempotency-replay"
+    assert len(action.calls) == 1
+
+
+def test_every_registered_workflow_compiles_to_langgraph(tmp_path: Path) -> None:
+    project_root = Path(__file__).parents[1]
+    registry = build_default_registry(
+        Settings(project_root=project_root, data_root=tmp_path)
+    )
+
+    compiled_count = 0
+    for entry in registry.catalog.workflows:
+        if not registry.is_registered(entry.workflow_id):
+            continue
+        definition = registry.resolve(entry.workflow_id).definition
+        compiled = compile_workflow_graph(definition, run_step=_success_state)
+        assert isinstance(compiled, CompiledStateGraph)
+        compiled_count += 1
+
+    assert compiled_count > 0
+
+
+def test_jd_self_operated_uses_source_timeouts_and_continues_after_brand_failure(
+    tmp_path: Path,
+) -> None:
+    calls: list[tuple[str, float | None]] = []
+
+    class RecordedCommand:
+        def __init__(self, entry: WorkflowEntry) -> None:
+            self.entry = entry
+
+        def execute(
+            self,
+            *,
+            context: RunContext,
+            timeout_seconds: float | None,
+            dry_run: bool,
+        ) -> StepExecution:
+            del context
+            assert dry_run is False
+            calls.append((self.entry.entry, timeout_seconds))
+            if self.entry.entry.endswith("jd_self_operated_brand_daily.py"):
+                return StepExecution(exit_code=3, error="brand failed")
+            return StepExecution(exit_code=0)
+
+    def command_factory(
+        entry: WorkflowEntry,
+        _context: RunContext,
+    ) -> RecordedCommand:
+        return RecordedCommand(entry)
+
+    catalog = WorkflowCatalog.load(Path(__file__).parents[1] / "config")
+    workflow = next(
+        item
+        for item in ShopIntelligenceModule.from_catalog(
+            catalog,
+            command_factory=command_factory,
+        ).workflow_definitions()
+        if item.workflow_id == "shop.jd_self_operated.daily"
+    )
+    context = RunContext.create(
+        workflow.workflow_id,
+        "2026-08-02",
+        random_suffix="jdself1",
+    )
+
+    result = WorkflowEngine(LockManager(tmp_path / "locks")).execute(
+        workflow,
+        context=context,
+        journal=RunJournal.create(DataLayout(tmp_path), context),
+    )
+
+    assert calls == [
+        ("collectors/jd_self_operated_brand_daily.py", 600),
+        ("collectors/jd_self_operated_product_daily.py", 1_800),
+    ]
+    assert list(result.steps) == ["brand", "product"]
+    assert result.steps["brand"].status == "failed"
+    assert result.steps["product"].status == "success"
+    assert result.status == "failed"
+    assert result.exit_code == 1