feat: expand module workflows, dynamic config, notifications and console API
This commit is contained in:
@@ -18,7 +18,7 @@ from gyxx_flow.core.locks import LockManager
|
||||
from gyxx_flow.core.records import RunJournal
|
||||
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
|
||||
from gyxx_flow.ops.effects import EffectLedger
|
||||
from gyxx_flow.workflow.engine import WorkflowEngine
|
||||
from gyxx_flow.workflow.engine import WorkflowEngine, WorkflowRunResult
|
||||
from gyxx_flow.workflow.graph import (
|
||||
GraphStepResult,
|
||||
WorkflowGraphState,
|
||||
@@ -173,6 +173,119 @@ def test_compiler_joins_uneven_dependency_branches_before_running_once() -> None
|
||||
assert publish_inputs == [{"source", "parallel", "derived"}]
|
||||
|
||||
|
||||
def test_linear_platform_chain_imports_without_waiting_for_other_roots(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
jd_started = threading.Event()
|
||||
douyin_started = threading.Event()
|
||||
release_jd = threading.Event()
|
||||
release_douyin = threading.Event()
|
||||
tmall_import_started = threading.Event()
|
||||
|
||||
class BlockingDownload:
|
||||
def __init__(self, started: threading.Event, release: threading.Event) -> None:
|
||||
self.started = started
|
||||
self.release = release
|
||||
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
context: RunContext,
|
||||
timeout_seconds: float | None,
|
||||
dry_run: bool,
|
||||
) -> StepExecution:
|
||||
del context, timeout_seconds, dry_run
|
||||
self.started.set()
|
||||
if not self.release.wait(timeout=5):
|
||||
return StepExecution(exit_code=1, error="test release timeout")
|
||||
return StepExecution(exit_code=0)
|
||||
|
||||
class TmallDownload:
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
context: RunContext,
|
||||
timeout_seconds: float | None,
|
||||
dry_run: bool,
|
||||
) -> StepExecution:
|
||||
del context, timeout_seconds, dry_run
|
||||
return StepExecution(exit_code=0)
|
||||
|
||||
class TmallImport:
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
context: RunContext,
|
||||
timeout_seconds: float | None,
|
||||
dry_run: bool,
|
||||
) -> StepExecution:
|
||||
del context, timeout_seconds, dry_run
|
||||
tmall_import_started.set()
|
||||
return StepExecution(exit_code=0)
|
||||
|
||||
workflow = WorkflowDefinition(
|
||||
"product.ecommerce_costs.daily",
|
||||
(
|
||||
StepDefinition("tmall_download", TmallDownload()),
|
||||
StepDefinition(
|
||||
"tmall_import",
|
||||
TmallImport(),
|
||||
depends_on=("tmall_download",),
|
||||
),
|
||||
StepDefinition(
|
||||
"jd_download",
|
||||
BlockingDownload(jd_started, release_jd),
|
||||
),
|
||||
StepDefinition("jd_import", FakeStep(), depends_on=("jd_download",)),
|
||||
StepDefinition(
|
||||
"douyin_download",
|
||||
BlockingDownload(douyin_started, release_douyin),
|
||||
),
|
||||
StepDefinition(
|
||||
"douyin_import",
|
||||
FakeStep(),
|
||||
depends_on=("douyin_download",),
|
||||
),
|
||||
),
|
||||
)
|
||||
context = RunContext.create(
|
||||
"product.ecommerce_costs.daily",
|
||||
"2026-07-27",
|
||||
now=datetime(2026, 7, 27, 4, 0, tzinfo=timezone.utc),
|
||||
random_suffix="pipeline1",
|
||||
)
|
||||
journal = RunJournal.create(DataLayout(tmp_path), context)
|
||||
result_holder: dict[str, WorkflowRunResult] = {}
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def execute() -> None:
|
||||
try:
|
||||
result_holder["result"] = WorkflowEngine(
|
||||
LockManager(tmp_path / "locks")
|
||||
).execute(workflow, context=context, journal=journal)
|
||||
except BaseException as exc: # pragma: no cover - test harness guard
|
||||
errors.append(exc)
|
||||
|
||||
worker = threading.Thread(target=execute)
|
||||
worker.start()
|
||||
jd_ready = jd_started.wait(timeout=3)
|
||||
douyin_ready = douyin_started.wait(timeout=3)
|
||||
tmall_import_ready = tmall_import_started.wait(timeout=3)
|
||||
assert not release_jd.is_set()
|
||||
assert not release_douyin.is_set()
|
||||
|
||||
release_jd.set()
|
||||
release_douyin.set()
|
||||
worker.join(timeout=5)
|
||||
|
||||
assert not worker.is_alive()
|
||||
assert errors == []
|
||||
assert jd_ready
|
||||
assert douyin_ready
|
||||
assert tmall_import_ready
|
||||
assert result_holder["result"].status == "success"
|
||||
|
||||
|
||||
def test_engine_preserves_retry_dependency_and_journal_contract(tmp_path: Path) -> None:
|
||||
flaky = FakeStep(
|
||||
StepExecution(exit_code=7, error="temporary"),
|
||||
|
||||
Reference in New Issue
Block a user