feat: expand module workflows, dynamic config, notifications and console API

This commit is contained in:
2026-09-04 09:49:37 +08:00
parent 8df5266abb
commit b124d757b0
309 changed files with 89358 additions and 6232 deletions
+114 -1
View File
@@ -18,7 +18,7 @@ from gyxx_flow.core.locks import LockManager
from gyxx_flow.core.records import RunJournal
from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule
from gyxx_flow.ops.effects import EffectLedger
from gyxx_flow.workflow.engine import WorkflowEngine
from gyxx_flow.workflow.engine import WorkflowEngine, WorkflowRunResult
from gyxx_flow.workflow.graph import (
GraphStepResult,
WorkflowGraphState,
@@ -173,6 +173,119 @@ def test_compiler_joins_uneven_dependency_branches_before_running_once() -> None
assert publish_inputs == [{"source", "parallel", "derived"}]
def test_linear_platform_chain_imports_without_waiting_for_other_roots(
tmp_path: Path,
) -> None:
jd_started = threading.Event()
douyin_started = threading.Event()
release_jd = threading.Event()
release_douyin = threading.Event()
tmall_import_started = threading.Event()
class BlockingDownload:
def __init__(self, started: threading.Event, release: threading.Event) -> None:
self.started = started
self.release = release
def execute(
self,
*,
context: RunContext,
timeout_seconds: float | None,
dry_run: bool,
) -> StepExecution:
del context, timeout_seconds, dry_run
self.started.set()
if not self.release.wait(timeout=5):
return StepExecution(exit_code=1, error="test release timeout")
return StepExecution(exit_code=0)
class TmallDownload:
def execute(
self,
*,
context: RunContext,
timeout_seconds: float | None,
dry_run: bool,
) -> StepExecution:
del context, timeout_seconds, dry_run
return StepExecution(exit_code=0)
class TmallImport:
def execute(
self,
*,
context: RunContext,
timeout_seconds: float | None,
dry_run: bool,
) -> StepExecution:
del context, timeout_seconds, dry_run
tmall_import_started.set()
return StepExecution(exit_code=0)
workflow = WorkflowDefinition(
"product.ecommerce_costs.daily",
(
StepDefinition("tmall_download", TmallDownload()),
StepDefinition(
"tmall_import",
TmallImport(),
depends_on=("tmall_download",),
),
StepDefinition(
"jd_download",
BlockingDownload(jd_started, release_jd),
),
StepDefinition("jd_import", FakeStep(), depends_on=("jd_download",)),
StepDefinition(
"douyin_download",
BlockingDownload(douyin_started, release_douyin),
),
StepDefinition(
"douyin_import",
FakeStep(),
depends_on=("douyin_download",),
),
),
)
context = RunContext.create(
"product.ecommerce_costs.daily",
"2026-07-27",
now=datetime(2026, 7, 27, 4, 0, tzinfo=timezone.utc),
random_suffix="pipeline1",
)
journal = RunJournal.create(DataLayout(tmp_path), context)
result_holder: dict[str, WorkflowRunResult] = {}
errors: list[BaseException] = []
def execute() -> None:
try:
result_holder["result"] = WorkflowEngine(
LockManager(tmp_path / "locks")
).execute(workflow, context=context, journal=journal)
except BaseException as exc: # pragma: no cover - test harness guard
errors.append(exc)
worker = threading.Thread(target=execute)
worker.start()
jd_ready = jd_started.wait(timeout=3)
douyin_ready = douyin_started.wait(timeout=3)
tmall_import_ready = tmall_import_started.wait(timeout=3)
assert not release_jd.is_set()
assert not release_douyin.is_set()
release_jd.set()
release_douyin.set()
worker.join(timeout=5)
assert not worker.is_alive()
assert errors == []
assert jd_ready
assert douyin_ready
assert tmall_import_ready
assert result_holder["result"].status == "success"
def test_engine_preserves_retry_dependency_and_journal_contract(tmp_path: Path) -> None:
flaky = FakeStep(
StepExecution(exit_code=7, error="temporary"),