wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expand module workflows, dynamic config, notifications and console API

Browse Source
This commit is contained in:
wangyunlong
2026-09-04 09:49:37 +08:00
parent 8df5266abb
commit b124d757b0
309 changed files with 89358 additions and 6232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gyxx_flow/cli.py
+178 -4
View File
@@ -28,6 +28,11 @@ from gyxx_flow.core.locks import LockManager
from gyxx_flow.core.records import RunJournal
from gyxx_flow.diagnostics import run_doctor
from gyxx_flow.modules import create_default_registry as create_module_registry
from gyxx_flow.modules.product_commerce import (
TMALL_BAIBU_DEFAULT_IMPORT_MODE,
TMALL_BAIBU_WORKFLOW_ID,
validate_tmall_baibu_import_mode,
)
from gyxx_flow.ops import EffectLedger, EffectStateAmbiguous, RunIndex
from gyxx_flow.scheduler_service import (
PythonScheduler,
@@ -94,6 +99,38 @@ def build_parser() -> argparse.ArgumentParser:
help="one argument passed to the script; repeat for multiple arguments",
)
accounts_parser = commands.add_parser(
"accounts",
help="manage account-level browser login states shared by scripts",
)
account_commands = accounts_parser.add_subparsers(
dest="accounts_command", required=True
)
accounts_list_parser = account_commands.add_parser("list", help="list accounts")
accounts_list_parser.add_argument("--json", action="store_true")
accounts_login_parser = account_commands.add_parser(
"login", help="open the account login browser and save the vault state"
)
accounts_login_parser.add_argument("account_id")
accounts_login_parser.add_argument(
"--timeout", type=int, default=600, help="login wait timeout in seconds"
)
accounts_login_parser.add_argument("--target-url", help="page to open for login")
accounts_login_parser.add_argument("--force", action="store_true")
accounts_login_parser.add_argument("--json", action="store_true")
accounts_sync_parser = account_commands.add_parser(
"sync", help="push account vault cookies into member scripts"
)
accounts_sync_parser.add_argument("account_id", nargs="?")
accounts_sync_parser.add_argument("--json", action="store_true")
accounts_seed_parser = account_commands.add_parser(
"seed", help="import one existing script browser state into an account vault"
)
accounts_seed_parser.add_argument("account_id")
accounts_seed_parser.add_argument("--source-binding", required=True)
accounts_seed_parser.add_argument("--force", action="store_true")
accounts_seed_parser.add_argument("--json", action="store_true")
effects_parser = commands.add_parser(
"effects", help="inspect or reconcile guarded production effects"
)
@@ -196,17 +233,29 @@ def _add_execution_arguments(parser: argparse.ArgumentParser) -> None:
help="execute registered steps; omitted means no-side-effect dry-run",
)
parser.add_argument("--shadow", action="store_true")
parser.add_argument(
"--tmall-baibu-import-mode",
choices=("all", "without_hyperlinks"),
help="Tmall Baibu scope: all entries or only entries without hyperlinks",
)
recovery = parser.add_mutually_exclusive_group()
recovery.add_argument("--resume-from", metavar="STEP_ID")
recovery.add_argument("--rerun-step", metavar="STEP_ID")
def build_default_registry(settings: Settings) -> WorkflowRegistry:
def build_default_registry(
settings: Settings,
*,
tmall_baibu_import_mode: str | None = None,
) -> WorkflowRegistry:
"""Compose only workflows explicitly implemented by migrated modules."""
catalog = WorkflowCatalog.load(settings.project_root / "config")
registry = WorkflowRegistry(catalog)
modules = create_module_registry(catalog=catalog)
modules = create_module_registry(
catalog=catalog,
tmall_baibu_import_mode=tmall_baibu_import_mode,
)
for workflow_id in modules.workflow_ids:
registry.register(modules.workflow(workflow_id))
return registry
@@ -259,7 +308,33 @@ def main(
return _effects_reconcile_command(
arguments, resolved_settings, output
)
resolved_registry = registry or build_default_registry(resolved_settings)
if arguments.command == "accounts":
if arguments.accounts_command == "list":
return _accounts_list_command(arguments, resolved_settings, output)
if arguments.accounts_command == "login":
return _accounts_login_command(arguments, resolved_settings, output)
if arguments.accounts_command == "sync":
return _accounts_sync_command(arguments, resolved_settings, output)
if arguments.accounts_command == "seed":
return _accounts_seed_command(arguments, resolved_settings, output)
raise CliConfigurationError(
f"unsupported accounts command: {arguments.accounts_command}"
)
tmall_baibu_import_mode: str | None = None
if arguments.command in {"run", "backfill"}:
raw_import_mode = getattr(arguments, "tmall_baibu_import_mode", None)
if raw_import_mode is not None and arguments.workflow_id != TMALL_BAIBU_WORKFLOW_ID:
raise CliConfigurationError(
"--tmall-baibu-import-mode 仅适用于 product.tmall_baibu_apply"
)
if arguments.workflow_id == TMALL_BAIBU_WORKFLOW_ID:
tmall_baibu_import_mode = validate_tmall_baibu_import_mode(
raw_import_mode or TMALL_BAIBU_DEFAULT_IMPORT_MODE
)
resolved_registry = registry or build_default_registry(
resolved_settings,
tmall_baibu_import_mode=tmall_baibu_import_mode,
)
if arguments.command == "list":
return _list_workflows(arguments, resolved_registry, output)
if arguments.command == "run":
@@ -302,7 +377,12 @@ def _load_runtime_environment_files(arguments: argparse.Namespace) -> None:
for key, value in values.items():
if not isinstance(key, str) or not key or value is None:
continue
os.environ.setdefault(key, value)
# A service wrapper may export an empty placeholder for a secret
# or database setting. Treat that as unset so the explicitly
# requested runtime file can supply it; a non-empty process value
# remains authoritative.
if not os.environ.get(key, "").strip():
os.environ[key] = value
def _doctor_command(
@@ -342,6 +422,89 @@ def _effects_reconcile_command(
return EXIT_SUCCESS
def _accounts_list_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
from gyxx_flow.accounts import list_accounts
payload = list_accounts(settings)
if arguments.json:
_write_json(output, payload)
else:
for item in payload["accounts"]:
state = "valid" if item["vault_valid"] else (
"missing" if not item["vault_exists"] else "invalid"
)
output.write(
f"{item['account_id']}\tcdp={item['cdp_port']}\tstate={state}"
f"\tmembers={len(item['members'])}\n"
)
return EXIT_SUCCESS
def _accounts_login_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
from gyxx_flow.accounts import login_account
try:
result = login_account(
settings,
arguments.account_id,
output=output,
timeout_seconds=arguments.timeout,
target_url=arguments.target_url,
force=arguments.force,
)
except TimeoutError as exc:
raise CliRuntimeError(str(exc)) from exc
if arguments.json:
_write_json(output, result)
return EXIT_SUCCESS
def _accounts_sync_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
from gyxx_flow.accounts import sync_account
try:
result = sync_account(settings, arguments.account_id, output)
except ValueError as exc:
raise CliConfigurationError(str(exc)) from exc
if arguments.json:
_write_json(output, result)
return EXIT_SUCCESS
def _accounts_seed_command(
arguments: argparse.Namespace,
settings: Settings,
output: TextIO,
) -> int:
from gyxx_flow.accounts import seed_account
try:
result = seed_account(
settings,
arguments.account_id,
arguments.source_binding,
output,
force=arguments.force,
)
except ValueError as exc:
raise CliConfigurationError(str(exc)) from exc
if arguments.json:
_write_json(output, result)
return EXIT_SUCCESS
def _console_command(
arguments: argparse.Namespace,
settings: Settings,
@@ -355,6 +518,7 @@ def _console_command(
host=arguments.host,
port=arguments.port,
output=output,
env_files=arguments.env_file,
)
except OSError as exc:
raise CliRuntimeError("cannot start workflow console") from exc
@@ -419,6 +583,7 @@ def _list_scripts(arguments: argparse.Namespace, output: TextIO) -> int:
"entry": script.entry,
"kind": script.kind,
"default_args": list(script.default_args),
"notification_workflow_id": script.notification_workflow_id,
}
for script in catalog.scripts
if arguments.module is None or script.module == arguments.module
@@ -457,6 +622,7 @@ def _run_script(
trigger="manual",
entry=script.entry,
args=script_args,
notification_workflow_id=script.notification_workflow_id,
)
workflow = WorkflowDefinition(
workflow_id,
@@ -521,16 +687,23 @@ def _python_scheduler(
python_executable: Path | None = None,
misfire_grace_seconds: int = 21600,
dry_run: bool = False,
enable_account_keepalive: bool = False,
) -> PythonScheduler:
launcher = SubprocessWorkflowLauncher(
settings.project_root, settings.data_root, python_executable=python_executable
)
maintenance = None
if enable_account_keepalive and not dry_run:
from gyxx_flow.accounts import AccountKeepaliveManager
maintenance = AccountKeepaliveManager(settings).tick
return PythonScheduler(
catalog,
settings.data_root,
launcher=launcher,
misfire_grace_seconds=misfire_grace_seconds,
dry_run=dry_run,
maintenance=maintenance,
catalog_loader=lambda: WorkflowCatalog.load(
settings.project_root / "config"
),
@@ -553,6 +726,7 @@ def _schedule_run_command(
python_executable=arguments.python_executable,
misfire_grace_seconds=arguments.misfire_grace_seconds,
dry_run=arguments.dry_run,
enable_account_keepalive=True,
)
lock_path = settings.data_root / "state" / "scheduler" / "service.lock"
try: