feat: complete production workflow migration
This commit is contained in:
+272
-34
@@ -10,71 +10,254 @@ from gyxx_flow.cli import EXIT_SUCCESS, main
|
||||
from gyxx_flow.core.config import Settings
|
||||
from gyxx_flow.script_catalog import ScriptCatalog, ScriptCatalogError
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def test_script_catalog_discovers_python_main_and_launchers(tmp_path: Path) -> None:
|
||||
|
||||
def _write_registry(path: Path, commands: list[dict[str, object]]) -> None:
|
||||
path.write_text(
|
||||
json.dumps({"schema_version": 1, "commands": commands}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_catalog_exposes_only_explicit_commands(tmp_path: Path) -> None:
|
||||
runtime = tmp_path / "content_marketing" / "runtime"
|
||||
(runtime / "tools").mkdir(parents=True)
|
||||
(runtime / "tools" / "run_job.py").write_text(
|
||||
registered = runtime / "tools" / "run_job.py"
|
||||
registered.write_text(
|
||||
'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8"
|
||||
)
|
||||
(runtime / "library.py").write_text("def helper():\n return 1\n", encoding="utf-8")
|
||||
(runtime / "daily.bat").write_text("@echo off\n", encoding="utf-8")
|
||||
(runtime / "maint.ps1").write_text("Write-Output ok\n", encoding="utf-8")
|
||||
(runtime / "runtime_tests").mkdir()
|
||||
(runtime / "runtime_tests" / "test_old.py").write_text(
|
||||
'if __name__ == "__main__":\n pass\n', encoding="utf-8"
|
||||
(runtime / "undeclared.py").write_text(
|
||||
'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8"
|
||||
)
|
||||
(runtime / "undeclared.bat").write_text("@echo off\n", encoding="utf-8")
|
||||
(runtime / "undeclared.ps1").write_text("Write-Output ok\n", encoding="utf-8")
|
||||
config = tmp_path / "commands.json"
|
||||
_write_registry(
|
||||
config,
|
||||
[
|
||||
{
|
||||
"id": "content.test.run",
|
||||
"module": "content_marketing",
|
||||
"entry": "tools/run_job.py",
|
||||
"kind": "python",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
catalog = ScriptCatalog.discover({"content_marketing": runtime})
|
||||
catalog = ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
||||
|
||||
assert catalog.script_ids == (
|
||||
"content_marketing:daily.bat",
|
||||
"content_marketing:maint.ps1",
|
||||
"content_marketing:tools/run_job.py",
|
||||
assert catalog.command_ids == ("content.test.run",)
|
||||
assert catalog.script_ids == ("content_marketing:tools/run_job.py",)
|
||||
command = catalog.get("content.test.run")
|
||||
assert command.entry == "tools/run_job.py"
|
||||
assert catalog.get("content_marketing:tools/run_job.py") is command
|
||||
with pytest.raises(ScriptCatalogError, match="unknown command"):
|
||||
catalog.get("content_marketing:undeclared.py")
|
||||
|
||||
|
||||
def test_default_registry_covers_every_executable_workflow_entry() -> None:
|
||||
catalog = ScriptCatalog.load_default()
|
||||
payload = json.loads(
|
||||
(PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert catalog.get("content_marketing:tools/run_job.py").entry == "tools/run_job.py"
|
||||
workflow_entries: set[str] = set()
|
||||
scheduled_workflows = 0
|
||||
manual_workflows = 0
|
||||
for workflow in payload["workflows"]:
|
||||
assert workflow["trigger"] in {"scheduled", "manual"}
|
||||
if workflow["trigger"] == "scheduled":
|
||||
scheduled_workflows += 1
|
||||
else:
|
||||
manual_workflows += 1
|
||||
execution = workflow["execution"]
|
||||
steps = execution.get("steps")
|
||||
entries = (
|
||||
[step["entry"] for step in steps]
|
||||
if steps is not None
|
||||
else [execution["entry"]]
|
||||
)
|
||||
for entry in entries:
|
||||
workflow_entries.add(f"{workflow['module']}:{entry}")
|
||||
|
||||
|
||||
def test_default_script_catalog_discovers_every_project_owned_runtime() -> None:
|
||||
catalog = ScriptCatalog.discover_default()
|
||||
|
||||
modules = {item.module for item in catalog.scripts}
|
||||
assert modules == {
|
||||
assert scheduled_workflows == 23
|
||||
assert manual_workflows == 0
|
||||
assert len(workflow_entries) == 29
|
||||
assert len(catalog.scripts) == 33
|
||||
assert workflow_entries < set(catalog.script_ids)
|
||||
assert {
|
||||
"content.mapping.rebuild",
|
||||
"content.failed.retry",
|
||||
"shop.douyin_price_appeal",
|
||||
"shop.jd_self_operated.collect_brand",
|
||||
"product.backfill.run",
|
||||
"product.review.orchestrate",
|
||||
"supply.workflow.run",
|
||||
} < set(catalog.command_ids)
|
||||
assert {item.module for item in catalog.scripts} == {
|
||||
"content_marketing",
|
||||
"product_commerce",
|
||||
"shop_intelligence",
|
||||
"supply_chain",
|
||||
}
|
||||
assert len(catalog.scripts) > 21
|
||||
assert all(item.path.is_relative_to(PROJECT_ROOT) for item in catalog.scripts)
|
||||
assert catalog.get("shop.jd_self_operated.collect_brand").default_args == (
|
||||
"--start-date",
|
||||
"{business_date}",
|
||||
"--end-date",
|
||||
"{business_date}",
|
||||
"--headless",
|
||||
)
|
||||
assert catalog.get("shop.douyin_price_appeal").default_args == (
|
||||
"--price-appeal",
|
||||
"--execute",
|
||||
)
|
||||
assert catalog.get("product.backfill.run").default_args == (
|
||||
"--from",
|
||||
"{business_date}",
|
||||
"--to",
|
||||
"{business_date}",
|
||||
)
|
||||
assert catalog.get("product.review.orchestrate").default_args == (
|
||||
"--target-date",
|
||||
"{business_date}",
|
||||
)
|
||||
assert catalog.get("supply.workflow.run").default_args == (
|
||||
"mcp-run",
|
||||
"purchase-order-update",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("script_id", ("missing", "content_marketing:../run.py"))
|
||||
def test_script_catalog_rejects_unknown_or_unsafe_ids(
|
||||
tmp_path: Path, script_id: str
|
||||
@pytest.mark.parametrize(
|
||||
("record", "message"),
|
||||
(
|
||||
(
|
||||
{
|
||||
"id": "content.bad.path",
|
||||
"module": "content_marketing",
|
||||
"entry": "../outside.py",
|
||||
"kind": "python",
|
||||
},
|
||||
"unsafe command entry",
|
||||
),
|
||||
(
|
||||
{
|
||||
"id": "content.bad.kind",
|
||||
"module": "content_marketing",
|
||||
"entry": "job.py",
|
||||
"kind": "batch",
|
||||
},
|
||||
"kind does not match",
|
||||
),
|
||||
(
|
||||
{
|
||||
"id": "Content Bad",
|
||||
"module": "content_marketing",
|
||||
"entry": "job.py",
|
||||
"kind": "python",
|
||||
},
|
||||
"invalid command id",
|
||||
),
|
||||
(
|
||||
{
|
||||
"id": "content.bad.args",
|
||||
"module": "content_marketing",
|
||||
"entry": "job.py",
|
||||
"kind": "python",
|
||||
"default_args": "--unsafe-shape",
|
||||
},
|
||||
"invalid command default_args",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_catalog_rejects_unsafe_or_invalid_declarations(
|
||||
tmp_path: Path, record: dict[str, object], message: str
|
||||
) -> None:
|
||||
runtime = tmp_path / "content_marketing" / "runtime"
|
||||
runtime.mkdir(parents=True)
|
||||
catalog = ScriptCatalog.discover({"content_marketing": runtime})
|
||||
runtime = tmp_path / "runtime"
|
||||
runtime.mkdir()
|
||||
(runtime / "job.py").write_text("print('ok')\n", encoding="utf-8")
|
||||
config = tmp_path / "commands.json"
|
||||
_write_registry(config, [record])
|
||||
|
||||
with pytest.raises(ScriptCatalogError):
|
||||
catalog.get(script_id)
|
||||
with pytest.raises(ScriptCatalogError, match=message):
|
||||
ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
||||
|
||||
|
||||
def test_cli_lists_and_dry_runs_project_owned_scripts(tmp_path: Path) -> None:
|
||||
def test_catalog_rejects_duplicate_command_ids(tmp_path: Path) -> None:
|
||||
runtime = tmp_path / "runtime"
|
||||
runtime.mkdir()
|
||||
for name in ("first.py", "second.py"):
|
||||
(runtime / name).write_text("print('ok')\n", encoding="utf-8")
|
||||
config = tmp_path / "commands.json"
|
||||
_write_registry(
|
||||
config,
|
||||
[
|
||||
{
|
||||
"id": "content.test.run",
|
||||
"module": "content_marketing",
|
||||
"entry": "first.py",
|
||||
"kind": "python",
|
||||
},
|
||||
{
|
||||
"id": "content.test.run",
|
||||
"module": "content_marketing",
|
||||
"entry": "second.py",
|
||||
"kind": "python",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(ScriptCatalogError, match="duplicate command id"):
|
||||
ScriptCatalog.load(config, roots={"content_marketing": runtime})
|
||||
|
||||
|
||||
def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None:
|
||||
output = io.StringIO()
|
||||
settings = Settings(project_root=Path(__file__).parents[1], data_root=tmp_path)
|
||||
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
||||
|
||||
assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0
|
||||
rows = json.loads(output.getvalue())
|
||||
assert len(rows) > 21
|
||||
assert len(rows) == 33
|
||||
assert {row["module"] for row in rows} == {
|
||||
"content_marketing",
|
||||
"product_commerce",
|
||||
"shop_intelligence",
|
||||
"supply_chain",
|
||||
}
|
||||
assert {
|
||||
"id",
|
||||
"command_id",
|
||||
"legacy_script_id",
|
||||
"module",
|
||||
"entry",
|
||||
"kind",
|
||||
"default_args",
|
||||
} <= rows[0].keys()
|
||||
|
||||
output = io.StringIO()
|
||||
exit_code = main(
|
||||
[
|
||||
"scripts",
|
||||
"run",
|
||||
"content.metrics.collect_collaborators",
|
||||
"--date",
|
||||
"2026-07-27",
|
||||
],
|
||||
settings=settings,
|
||||
stdout=output,
|
||||
)
|
||||
payload = json.loads(output.getvalue())
|
||||
assert exit_code == EXIT_SUCCESS
|
||||
assert payload["dry_run"] is True
|
||||
assert payload["command_id"] == "content.metrics.collect_collaborators"
|
||||
assert payload["script_id"] == "content_marketing:run_all.py"
|
||||
assert payload["steps"] == {"module_script": "skipped"}
|
||||
|
||||
|
||||
def test_cli_accepts_legacy_script_id_alias(tmp_path: Path) -> None:
|
||||
output = io.StringIO()
|
||||
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
||||
|
||||
exit_code = main(
|
||||
[
|
||||
"scripts",
|
||||
@@ -86,8 +269,63 @@ def test_cli_lists_and_dry_runs_project_owned_scripts(tmp_path: Path) -> None:
|
||||
settings=settings,
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
payload = json.loads(output.getvalue())
|
||||
assert exit_code == EXIT_SUCCESS
|
||||
assert payload["dry_run"] is True
|
||||
assert payload["command_id"] == "content.metrics.collect_collaborators"
|
||||
assert payload["script_id"] == "content_marketing:run_all.py"
|
||||
assert payload["steps"] == {"module_script": "skipped"}
|
||||
|
||||
|
||||
def test_cli_script_effect_identity_includes_arguments(tmp_path: Path) -> None:
|
||||
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
||||
|
||||
def dry_run(*extra: str) -> dict[str, object]:
|
||||
output = io.StringIO()
|
||||
exit_code = main(
|
||||
[
|
||||
"scripts",
|
||||
"run",
|
||||
"content.marketing_report.generate",
|
||||
"--date",
|
||||
"2026-07-27",
|
||||
*extra,
|
||||
],
|
||||
settings=settings,
|
||||
stdout=output,
|
||||
)
|
||||
assert exit_code == EXIT_SUCCESS
|
||||
return json.loads(output.getvalue())
|
||||
|
||||
prompt_only = dry_run("--arg=--prompt-only")
|
||||
prompt_only_again = dry_run("--arg=--prompt-only")
|
||||
generate = dry_run()
|
||||
|
||||
assert prompt_only["workflow_id"] == prompt_only_again["workflow_id"]
|
||||
assert prompt_only["workflow_id"] != generate["workflow_id"]
|
||||
|
||||
|
||||
def test_cli_renders_command_business_date_defaults_into_effect_identity(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path)
|
||||
|
||||
def dry_run(business_date: str) -> dict[str, object]:
|
||||
output = io.StringIO()
|
||||
exit_code = main(
|
||||
[
|
||||
"scripts",
|
||||
"run",
|
||||
"product.backfill.run",
|
||||
"--date",
|
||||
business_date,
|
||||
],
|
||||
settings=settings,
|
||||
stdout=output,
|
||||
)
|
||||
assert exit_code == EXIT_SUCCESS
|
||||
return json.loads(output.getvalue())
|
||||
|
||||
first = dry_run("2026-07-27")
|
||||
second = dry_run("2026-07-28")
|
||||
|
||||
assert first["workflow_id"] != second["workflow_id"]
|
||||
|
||||
Reference in New Issue
Block a user