wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete production workflow migration

Browse Source
This commit is contained in:
wangyunlong
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_runtime_integrations.py
+326 -15
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
@@ -11,6 +13,9 @@ from gyxx_flow.adapters import (
RuntimeIntegrationCatalog,
RuntimeIntegrationError,
RuntimeServicePolicy,
binding_from_environment,
environment_for_child_script,
resolve_hermes_profile_api_key,
)
from gyxx_flow.adapters.bootstrap import _rewrite_browser_arguments
from gyxx_flow.adapters.native import ModuleCommandAdapter, ModuleSourceRoots
@@ -29,9 +34,9 @@ def _catalog(tmp_path: Path) -> ScriptCatalog:
second.write_text("print('second')\n", encoding="utf-8")
return ScriptCatalog(
(
ScriptEntry("demo:first.py", "demo", "first.py", "python", first),
ScriptEntry("demo.first", "demo", "first.py", "python", first),
ScriptEntry(
"demo:nested/second.py",
"demo.second",
"demo",
"nested/second.py",
"python",
@@ -45,17 +50,34 @@ def _write_config(path: Path) -> None:
path.write_text(
json.dumps(
{
"schema_version": 1,
"schema_version": 2,
"cdp_host": "127.0.0.1",
"scripts": {
"demo:first.py": 22001,
"demo:nested/second.py": 22002,
"demo.first": {
"script_id": "demo:first.py",
"state_key": "demo:first.py",
"aliases": ["demo:first.py"],
"cdp_port": 22001,
"login_mode": "C",
"required_cookie_domains": ["example.test"],
"required_cookie_names": ["session"],
"credential_env_names": ["DEMO_ACCOUNT", "DEMO_PASSWORD"],
},
"demo.second": {
"script_id": "demo:nested/second.py",
"state_key": "demo:nested/second.py",
"aliases": ["demo:nested/second.py"],
"cdp_port": 22002,
},
},
"services": {
"feishu": "legacy",
"postgres": "cloud",
"hermes": "local",
"hermes_url": "http://127.0.0.1:8642/v1/chat/completions",
"hermes_collector_url": "http://127.0.0.1:8643/v1/chat/completions",
"hermes_analyzer_gateway_url": "http://127.0.0.1:8642/v1",
"hermes_collector_gateway_url": "http://127.0.0.1:8643/v1",
},
}
),
@@ -77,12 +99,18 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
data_root=project_root / "var",
)
assert len(scripts.script_ids) == 131
bindings = [first.binding_for(script_id) for script_id in scripts.script_ids]
assert len({binding.cdp_port for binding in bindings}) == 131
assert len(scripts.command_ids) == 33
assert len(first.command_ids) == 137
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 137
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
appeal = first.binding_for("shop.douyin_price_appeal")
assert appeal is first.binding_for(
"shop_intelligence:collectors/dy_store_competitor_store_scraping.py"
)
assert appeal.cdp_port == 22104
def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Path) -> None:
@@ -107,6 +135,65 @@ def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Pa
assert not first.profile_dir.exists()
def test_schema_v1_keeps_legacy_script_ids_and_existing_state_paths(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
stable_config = tmp_path / "stable.json"
legacy_config = tmp_path / "legacy.json"
_write_config(stable_config)
payload = json.loads(stable_config.read_text(encoding="utf-8"))
payload["schema_version"] = 1
payload["scripts"] = {
item["script_id"]: item["cdp_port"]
for item in payload["scripts"].values()
}
legacy_config.write_text(json.dumps(payload), encoding="utf-8")
stable = RuntimeIntegrationCatalog.load(
stable_config, scripts=scripts, data_root=tmp_path / "data"
)
legacy = RuntimeIntegrationCatalog.load(
legacy_config, scripts=scripts, data_root=tmp_path / "data"
)
assert stable.binding_for("demo.first").profile_dir == legacy.binding_for(
"demo:first.py"
).profile_dir
assert stable.binding_for("demo.first").cookie_file == legacy.binding_for(
"demo:first.py"
).cookie_file
def test_state_key_survives_a_physical_entry_move_and_old_alias_still_resolves(
tmp_path: Path,
) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
before = RuntimeIntegrationCatalog.load(
config, scripts=scripts, data_root=tmp_path / "data"
).binding_for("demo.first")
payload = json.loads(config.read_text(encoding="utf-8"))
payload["scripts"]["demo.first"]["script_id"] = "demo:moved/first.py"
config.write_text(json.dumps(payload), encoding="utf-8")
moved_path = tmp_path / "runtime" / "moved" / "first.py"
moved_path.parent.mkdir()
moved_path.write_text("print('moved')\n", encoding="utf-8")
moved_scripts = ScriptCatalog(
(ScriptEntry("demo.first", "demo", "moved/first.py", "python", moved_path),)
)
catalog = RuntimeIntegrationCatalog.load(
config, scripts=moved_scripts, data_root=tmp_path / "data"
)
after = catalog.binding_for("demo.first")
assert catalog.binding_for("demo:first.py") is after
assert catalog.binding_for("demo:moved/first.py") is after
assert after.profile_dir == before.profile_dir
assert after.cookie_file == before.cookie_file
assert after.storage_state_file == before.storage_state_file
def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
@@ -119,10 +206,128 @@ def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path:
assert child["KEEP"] == "yes"
assert parent["GYXX_SCRIPT_ID"] == "demo:first.py"
assert child["GYXX_SCRIPT_ID"] == "demo:nested/second.py"
assert parent["GYXX_COMMAND_ID"] == "demo.first"
assert child["GYXX_COMMAND_ID"] == "demo.second"
assert parent["GYXX_BROWSER_STATE_KEY"] == "demo:first.py"
assert parent["GYXX_BROWSER_LOGIN_MODE"] == "C"
assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_DOMAINS"]) == [
"example.test"
]
assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_NAMES"]) == ["session"]
assert json.loads(parent["GYXX_BROWSER_CREDENTIAL_ENV_NAMES"]) == [
"DEMO_ACCOUNT",
"DEMO_PASSWORD",
]
assert parent["DY_COOKIES_FILE"] == parent["GYXX_BROWSER_COOKIE_FILE"]
assert parent["GYXX_BROWSER_CDP_PORT"] != child["GYXX_BROWSER_CDP_PORT"]
assert parent["GYXX_BROWSER_PROFILE_DIR"] != child["GYXX_BROWSER_PROFILE_DIR"]
assert parent["GYXX_BROWSER_COOKIE_FILE"] != child["GYXX_BROWSER_COOKIE_FILE"]
reconstructed = binding_from_environment(parent)
assert reconstructed.login_mode == "C"
assert reconstructed.required_cookie_domains == ("example.test",)
assert reconstructed.required_cookie_names == ("session",)
assert reconstructed.credential_env_names == ("DEMO_ACCOUNT", "DEMO_PASSWORD")
def test_private_nested_child_keeps_legacy_alias_binding(tmp_path: Path) -> None:
project_root = Path(__file__).resolve().parents[1]
module_root = (
project_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "runtime"
)
environment = environment_for_child_script(
module_root / "taobao_sycm_products.py",
{
"GYXX_MODULE_ID": "product_commerce",
"GYXX_MODULE_ROOT": str(module_root),
"GYXX_PROJECT_ROOT": str(project_root),
"GYXX_DATA_ROOT": str(tmp_path / "data"),
},
)
assert environment["GYXX_SCRIPT_ID"] == "product_commerce:taobao_sycm_products.py"
assert environment["GYXX_COMMAND_ID"] == "product_commerce:taobao_sycm_products.py"
assert environment["GYXX_BROWSER_STATE_KEY"] == (
"product_commerce:taobao_sycm_products.py"
)
assert environment["GYXX_BROWSER_CDP_PORT"] == "22096"
def test_jd_self_operated_collectors_consume_their_own_browser_binding(
tmp_path: Path,
) -> None:
project_root = Path(__file__).resolve().parents[1]
scripts = ScriptCatalog.discover_default()
catalog = RuntimeIntegrationCatalog.load(
project_root / "config" / "runtime-bindings.json",
scripts=scripts,
data_root=tmp_path / "portable-data",
)
module_root = (
project_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "shop_intelligence"
)
cases = (
(
"shop_intelligence:collectors/jd_self_operated_brand_daily.py",
"gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily",
),
(
"shop_intelligence:collectors/jd_self_operated_product_daily.py",
"gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily",
),
)
observed_ports: list[int] = []
for script_id, module_name in cases:
binding = catalog.binding_for(script_id)
environment = catalog.environment_for(script_id, os.environ)
environment["PYTHONPATH"] = os.pathsep.join(
(
str(project_root / "src"),
str(module_root),
environment.get("PYTHONPATH", ""),
)
)
command = (
"import json; "
f"import {module_name} as module; "
"print(json.dumps({"
"'port': module.CDP_PORT, "
"'profile': str(module.BROWSER_PROFILE_DIR), "
"'storage': str(module.STORAGE_STATE_PATH)"
"}))"
)
result = subprocess.run(
[sys.executable, "-c", command],
cwd=module_root,
env=environment,
capture_output=True,
text=True,
encoding="utf-8",
timeout=30,
check=False,
)
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout.strip().splitlines()[-1])
assert payload == {
"port": binding.cdp_port,
"profile": str(binding.profile_dir),
"storage": str(binding.storage_state_file),
}
observed_ports.append(payload["port"])
assert observed_ports == [22132, 22133]
def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
@@ -144,6 +349,8 @@ def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) ->
assert command.env["KEEP"] == "yes"
assert command.env["GYXX_SCRIPT_ID"] == "demo:first.py"
assert command.env["GYXX_COMMAND_ID"] == "demo.first"
assert command.env["GYXX_BROWSER_STATE_KEY"] == "demo:first.py"
assert command.env["GYXX_BROWSER_CDP_PORT"] == "22001"
@@ -166,13 +373,14 @@ def test_cookie_store_round_trip_and_storage_state_are_atomic(tmp_path: Path) ->
assert "secret-sentinel" not in repr(store)
def test_service_policy_preserves_original_feishu_and_cloud_db_and_local_hermes() -> None:
def test_service_policy_preserves_feishu_and_uses_cloud_pg_and_local_hermes() -> None:
policy = RuntimeServicePolicy(
hermes_url="http://127.0.0.1:8642/v1/chat/completions"
)
original = {
"LARK_PROFILE": "original-profile",
"PG_HOST": "cloud-db.example.test",
"PG_PORT": "5432",
"PG_PASSWORD": "placeholder",
"DB_HOST": "",
"CUSTOM": "keep",
@@ -185,21 +393,28 @@ def test_service_policy_preserves_original_feishu_and_cloud_db_and_local_hermes(
assert result["PG_PASSWORD"] == "placeholder"
assert result["DB_HOST"] == "cloud-db.example.test"
assert result["AUTOFLOW_PG_HOST"] == "cloud-db.example.test"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_PORT"] == "5432"
assert result["GYXX_FEISHU_MODE"] == "legacy"
assert result["GYXX_POSTGRES_MODE"] == "cloud"
assert result["GYXX_HERMES_MODE"] == "local"
assert result["HERMES_ANALYZER_URL"].startswith("http://127.0.0.1:")
assert result["COLLECTOR_API_SERVER_URL"].startswith("http://127.0.0.1:")
assert result["ANALYZER_HERMES_GATEWAY_URL"].endswith(":8642/v1")
assert result["COLLECTOR_HERMES_GATEWAY_URL"].endswith(":8643/v1")
assert "secret-sentinel" not in repr(policy)
@pytest.mark.parametrize(
("environment", "message"),
[
({"PG_HOST": "127.0.0.1"}, "cloud"),
({"DB_HOST": "localhost"}, "cloud"),
({"AUTOFLOW_PG_HOST": "::1"}, "cloud"),
({"PG_HOST": "127.0.0.1"}, "remote"),
({"DB_HOST": "localhost"}, "remote"),
({"AUTOFLOW_PG_HOST": "::1"}, "remote"),
({"DATABASE_URL": "postgresql://user@127.0.0.1/gyxx"}, "remote"),
({"HERMES_ANALYZER_URL": "https://remote.example.test/v1"}, "local"),
({"ANALYZER_API_SERVER_URL": "http://10.0.0.8:8642/v1"}, "local"),
({"COLLECTOR_HERMES_GATEWAY_URL": "https://remote.example.test/v1"}, "local"),
],
)
def test_service_policy_rejects_local_database_or_remote_hermes(
@@ -209,18 +424,114 @@ def test_service_policy_rejects_local_database_or_remote_hermes(
RuntimeServicePolicy().apply(environment)
def test_service_policy_supplies_portable_local_pg_defaults_without_password() -> None:
result = RuntimeServicePolicy(
postgres_mode="local",
postgres_host="127.0.0.1",
postgres_database="gyxx_super_data",
postgres_user="gyxx_flow",
).apply({})
assert result["PG_HOST"] == "127.0.0.1"
assert result["DB_HOST"] == "127.0.0.1"
assert result["AUTOFLOW_PG_HOST"] == "127.0.0.1"
assert result["PG_PORT"] == "5432"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_PORT"] == "5432"
assert result["PG_DB"] == "gyxx_super_data"
assert result["DB_NAME"] == "gyxx_super_data"
assert result["PG_USER"] == "gyxx_flow"
assert "PG_PASSWORD" not in result
def test_service_policy_maps_canonical_secrets_without_exposing_defaults() -> None:
result = RuntimeServicePolicy().apply(
{
"GYXX_POSTGRES_PASSWORD": "placeholder",
"GYXX_HERMES_API_KEY": "placeholder",
}
)
assert result["PG_PASSWORD"] == "placeholder"
assert result["DB_PASSWORD"] == "placeholder"
assert result["AUTOFLOW_PG_PASSWORD"] == "placeholder"
assert result["HERMES_API_KEY"] == "placeholder"
assert result["HERMES_ANALYZER_TOKEN"] == "placeholder"
assert result["GYXX_SUPPLY_HERMES_TOKEN"] == "placeholder"
def test_shared_hermes_resolver_reads_runtime_profile_without_copying_secret(
tmp_path: Path,
) -> None:
profile = tmp_path / "profiles" / "data-analyzer"
profile.mkdir(parents=True)
(profile / ".env").write_text(
"API_SERVER_PORT=8642\nAPI_SERVER_KEY=local-only-key\n",
encoding="utf-8",
)
assert resolve_hermes_profile_api_key(
"data-analyzer",
{"HERMES_HOME": str(tmp_path)},
) == "local-only-key"
def test_shared_hermes_resolver_prefers_explicit_runtime_secret(tmp_path: Path) -> None:
assert resolve_hermes_profile_api_key(
"data-analyzer",
{
"HERMES_HOME": str(tmp_path),
"HERMES_ANALYZER_TOKEN": "explicit-key",
},
preferred_environment_names=("HERMES_ANALYZER_TOKEN",),
) == "explicit-key"
def test_service_policy_replaces_present_but_blank_local_pg_aliases() -> None:
result = RuntimeServicePolicy(
postgres_mode="local",
postgres_host="127.0.0.1",
postgres_database="gyxx_super_data",
postgres_user="gyxx_flow",
).apply(
{"PG_HOST": "", "DB_PORT": " ", "AUTOFLOW_PG_DB": ""}
)
assert result["PG_HOST"] == "127.0.0.1"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_DB"] == "gyxx_super_data"
def test_service_policy_maps_cloud_dsn_to_all_database_aliases() -> None:
result = RuntimeServicePolicy().apply(
{"GYXX_POSTGRES_DSN": "postgresql://cloud_user:placeholder@db.example.test/data_hub"}
)
assert result["GYXX_POSTGRES_MODE"] == "cloud"
assert result["PG_HOST"] == "db.example.test"
assert result["DB_HOST"] == "db.example.test"
assert result["AUTOFLOW_PG_HOST"] == "db.example.test"
assert result["PG_DB"] == "data_hub"
assert result["AUTOFLOW_PG_USER"] == "cloud_user"
def test_registry_rejects_missing_duplicate_or_unknown_script_allocations(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
payload = json.loads(config.read_text(encoding="utf-8"))
payload["scripts"]["demo:nested/second.py"] = 22001
payload["scripts"]["demo.second"]["cdp_port"] = 22001
config.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeIntegrationError, match="unique"):
RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")
payload["scripts"].pop("demo:nested/second.py")
payload["scripts"]["demo:unknown.py"] = 22003
payload["scripts"].pop("demo.second")
payload["scripts"]["demo.unknown"] = {
"script_id": "demo:unknown.py",
"state_key": "demo:unknown.py",
"aliases": ["demo:unknown.py"],
"cdp_port": 22003,
}
config.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeIntegrationError, match="coverage"):
RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")