Files
gyxx-flow/tests/test_runtime_integrations.py
T

564 lines
21 KiB
Python

from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
from gyxx_flow.adapters import (
BrowserCookieStore,
RuntimeIntegrationCatalog,
RuntimeIntegrationError,
RuntimeServicePolicy,
binding_from_environment,
environment_for_child_script,
resolve_hermes_profile_api_key,
)
from gyxx_flow.adapters.bootstrap import _rewrite_browser_arguments
from gyxx_flow.adapters.native import ModuleCommandAdapter, ModuleSourceRoots
from gyxx_flow.catalog import WorkflowEntry
from gyxx_flow.core.context import RunContext
from gyxx_flow.script_catalog import ScriptCatalog, ScriptEntry
def _catalog(tmp_path: Path) -> ScriptCatalog:
root = tmp_path / "runtime"
root.mkdir()
first = root / "first.py"
second = root / "nested" / "second.py"
second.parent.mkdir()
first.write_text("print('first')\n", encoding="utf-8")
second.write_text("print('second')\n", encoding="utf-8")
return ScriptCatalog(
(
ScriptEntry("demo.first", "demo", "first.py", "python", first),
ScriptEntry(
"demo.second",
"demo",
"nested/second.py",
"python",
second,
),
)
)
def _write_config(path: Path) -> None:
path.write_text(
json.dumps(
{
"schema_version": 2,
"cdp_host": "127.0.0.1",
"scripts": {
"demo.first": {
"script_id": "demo:first.py",
"state_key": "demo:first.py",
"aliases": ["demo:first.py"],
"cdp_port": 22001,
"login_mode": "C",
"required_cookie_domains": ["example.test"],
"required_cookie_names": ["session"],
"credential_env_names": ["DEMO_ACCOUNT", "DEMO_PASSWORD"],
},
"demo.second": {
"script_id": "demo:nested/second.py",
"state_key": "demo:nested/second.py",
"aliases": ["demo:nested/second.py"],
"cdp_port": 22002,
},
},
"services": {
"feishu": "legacy",
"postgres": "cloud",
"hermes": "local",
"hermes_url": "http://127.0.0.1:8642/v1/chat/completions",
"hermes_collector_url": "http://127.0.0.1:8643/v1/chat/completions",
"hermes_analyzer_gateway_url": "http://127.0.0.1:8642/v1",
"hermes_collector_gateway_url": "http://127.0.0.1:8643/v1",
},
}
),
encoding="utf-8",
)
def test_project_registry_covers_every_script_with_unique_stable_port() -> None:
project_root = Path(__file__).resolve().parents[1]
scripts = ScriptCatalog.discover_default()
first = RuntimeIntegrationCatalog.load(
project_root / "config" / "runtime-bindings.json",
scripts=scripts,
data_root=project_root / "var",
)
second = RuntimeIntegrationCatalog.load(
project_root / "config" / "runtime-bindings.json",
scripts=scripts,
data_root=project_root / "var",
)
assert len(scripts.command_ids) == 33
assert len(first.command_ids) == 137
bindings = [first.binding_for(command_id) for command_id in first.command_ids]
assert len({binding.cdp_port for binding in bindings}) == 137
assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings)
assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings)
assert bindings == [second.binding_for(item.script_id) for item in bindings]
appeal = first.binding_for("shop.douyin_price_appeal")
assert appeal is first.binding_for(
"shop_intelligence:collectors/dy_store_competitor_store_scraping.py"
)
assert appeal.cdp_port == 22104
def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
data_root = tmp_path / "portable-data"
catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=data_root)
first = catalog.binding_for("demo:first.py")
second = catalog.binding_for("demo:nested/second.py")
assert first.profile_dir != second.profile_dir
assert first.cookie_file != second.cookie_file
assert first.storage_state_file != second.storage_state_file
for path in (
first.profile_dir,
first.cookie_file,
first.storage_state_file,
second.profile_dir,
):
assert path.is_relative_to(data_root.resolve())
assert not first.profile_dir.exists()
def test_schema_v1_keeps_legacy_script_ids_and_existing_state_paths(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
stable_config = tmp_path / "stable.json"
legacy_config = tmp_path / "legacy.json"
_write_config(stable_config)
payload = json.loads(stable_config.read_text(encoding="utf-8"))
payload["schema_version"] = 1
payload["scripts"] = {
item["script_id"]: item["cdp_port"]
for item in payload["scripts"].values()
}
legacy_config.write_text(json.dumps(payload), encoding="utf-8")
stable = RuntimeIntegrationCatalog.load(
stable_config, scripts=scripts, data_root=tmp_path / "data"
)
legacy = RuntimeIntegrationCatalog.load(
legacy_config, scripts=scripts, data_root=tmp_path / "data"
)
assert stable.binding_for("demo.first").profile_dir == legacy.binding_for(
"demo:first.py"
).profile_dir
assert stable.binding_for("demo.first").cookie_file == legacy.binding_for(
"demo:first.py"
).cookie_file
def test_state_key_survives_a_physical_entry_move_and_old_alias_still_resolves(
tmp_path: Path,
) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
before = RuntimeIntegrationCatalog.load(
config, scripts=scripts, data_root=tmp_path / "data"
).binding_for("demo.first")
payload = json.loads(config.read_text(encoding="utf-8"))
payload["scripts"]["demo.first"]["script_id"] = "demo:moved/first.py"
config.write_text(json.dumps(payload), encoding="utf-8")
moved_path = tmp_path / "runtime" / "moved" / "first.py"
moved_path.parent.mkdir()
moved_path.write_text("print('moved')\n", encoding="utf-8")
moved_scripts = ScriptCatalog(
(ScriptEntry("demo.first", "demo", "moved/first.py", "python", moved_path),)
)
catalog = RuntimeIntegrationCatalog.load(
config, scripts=moved_scripts, data_root=tmp_path / "data"
)
after = catalog.binding_for("demo.first")
assert catalog.binding_for("demo:first.py") is after
assert catalog.binding_for("demo:moved/first.py") is after
assert after.profile_dir == before.profile_dir
assert after.cookie_file == before.cookie_file
assert after.storage_state_file == before.storage_state_file
def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")
parent = catalog.environment_for("demo:first.py", {"KEEP": "yes"})
child = catalog.environment_for("demo:nested/second.py", parent)
assert child["KEEP"] == "yes"
assert parent["GYXX_SCRIPT_ID"] == "demo:first.py"
assert child["GYXX_SCRIPT_ID"] == "demo:nested/second.py"
assert parent["GYXX_COMMAND_ID"] == "demo.first"
assert child["GYXX_COMMAND_ID"] == "demo.second"
assert parent["GYXX_BROWSER_STATE_KEY"] == "demo:first.py"
assert parent["GYXX_BROWSER_LOGIN_MODE"] == "C"
assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_DOMAINS"]) == [
"example.test"
]
assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_NAMES"]) == ["session"]
assert json.loads(parent["GYXX_BROWSER_CREDENTIAL_ENV_NAMES"]) == [
"DEMO_ACCOUNT",
"DEMO_PASSWORD",
]
assert parent["DY_COOKIES_FILE"] == parent["GYXX_BROWSER_COOKIE_FILE"]
assert parent["GYXX_BROWSER_CDP_PORT"] != child["GYXX_BROWSER_CDP_PORT"]
assert parent["GYXX_BROWSER_PROFILE_DIR"] != child["GYXX_BROWSER_PROFILE_DIR"]
assert parent["GYXX_BROWSER_COOKIE_FILE"] != child["GYXX_BROWSER_COOKIE_FILE"]
reconstructed = binding_from_environment(parent)
assert reconstructed.login_mode == "C"
assert reconstructed.required_cookie_domains == ("example.test",)
assert reconstructed.required_cookie_names == ("session",)
assert reconstructed.credential_env_names == ("DEMO_ACCOUNT", "DEMO_PASSWORD")
def test_private_nested_child_keeps_legacy_alias_binding(tmp_path: Path) -> None:
project_root = Path(__file__).resolve().parents[1]
module_root = (
project_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "runtime"
)
environment = environment_for_child_script(
module_root / "taobao_sycm_products.py",
{
"GYXX_MODULE_ID": "product_commerce",
"GYXX_MODULE_ROOT": str(module_root),
"GYXX_PROJECT_ROOT": str(project_root),
"GYXX_DATA_ROOT": str(tmp_path / "data"),
},
)
assert environment["GYXX_SCRIPT_ID"] == "product_commerce:taobao_sycm_products.py"
assert environment["GYXX_COMMAND_ID"] == "product_commerce:taobao_sycm_products.py"
assert environment["GYXX_BROWSER_STATE_KEY"] == (
"product_commerce:taobao_sycm_products.py"
)
assert environment["GYXX_BROWSER_CDP_PORT"] == "22096"
def test_jd_self_operated_collectors_consume_their_own_browser_binding(
tmp_path: Path,
) -> None:
project_root = Path(__file__).resolve().parents[1]
scripts = ScriptCatalog.discover_default()
catalog = RuntimeIntegrationCatalog.load(
project_root / "config" / "runtime-bindings.json",
scripts=scripts,
data_root=tmp_path / "portable-data",
)
module_root = (
project_root
/ "src"
/ "gyxx_flow"
/ "modules"
/ "shop_intelligence"
)
cases = (
(
"shop_intelligence:collectors/jd_self_operated_brand_daily.py",
"gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily",
),
(
"shop_intelligence:collectors/jd_self_operated_product_daily.py",
"gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily",
),
)
observed_ports: list[int] = []
for script_id, module_name in cases:
binding = catalog.binding_for(script_id)
environment = catalog.environment_for(script_id, os.environ)
environment["PYTHONPATH"] = os.pathsep.join(
(
str(project_root / "src"),
str(module_root),
environment.get("PYTHONPATH", ""),
)
)
command = (
"import json; "
f"import {module_name} as module; "
"print(json.dumps({"
"'port': module.CDP_PORT, "
"'profile': str(module.BROWSER_PROFILE_DIR), "
"'storage': str(module.STORAGE_STATE_PATH)"
"}))"
)
result = subprocess.run(
[sys.executable, "-c", command],
cwd=module_root,
env=environment,
capture_output=True,
text=True,
encoding="utf-8",
timeout=30,
check=False,
)
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout.strip().splitlines()[-1])
assert payload == {
"port": binding.cdp_port,
"profile": str(binding.profile_dir),
"storage": str(binding.storage_state_file),
}
observed_ports.append(payload["port"])
assert observed_ports == [22132, 22133]
def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")
runtime_root = tmp_path / "runtime"
adapter = ModuleCommandAdapter(
ModuleSourceRoots({"demo": runtime_root}),
base_env={"KEEP": "yes"},
project_root=tmp_path,
data_root=tmp_path / "data",
integration_catalog=catalog,
)
entry = WorkflowEntry("demo.workflow", "demo", "manual", "first.py")
context = RunContext.create("demo.workflow", "2026-07-27")
command = adapter.build(entry, context=context)
assert command.env["KEEP"] == "yes"
assert command.env["GYXX_SCRIPT_ID"] == "demo:first.py"
assert command.env["GYXX_COMMAND_ID"] == "demo.first"
assert command.env["GYXX_BROWSER_STATE_KEY"] == "demo:first.py"
assert command.env["GYXX_BROWSER_CDP_PORT"] == "22001"
def test_cookie_store_round_trip_and_storage_state_are_atomic(tmp_path: Path) -> None:
store = BrowserCookieStore(
cookie_file=tmp_path / "cookies" / "cookies.json",
storage_state_file=tmp_path / "cookies" / "storage_state.json",
)
cookies = [{"name": "session", "value": "secret-sentinel", "domain": ".example.test", "path": "/"}]
state = {"cookies": cookies, "origins": []}
assert store.load_cookies() == []
assert store.load_storage_state() is None
store.save_cookies(cookies)
store.save_storage_state(state)
assert store.load_cookies() == cookies
assert store.load_storage_state() == state
assert not list((tmp_path / "cookies").glob("*.tmp"))
assert "secret-sentinel" not in repr(store)
def test_service_policy_preserves_feishu_and_uses_cloud_pg_and_local_hermes() -> None:
policy = RuntimeServicePolicy(
hermes_url="http://127.0.0.1:8642/v1/chat/completions"
)
original = {
"LARK_PROFILE": "original-profile",
"PG_HOST": "cloud-db.example.test",
"PG_PORT": "5432",
"PG_PASSWORD": "placeholder",
"DB_HOST": "",
"CUSTOM": "keep",
}
result = policy.apply(original)
assert result["LARK_PROFILE"] == "original-profile"
assert result["PG_HOST"] == "cloud-db.example.test"
assert result["PG_PASSWORD"] == "placeholder"
assert result["DB_HOST"] == "cloud-db.example.test"
assert result["AUTOFLOW_PG_HOST"] == "cloud-db.example.test"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_PORT"] == "5432"
assert result["GYXX_FEISHU_MODE"] == "legacy"
assert result["GYXX_POSTGRES_MODE"] == "cloud"
assert result["GYXX_HERMES_MODE"] == "local"
assert result["HERMES_ANALYZER_URL"].startswith("http://127.0.0.1:")
assert result["COLLECTOR_API_SERVER_URL"].startswith("http://127.0.0.1:")
assert result["ANALYZER_HERMES_GATEWAY_URL"].endswith(":8642/v1")
assert result["COLLECTOR_HERMES_GATEWAY_URL"].endswith(":8643/v1")
assert "secret-sentinel" not in repr(policy)
@pytest.mark.parametrize(
("environment", "message"),
[
({"PG_HOST": "127.0.0.1"}, "remote"),
({"DB_HOST": "localhost"}, "remote"),
({"AUTOFLOW_PG_HOST": "::1"}, "remote"),
({"DATABASE_URL": "postgresql://user@127.0.0.1/gyxx"}, "remote"),
({"HERMES_ANALYZER_URL": "https://remote.example.test/v1"}, "local"),
({"ANALYZER_API_SERVER_URL": "http://10.0.0.8:8642/v1"}, "local"),
({"COLLECTOR_HERMES_GATEWAY_URL": "https://remote.example.test/v1"}, "local"),
],
)
def test_service_policy_rejects_local_database_or_remote_hermes(
environment: dict[str, str], message: str
) -> None:
with pytest.raises(RuntimeIntegrationError, match=message):
RuntimeServicePolicy().apply(environment)
def test_service_policy_supplies_portable_local_pg_defaults_without_password() -> None:
result = RuntimeServicePolicy(
postgres_mode="local",
postgres_host="127.0.0.1",
postgres_database="gyxx_super_data",
postgres_user="gyxx_flow",
).apply({})
assert result["PG_HOST"] == "127.0.0.1"
assert result["DB_HOST"] == "127.0.0.1"
assert result["AUTOFLOW_PG_HOST"] == "127.0.0.1"
assert result["PG_PORT"] == "5432"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_PORT"] == "5432"
assert result["PG_DB"] == "gyxx_super_data"
assert result["DB_NAME"] == "gyxx_super_data"
assert result["PG_USER"] == "gyxx_flow"
assert "PG_PASSWORD" not in result
def test_service_policy_maps_canonical_secrets_without_exposing_defaults() -> None:
result = RuntimeServicePolicy().apply(
{
"GYXX_POSTGRES_PASSWORD": "placeholder",
"GYXX_HERMES_API_KEY": "placeholder",
}
)
assert result["PG_PASSWORD"] == "placeholder"
assert result["DB_PASSWORD"] == "placeholder"
assert result["AUTOFLOW_PG_PASSWORD"] == "placeholder"
assert result["HERMES_API_KEY"] == "placeholder"
assert result["HERMES_ANALYZER_TOKEN"] == "placeholder"
assert result["GYXX_SUPPLY_HERMES_TOKEN"] == "placeholder"
def test_shared_hermes_resolver_reads_runtime_profile_without_copying_secret(
tmp_path: Path,
) -> None:
profile = tmp_path / "profiles" / "data-analyzer"
profile.mkdir(parents=True)
(profile / ".env").write_text(
"API_SERVER_PORT=8642\nAPI_SERVER_KEY=local-only-key\n",
encoding="utf-8",
)
assert resolve_hermes_profile_api_key(
"data-analyzer",
{"HERMES_HOME": str(tmp_path)},
) == "local-only-key"
def test_shared_hermes_resolver_prefers_explicit_runtime_secret(tmp_path: Path) -> None:
assert resolve_hermes_profile_api_key(
"data-analyzer",
{
"HERMES_HOME": str(tmp_path),
"HERMES_ANALYZER_TOKEN": "explicit-key",
},
preferred_environment_names=("HERMES_ANALYZER_TOKEN",),
) == "explicit-key"
def test_service_policy_replaces_present_but_blank_local_pg_aliases() -> None:
result = RuntimeServicePolicy(
postgres_mode="local",
postgres_host="127.0.0.1",
postgres_database="gyxx_super_data",
postgres_user="gyxx_flow",
).apply(
{"PG_HOST": "", "DB_PORT": " ", "AUTOFLOW_PG_DB": ""}
)
assert result["PG_HOST"] == "127.0.0.1"
assert result["DB_PORT"] == "5432"
assert result["AUTOFLOW_PG_DB"] == "gyxx_super_data"
def test_service_policy_maps_cloud_dsn_to_all_database_aliases() -> None:
result = RuntimeServicePolicy().apply(
{"GYXX_POSTGRES_DSN": "postgresql://cloud_user:placeholder@db.example.test/data_hub"}
)
assert result["GYXX_POSTGRES_MODE"] == "cloud"
assert result["PG_HOST"] == "db.example.test"
assert result["DB_HOST"] == "db.example.test"
assert result["AUTOFLOW_PG_HOST"] == "db.example.test"
assert result["PG_DB"] == "data_hub"
assert result["AUTOFLOW_PG_USER"] == "cloud_user"
def test_registry_rejects_missing_duplicate_or_unknown_script_allocations(tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
payload = json.loads(config.read_text(encoding="utf-8"))
payload["scripts"]["demo.second"]["cdp_port"] = 22001
config.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeIntegrationError, match="unique"):
RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")
payload["scripts"].pop("demo.second")
payload["scripts"]["demo.unknown"] = {
"script_id": "demo:unknown.py",
"state_key": "demo:unknown.py",
"aliases": ["demo:unknown.py"],
"cdp_port": 22003,
}
config.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeIntegrationError, match="coverage"):
RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data")
def test_nested_script_browser_arguments_are_rebound(monkeypatch, tmp_path: Path) -> None:
scripts = _catalog(tmp_path)
config = tmp_path / "bindings.json"
_write_config(config)
binding = RuntimeIntegrationCatalog.load(
config, scripts=scripts, data_root=tmp_path / "data"
).binding_for("demo:nested/second.py")
monkeypatch.setattr(
"sys.argv",
[
"second.py",
"--user-data-dir",
"parent-profile",
"--cdp-url=http://127.0.0.1:22001",
"--cdp-port",
"22001",
],
)
_rewrite_browser_arguments(binding)
assert sys.argv[2] == str(binding.profile_dir)
assert sys.argv[3] == f"--cdp-url={binding.cdp_url}"
assert sys.argv[5] == str(binding.cdp_port)