from __future__ import annotations import json import os import subprocess import sys from pathlib import Path import pytest from gyxx_flow.adapters import ( BrowserCookieStore, RuntimeIntegrationCatalog, RuntimeIntegrationError, RuntimeServicePolicy, binding_from_environment, environment_for_child_script, resolve_hermes_profile_api_key, ) from gyxx_flow.adapters.bootstrap import _rewrite_browser_arguments from gyxx_flow.adapters.native import ModuleCommandAdapter, ModuleSourceRoots from gyxx_flow.catalog import WorkflowEntry from gyxx_flow.core.context import RunContext from gyxx_flow.script_catalog import ScriptCatalog, ScriptEntry def _catalog(tmp_path: Path) -> ScriptCatalog: root = tmp_path / "runtime" root.mkdir() first = root / "first.py" second = root / "nested" / "second.py" second.parent.mkdir() first.write_text("print('first')\n", encoding="utf-8") second.write_text("print('second')\n", encoding="utf-8") return ScriptCatalog( ( ScriptEntry("demo.first", "demo", "first.py", "python", first), ScriptEntry( "demo.second", "demo", "nested/second.py", "python", second, ), ) ) def _write_config(path: Path) -> None: path.write_text( json.dumps( { "schema_version": 2, "cdp_host": "127.0.0.1", "scripts": { "demo.first": { "script_id": "demo:first.py", "state_key": "demo:first.py", "aliases": ["demo:first.py"], "cdp_port": 22001, "login_mode": "C", "required_cookie_domains": ["example.test"], "required_cookie_names": ["session"], "credential_env_names": ["DEMO_ACCOUNT", "DEMO_PASSWORD"], }, "demo.second": { "script_id": "demo:nested/second.py", "state_key": "demo:nested/second.py", "aliases": ["demo:nested/second.py"], "cdp_port": 22002, }, }, "services": { "feishu": "legacy", "postgres": "cloud", "hermes": "local", "hermes_url": "http://127.0.0.1:8642/v1/chat/completions", "hermes_collector_url": "http://127.0.0.1:8643/v1/chat/completions", "hermes_analyzer_gateway_url": "http://127.0.0.1:8642/v1", "hermes_collector_gateway_url": "http://127.0.0.1:8643/v1", }, } ), encoding="utf-8", ) def test_project_registry_covers_every_script_with_unique_stable_port() -> None: project_root = Path(__file__).resolve().parents[1] scripts = ScriptCatalog.discover_default() first = RuntimeIntegrationCatalog.load( project_root / "config" / "runtime-bindings.json", scripts=scripts, data_root=project_root / "var", ) second = RuntimeIntegrationCatalog.load( project_root / "config" / "runtime-bindings.json", scripts=scripts, data_root=project_root / "var", ) assert len(scripts.command_ids) == 33 assert len(first.command_ids) == 137 bindings = [first.binding_for(command_id) for command_id in first.command_ids] assert len({binding.cdp_port for binding in bindings}) == 137 assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings) assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings) assert bindings == [second.binding_for(item.script_id) for item in bindings] appeal = first.binding_for("shop.douyin_price_appeal") assert appeal is first.binding_for( "shop_intelligence:collectors/dy_store_competitor_store_scraping.py" ) assert appeal.cdp_port == 22104 def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) data_root = tmp_path / "portable-data" catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=data_root) first = catalog.binding_for("demo:first.py") second = catalog.binding_for("demo:nested/second.py") assert first.profile_dir != second.profile_dir assert first.cookie_file != second.cookie_file assert first.storage_state_file != second.storage_state_file for path in ( first.profile_dir, first.cookie_file, first.storage_state_file, second.profile_dir, ): assert path.is_relative_to(data_root.resolve()) assert not first.profile_dir.exists() def test_schema_v1_keeps_legacy_script_ids_and_existing_state_paths(tmp_path: Path) -> None: scripts = _catalog(tmp_path) stable_config = tmp_path / "stable.json" legacy_config = tmp_path / "legacy.json" _write_config(stable_config) payload = json.loads(stable_config.read_text(encoding="utf-8")) payload["schema_version"] = 1 payload["scripts"] = { item["script_id"]: item["cdp_port"] for item in payload["scripts"].values() } legacy_config.write_text(json.dumps(payload), encoding="utf-8") stable = RuntimeIntegrationCatalog.load( stable_config, scripts=scripts, data_root=tmp_path / "data" ) legacy = RuntimeIntegrationCatalog.load( legacy_config, scripts=scripts, data_root=tmp_path / "data" ) assert stable.binding_for("demo.first").profile_dir == legacy.binding_for( "demo:first.py" ).profile_dir assert stable.binding_for("demo.first").cookie_file == legacy.binding_for( "demo:first.py" ).cookie_file def test_state_key_survives_a_physical_entry_move_and_old_alias_still_resolves( tmp_path: Path, ) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) before = RuntimeIntegrationCatalog.load( config, scripts=scripts, data_root=tmp_path / "data" ).binding_for("demo.first") payload = json.loads(config.read_text(encoding="utf-8")) payload["scripts"]["demo.first"]["script_id"] = "demo:moved/first.py" config.write_text(json.dumps(payload), encoding="utf-8") moved_path = tmp_path / "runtime" / "moved" / "first.py" moved_path.parent.mkdir() moved_path.write_text("print('moved')\n", encoding="utf-8") moved_scripts = ScriptCatalog( (ScriptEntry("demo.first", "demo", "moved/first.py", "python", moved_path),) ) catalog = RuntimeIntegrationCatalog.load( config, scripts=moved_scripts, data_root=tmp_path / "data" ) after = catalog.binding_for("demo.first") assert catalog.binding_for("demo:first.py") is after assert catalog.binding_for("demo:moved/first.py") is after assert after.profile_dir == before.profile_dir assert after.cookie_file == before.cookie_file assert after.storage_state_file == before.storage_state_file def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") parent = catalog.environment_for("demo:first.py", {"KEEP": "yes"}) child = catalog.environment_for("demo:nested/second.py", parent) assert child["KEEP"] == "yes" assert parent["GYXX_SCRIPT_ID"] == "demo:first.py" assert child["GYXX_SCRIPT_ID"] == "demo:nested/second.py" assert parent["GYXX_COMMAND_ID"] == "demo.first" assert child["GYXX_COMMAND_ID"] == "demo.second" assert parent["GYXX_BROWSER_STATE_KEY"] == "demo:first.py" assert parent["GYXX_BROWSER_LOGIN_MODE"] == "C" assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_DOMAINS"]) == [ "example.test" ] assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_NAMES"]) == ["session"] assert json.loads(parent["GYXX_BROWSER_CREDENTIAL_ENV_NAMES"]) == [ "DEMO_ACCOUNT", "DEMO_PASSWORD", ] assert parent["DY_COOKIES_FILE"] == parent["GYXX_BROWSER_COOKIE_FILE"] assert parent["GYXX_BROWSER_CDP_PORT"] != child["GYXX_BROWSER_CDP_PORT"] assert parent["GYXX_BROWSER_PROFILE_DIR"] != child["GYXX_BROWSER_PROFILE_DIR"] assert parent["GYXX_BROWSER_COOKIE_FILE"] != child["GYXX_BROWSER_COOKIE_FILE"] reconstructed = binding_from_environment(parent) assert reconstructed.login_mode == "C" assert reconstructed.required_cookie_domains == ("example.test",) assert reconstructed.required_cookie_names == ("session",) assert reconstructed.credential_env_names == ("DEMO_ACCOUNT", "DEMO_PASSWORD") def test_private_nested_child_keeps_legacy_alias_binding(tmp_path: Path) -> None: project_root = Path(__file__).resolve().parents[1] module_root = ( project_root / "src" / "gyxx_flow" / "modules" / "product_commerce" / "runtime" ) environment = environment_for_child_script( module_root / "taobao_sycm_products.py", { "GYXX_MODULE_ID": "product_commerce", "GYXX_MODULE_ROOT": str(module_root), "GYXX_PROJECT_ROOT": str(project_root), "GYXX_DATA_ROOT": str(tmp_path / "data"), }, ) assert environment["GYXX_SCRIPT_ID"] == "product_commerce:taobao_sycm_products.py" assert environment["GYXX_COMMAND_ID"] == "product_commerce:taobao_sycm_products.py" assert environment["GYXX_BROWSER_STATE_KEY"] == ( "product_commerce:taobao_sycm_products.py" ) assert environment["GYXX_BROWSER_CDP_PORT"] == "22096" def test_jd_self_operated_collectors_consume_their_own_browser_binding( tmp_path: Path, ) -> None: project_root = Path(__file__).resolve().parents[1] scripts = ScriptCatalog.discover_default() catalog = RuntimeIntegrationCatalog.load( project_root / "config" / "runtime-bindings.json", scripts=scripts, data_root=tmp_path / "portable-data", ) module_root = ( project_root / "src" / "gyxx_flow" / "modules" / "shop_intelligence" ) cases = ( ( "shop_intelligence:collectors/jd_self_operated_brand_daily.py", "gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily", ), ( "shop_intelligence:collectors/jd_self_operated_product_daily.py", "gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily", ), ) observed_ports: list[int] = [] for script_id, module_name in cases: binding = catalog.binding_for(script_id) environment = catalog.environment_for(script_id, os.environ) environment["PYTHONPATH"] = os.pathsep.join( ( str(project_root / "src"), str(module_root), environment.get("PYTHONPATH", ""), ) ) command = ( "import json; " f"import {module_name} as module; " "print(json.dumps({" "'port': module.CDP_PORT, " "'profile': str(module.BROWSER_PROFILE_DIR), " "'storage': str(module.STORAGE_STATE_PATH)" "}))" ) result = subprocess.run( [sys.executable, "-c", command], cwd=module_root, env=environment, capture_output=True, text=True, encoding="utf-8", timeout=30, check=False, ) assert result.returncode == 0, result.stderr payload = json.loads(result.stdout.strip().splitlines()[-1]) assert payload == { "port": binding.cdp_port, "profile": str(binding.profile_dir), "storage": str(binding.storage_state_file), } observed_ports.append(payload["port"]) assert observed_ports == [22132, 22133] def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) catalog = RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") runtime_root = tmp_path / "runtime" adapter = ModuleCommandAdapter( ModuleSourceRoots({"demo": runtime_root}), base_env={"KEEP": "yes"}, project_root=tmp_path, data_root=tmp_path / "data", integration_catalog=catalog, ) entry = WorkflowEntry("demo.workflow", "demo", "manual", "first.py") context = RunContext.create("demo.workflow", "2026-07-27") command = adapter.build(entry, context=context) assert command.env["KEEP"] == "yes" assert command.env["GYXX_SCRIPT_ID"] == "demo:first.py" assert command.env["GYXX_COMMAND_ID"] == "demo.first" assert command.env["GYXX_BROWSER_STATE_KEY"] == "demo:first.py" assert command.env["GYXX_BROWSER_CDP_PORT"] == "22001" def test_cookie_store_round_trip_and_storage_state_are_atomic(tmp_path: Path) -> None: store = BrowserCookieStore( cookie_file=tmp_path / "cookies" / "cookies.json", storage_state_file=tmp_path / "cookies" / "storage_state.json", ) cookies = [{"name": "session", "value": "secret-sentinel", "domain": ".example.test", "path": "/"}] state = {"cookies": cookies, "origins": []} assert store.load_cookies() == [] assert store.load_storage_state() is None store.save_cookies(cookies) store.save_storage_state(state) assert store.load_cookies() == cookies assert store.load_storage_state() == state assert not list((tmp_path / "cookies").glob("*.tmp")) assert "secret-sentinel" not in repr(store) def test_service_policy_preserves_feishu_and_uses_cloud_pg_and_local_hermes() -> None: policy = RuntimeServicePolicy( hermes_url="http://127.0.0.1:8642/v1/chat/completions" ) original = { "LARK_PROFILE": "original-profile", "PG_HOST": "cloud-db.example.test", "PG_PORT": "5432", "PG_PASSWORD": "placeholder", "DB_HOST": "", "CUSTOM": "keep", } result = policy.apply(original) assert result["LARK_PROFILE"] == "original-profile" assert result["PG_HOST"] == "cloud-db.example.test" assert result["PG_PASSWORD"] == "placeholder" assert result["DB_HOST"] == "cloud-db.example.test" assert result["AUTOFLOW_PG_HOST"] == "cloud-db.example.test" assert result["DB_PORT"] == "5432" assert result["AUTOFLOW_PG_PORT"] == "5432" assert result["GYXX_FEISHU_MODE"] == "legacy" assert result["GYXX_POSTGRES_MODE"] == "cloud" assert result["GYXX_HERMES_MODE"] == "local" assert result["HERMES_ANALYZER_URL"].startswith("http://127.0.0.1:") assert result["COLLECTOR_API_SERVER_URL"].startswith("http://127.0.0.1:") assert result["ANALYZER_HERMES_GATEWAY_URL"].endswith(":8642/v1") assert result["COLLECTOR_HERMES_GATEWAY_URL"].endswith(":8643/v1") assert "secret-sentinel" not in repr(policy) @pytest.mark.parametrize( ("environment", "message"), [ ({"PG_HOST": "127.0.0.1"}, "remote"), ({"DB_HOST": "localhost"}, "remote"), ({"AUTOFLOW_PG_HOST": "::1"}, "remote"), ({"DATABASE_URL": "postgresql://user@127.0.0.1/gyxx"}, "remote"), ({"HERMES_ANALYZER_URL": "https://remote.example.test/v1"}, "local"), ({"ANALYZER_API_SERVER_URL": "http://10.0.0.8:8642/v1"}, "local"), ({"COLLECTOR_HERMES_GATEWAY_URL": "https://remote.example.test/v1"}, "local"), ], ) def test_service_policy_rejects_local_database_or_remote_hermes( environment: dict[str, str], message: str ) -> None: with pytest.raises(RuntimeIntegrationError, match=message): RuntimeServicePolicy().apply(environment) def test_service_policy_supplies_portable_local_pg_defaults_without_password() -> None: result = RuntimeServicePolicy( postgres_mode="local", postgres_host="127.0.0.1", postgres_database="gyxx_super_data", postgres_user="gyxx_flow", ).apply({}) assert result["PG_HOST"] == "127.0.0.1" assert result["DB_HOST"] == "127.0.0.1" assert result["AUTOFLOW_PG_HOST"] == "127.0.0.1" assert result["PG_PORT"] == "5432" assert result["DB_PORT"] == "5432" assert result["AUTOFLOW_PG_PORT"] == "5432" assert result["PG_DB"] == "gyxx_super_data" assert result["DB_NAME"] == "gyxx_super_data" assert result["PG_USER"] == "gyxx_flow" assert "PG_PASSWORD" not in result def test_service_policy_maps_canonical_secrets_without_exposing_defaults() -> None: result = RuntimeServicePolicy().apply( { "GYXX_POSTGRES_PASSWORD": "placeholder", "GYXX_HERMES_API_KEY": "placeholder", } ) assert result["PG_PASSWORD"] == "placeholder" assert result["DB_PASSWORD"] == "placeholder" assert result["AUTOFLOW_PG_PASSWORD"] == "placeholder" assert result["HERMES_API_KEY"] == "placeholder" assert result["HERMES_ANALYZER_TOKEN"] == "placeholder" assert result["GYXX_SUPPLY_HERMES_TOKEN"] == "placeholder" def test_shared_hermes_resolver_reads_runtime_profile_without_copying_secret( tmp_path: Path, ) -> None: profile = tmp_path / "profiles" / "data-analyzer" profile.mkdir(parents=True) (profile / ".env").write_text( "API_SERVER_PORT=8642\nAPI_SERVER_KEY=local-only-key\n", encoding="utf-8", ) assert resolve_hermes_profile_api_key( "data-analyzer", {"HERMES_HOME": str(tmp_path)}, ) == "local-only-key" def test_shared_hermes_resolver_prefers_explicit_runtime_secret(tmp_path: Path) -> None: assert resolve_hermes_profile_api_key( "data-analyzer", { "HERMES_HOME": str(tmp_path), "HERMES_ANALYZER_TOKEN": "explicit-key", }, preferred_environment_names=("HERMES_ANALYZER_TOKEN",), ) == "explicit-key" def test_service_policy_replaces_present_but_blank_local_pg_aliases() -> None: result = RuntimeServicePolicy( postgres_mode="local", postgres_host="127.0.0.1", postgres_database="gyxx_super_data", postgres_user="gyxx_flow", ).apply( {"PG_HOST": "", "DB_PORT": " ", "AUTOFLOW_PG_DB": ""} ) assert result["PG_HOST"] == "127.0.0.1" assert result["DB_PORT"] == "5432" assert result["AUTOFLOW_PG_DB"] == "gyxx_super_data" def test_service_policy_maps_cloud_dsn_to_all_database_aliases() -> None: result = RuntimeServicePolicy().apply( {"GYXX_POSTGRES_DSN": "postgresql://cloud_user:placeholder@db.example.test/data_hub"} ) assert result["GYXX_POSTGRES_MODE"] == "cloud" assert result["PG_HOST"] == "db.example.test" assert result["DB_HOST"] == "db.example.test" assert result["AUTOFLOW_PG_HOST"] == "db.example.test" assert result["PG_DB"] == "data_hub" assert result["AUTOFLOW_PG_USER"] == "cloud_user" def test_registry_rejects_missing_duplicate_or_unknown_script_allocations(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) payload = json.loads(config.read_text(encoding="utf-8")) payload["scripts"]["demo.second"]["cdp_port"] = 22001 config.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeIntegrationError, match="unique"): RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") payload["scripts"].pop("demo.second") payload["scripts"]["demo.unknown"] = { "script_id": "demo:unknown.py", "state_key": "demo:unknown.py", "aliases": ["demo:unknown.py"], "cdp_port": 22003, } config.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeIntegrationError, match="coverage"): RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") def test_nested_script_browser_arguments_are_rebound(monkeypatch, tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) binding = RuntimeIntegrationCatalog.load( config, scripts=scripts, data_root=tmp_path / "data" ).binding_for("demo:nested/second.py") monkeypatch.setattr( "sys.argv", [ "second.py", "--user-data-dir", "parent-profile", "--cdp-url=http://127.0.0.1:22001", "--cdp-port", "22001", ], ) _rewrite_browser_arguments(binding) assert sys.argv[2] == str(binding.profile_dir) assert sys.argv[3] == f"--cdp-url={binding.cdp_url}" assert sys.argv[5] == str(binding.cdp_port)