from __future__ import annotations import ast import re import tomllib import warnings from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[1] SOURCE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" SOURCE_RESOURCE_EXCLUSIONS = { SOURCE_ROOT / "modules" / "product_commerce" / "vendors" / "dy-data-flow" / "dynamic_session_src.py", } BLOCKED_BROWSER_PACKAGES = {"patchright", "playwright", "selenium"} BLOCKED_ENGINE_FACTORIES = {"async_playwright", "sync_playwright"} BLOCKED_ENGINE_METHODS = { "connect_over_cdp", "launch_persistent_context", } def _native_browser_references(path: Path) -> list[str]: with warnings.catch_warnings(): warnings.simplefilter("ignore", SyntaxWarning) tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) failures: list[str] = [] for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: if alias.name.partition(".")[0] in BLOCKED_BROWSER_PACKAGES: failures.append(f"line {node.lineno}: import {alias.name}") elif isinstance(node, ast.ImportFrom) and node.module: if node.module.partition(".")[0] in BLOCKED_BROWSER_PACKAGES: failures.append(f"line {node.lineno}: from {node.module} import ...") elif ( isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in BLOCKED_ENGINE_FACTORIES ): failures.append(f"line {node.lineno}: {node.func.id}()") elif ( isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in BLOCKED_ENGINE_METHODS ): failures.append(f"line {node.lineno}: .{node.func.attr}()") elif isinstance(node, ast.Attribute) and node.attr == "playwright": failures.append(f"line {node.lineno}: .playwright") return failures def test_production_code_uses_scrapling_instead_of_native_browser_packages() -> None: failures: list[str] = [] for path in SOURCE_ROOT.rglob("*.py"): if path in SOURCE_RESOURCE_EXCLUSIONS: continue references = _native_browser_references(path) if references: relative = path.relative_to(PROJECT_ROOT).as_posix() failures.extend(f"{relative}: {reference}" for reference in references) assert failures == [] def test_project_declares_only_scrapling_as_direct_browser_framework() -> None: project = tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8")) dependencies = project["project"]["dependencies"] normalized = { re.split(r"[\[<>=!~]", item, maxsplit=1)[0].casefold() for item in dependencies } assert not (normalized & BLOCKED_BROWSER_PACKAGES) assert "scrapling[fetchers]==0.4.13" in dependencies