80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import ast
|
|
import re
|
|
import tomllib
|
|
import warnings
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
SOURCE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow"
|
|
SOURCE_RESOURCE_EXCLUSIONS = {
|
|
SOURCE_ROOT
|
|
/ "modules"
|
|
/ "product_commerce"
|
|
/ "vendors"
|
|
/ "dy-data-flow"
|
|
/ "dynamic_session_src.py",
|
|
}
|
|
BLOCKED_BROWSER_PACKAGES = {"patchright", "playwright", "selenium"}
|
|
BLOCKED_ENGINE_FACTORIES = {"async_playwright", "sync_playwright"}
|
|
BLOCKED_ENGINE_METHODS = {
|
|
"connect_over_cdp",
|
|
"launch_persistent_context",
|
|
}
|
|
|
|
|
|
def _native_browser_references(path: Path) -> list[str]:
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore", SyntaxWarning)
|
|
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
|
failures: list[str] = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.partition(".")[0] in BLOCKED_BROWSER_PACKAGES:
|
|
failures.append(f"line {node.lineno}: import {alias.name}")
|
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
if node.module.partition(".")[0] in BLOCKED_BROWSER_PACKAGES:
|
|
failures.append(f"line {node.lineno}: from {node.module} import ...")
|
|
elif (
|
|
isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Name)
|
|
and node.func.id in BLOCKED_ENGINE_FACTORIES
|
|
):
|
|
failures.append(f"line {node.lineno}: {node.func.id}()")
|
|
elif (
|
|
isinstance(node, ast.Call)
|
|
and isinstance(node.func, ast.Attribute)
|
|
and node.func.attr in BLOCKED_ENGINE_METHODS
|
|
):
|
|
failures.append(f"line {node.lineno}: .{node.func.attr}()")
|
|
elif isinstance(node, ast.Attribute) and node.attr == "playwright":
|
|
failures.append(f"line {node.lineno}: .playwright")
|
|
return failures
|
|
|
|
|
|
def test_production_code_uses_scrapling_instead_of_native_browser_packages() -> None:
|
|
failures: list[str] = []
|
|
for path in SOURCE_ROOT.rglob("*.py"):
|
|
if path in SOURCE_RESOURCE_EXCLUSIONS:
|
|
continue
|
|
references = _native_browser_references(path)
|
|
if references:
|
|
relative = path.relative_to(PROJECT_ROOT).as_posix()
|
|
failures.extend(f"{relative}: {reference}" for reference in references)
|
|
|
|
assert failures == []
|
|
|
|
|
|
def test_project_declares_only_scrapling_as_direct_browser_framework() -> None:
|
|
project = tomllib.loads((PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8"))
|
|
dependencies = project["project"]["dependencies"]
|
|
normalized = {
|
|
re.split(r"[\[<>=!~]", item, maxsplit=1)[0].casefold()
|
|
for item in dependencies
|
|
}
|
|
|
|
assert not (normalized & BLOCKED_BROWSER_PACKAGES)
|
|
assert "scrapling[fetchers]==0.4.13" in dependencies
|