diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..0eb15d3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,38 @@ +image: ghcr.io/astral-sh/uv:python3.12-bookworm-slim + +stages: + - quality + - test + - build + +variables: + UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache" + +cache: + paths: + - .uv-cache/ + +before_script: + - uv sync --frozen --group dev + +lint: + stage: quality + script: + - uv run ruff check src tests + - uv run ruff check src/gyxx_flow/modules/content_marketing/daily_creator_exposure_scope.py src/gyxx_flow/modules/product_commerce/market_rank_limits.py src/gyxx_flow/modules/product_commerce/market_rank_product_import.py + +test: + stage: test + script: + - uv run pytest + - uv run pytest tests/modules/content_marketing/test_daily_creator_exposure_scope.py tests/modules/content_marketing/test_chanmama_refresh_before_export.py tests/modules/content_marketing/test_daily_exposure_fields.py + - uv run pytest tests/modules/product_commerce/test_market_rank_limits.py tests/modules/product_commerce/test_market_rank_product_import.py + +build: + stage: build + script: + - uv build + artifacts: + paths: + - dist/ + expire_in: 7 days diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8ea9986 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,39 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +GYXX Flow is a Python 3.12 `src`-layout package. Framework code lives in `src/gyxx_flow/`; business implementations belong directly in `src/gyxx_flow/modules//`. The former `runtime/` directories are compatibility namespaces only and must not receive new production code. + +All project and module tests live in `tests/`. Configuration lives in `config/`, runbooks in `docs/`, and service scripts in `deploy/`. Treat `var/` as generated state, not source; production deployments must set an external `GYXX_DATA_ROOT`. + +## Build, Test, and Development Commands + +Use `uv` from the repository root: + +```powershell +uv sync --python 3.12 --group dev # create/update the development environment +uv run pytest # run the project test suite +uv run pytest tests/test_cli.py # run a focused test file +uv run ruff check src tests # lint imports and Python errors +uv build # create distribution artifacts +uv run gyxx doctor --json # validate local configuration +uv run gyxx list # inspect registered workflows +``` + +Workflow and command execution defaults to dry-run behavior. Add `--execute` only when real external effects are intended. Public manual commands must be declared in `config/commands.json`; never reintroduce recursive executable-script discovery. + +Production schedules run only through the Python scheduler service; do not add Windows Task Scheduler registration. Manual commands remain dry-run by default, while `gyxx schedule run` executes enabled jobs unless `--dry-run` is supplied. PostgreSQL uses the cloud service selected through runtime-only credentials, both Hermes roles remain loopback-only, Feishu keeps the existing integration behavior, credentials must not have non-empty source defaults, and every browser entry must retain its unique CDP/Profile/Cookie/storage-state binding. + +Each executable `workflow_id` compiles to a LangGraph `StateGraph`. Preserve the public workflow model, journal, lock, shadow, and effect-ledger contracts when changing graph execution. Check upstream drift with `uv run gyxx sources status`; automated source application is limited to untransformed, conflict-free entries and still requires `--execute`. + +## Coding Style & Naming Conventions + +Follow existing Python conventions: four-space indentation, type annotations, concise docstrings, and imports ordered as standard library, third party, then local modules. Ruff enforces `E4`, `E7`, `E9`, `F`, and `I` rules. Use `snake_case` for modules, functions, variables, and test files; `PascalCase` for classes; and uppercase names for constants. Keep cross-module contracts in shared framework packages rather than importing another business module’s internals. + +## Testing Guidelines + +Tests use pytest and follow `test_*.py` / `test_*` naming. Add focused unit tests for new logic and regression tests for fixed defects. Changes to workflows, schedules, manifests, runtime paths, or CLI behavior should include acceptance or boundary coverage. Avoid tests that require live credentials or mutate production services. + +## Commit & Pull Request Guidelines + +Use short Conventional Commit subjects such as `feat: consolidate legacy workflows`. Keep commits scoped. Pull requests should explain behavior, list verification, link issues, and call out configuration or migration effects. Never commit secrets, cookies, tokens, `.env` files, or generated `var/` contents. diff --git a/README.md b/README.md index ac3f0f4..d622b6f 100644 --- a/README.md +++ b/README.md @@ -1,99 +1,156 @@ # GYXX Flow -GYXX Flow 是面向业务自动化的模块化工作流平台，统一管理数据采集、经营分析、供应链处理、定时调度和外部系统集成。 +GYXX Flow 是一个 Python 3.12 业务自动化平台，用 LangGraph 编排内容营销、商品经营、 +店铺分析和供应链工作流，并由项目内 Python 调度器统一定时运行。 -- `content_marketing`：内容与营销采集 -- `product_commerce`：商品、平台和经营分析 -- `shop_intelligence`：店铺与竞店采集 -- `supply_chain`：供应链采集与通知 +项目采用模块化单体：公共调度、工作流、数据和外部系统契约集中维护，业务实现留在各自 +模块内。运行时不依赖其他源码目录，也不使用 Windows Task Scheduler。 -项目采用模块化单体结构。框架能力位于 `src/gyxx_flow`，各业务模块运行时代码位于 -`src/gyxx_flow/modules//runtime`。工作流、脚本、数据目录和外部系统配置均由 -统一入口管理，同时保持模块间高内聚、低耦合。 +## 能力概览 -## 核心能力 +- 23 条调度工作流（内容 8、商品 8、店铺 4、供应链 3），当前全部启用。 +- 工作流目录只保存定时任务；补采、重试、映射刷新和受保护写操作统一由手动命令承载。 +- Python 常驻调度支持日、周、月、间隔日、错过触发补偿、防重复和优雅停止。 +- 31 个显式命令覆盖工作流节点和手动补偿入口；136 条内部浏览器绑定继续使用唯一 CDP、Profile、Cookie 和 storage state。 +- PostgreSQL 使用运行时注入的云端 DSN；地址、数据库、用户和密码均不在源码中提供默认值。 +- Hermes 保持本机 `data-collector`、`data-analyzer` 两个角色；飞书保持既有身份和接口。 +- JSON、Markdown、CSV、Excel、下载文件和截图统一写入可迁移的数据根。 +- `run`、`backfill`、`scripts run` 默认 dry-run，只有 `--execute` 允许真实副作用。 -- 声明式工作流目录、依赖编排、超时、重试和失败恢复 -- 计划任务生成、漂移检查和逐任务部署 -- 手工执行、定时执行、日期回填、dry-run 和 shadow 模式 -- 每次运行使用稳定的 `run_id`，记录步骤状态、日志、产物和外部写入 -- JSON、Markdown、CSV、Excel、下载文件和截图统一分层存储 -- 每个浏览器脚本独立 CDP 端口、Profile、Cookie 和 storage state -- 飞书、云端 PostgreSQL、本机 Hermes 和浏览器能力统一接入 +## 目录 -## 安装与验证 +```text +config/ 工作流、时间表、脚本绑定和服务策略 +deploy/ PostgreSQL 与调度服务部署文件 +docs/ 部署、运维、源码同步和回滚手册 +src/gyxx_flow/ + adapters/ PostgreSQL、Hermes、飞书、浏览器适配边界 + core/ 配置、运行上下文、日志、锁、产物 + workflow/ LangGraph 模型、工厂与执行引擎 + modules// 四个业务域的工作流入口与内部实现 + scheduler_service.py Python 常驻调度器 +tests/ 项目级契约与回归测试 +var/ 默认运行数据；不属于源码 +``` -要求 Python 3.12，推荐使用 `uv`： +业务模块： + +- `content_marketing`：内容指标、达人、评论和营销报告。 +- `product_commerce`：商品数据、人群画像、市场排行、主图和经营分析。 +- `shop_intelligence`：店铺、竞店和京东自营业绩。 +- `supply_chain`：采购确认、补货、库存预警和采购单更新。 + +## 开发环境 ```powershell cd D:\gyxx-flow -uv sync --python 3.12 --extra test +uv sync --python 3.12 --group dev +uv run ruff check src tests uv run pytest +uv build uv run gyxx doctor --json uv run gyxx acceptance status --json ``` -## 运行工作流 +## 云端 PostgreSQL + +由密钥系统向当前进程注入完整云端 DSN： + +```powershell +# 示例只展示变量名；真实 DSN 由部署环境提供 +$env:GYXX_POSTGRES_DSN = '' +``` + +应用优先从 `GYXX_POSTGRES_DSN` 读取云端连接，并映射到各业务模块使用的 `PG_*`、`DB_*` 和 +`AUTOFLOW_PG_*` 变量。源码、示例文件和运行报告均不保存真实地址或凭据；云端模式会拒绝回环数据库地址。 + +## 本机 Hermes + +运行时使用两个本机业务 API base： + +- analyzer：`http://127.0.0.1:8642/v1` +- collector：`http://127.0.0.1:8643/v1` + +`28790/28791` 不作为工作流业务端点。 + +密钥通过 `GYXX_HERMES_API_KEY` 注入。非回环 Hermes 地址会在业务脚本启动前被拒绝。 +不使用 Hermes 的纯采集工作流可在无 AI 环境运行；依赖分析或通知的工作流需要对应本机 +角色可用。 + +## 工作流与脚本 ```powershell uv run gyxx list -uv run gyxx run product.daily --date 2026-07-27 -uv run gyxx run product.daily --date 2026-07-27 --execute +uv run gyxx run product.daily --date 2026-08-01 +uv run gyxx run product.daily --date 2026-08-01 --execute +uv run gyxx scripts list --module shop_intelligence +uv run gyxx scripts run shop.jd_self_operated.collect_product --date 2026-08-01 ``` -`run`、`backfill` 和 `scripts run` 默认都是无副作用 dry-run；只有显式加 -`--execute` 才会启动项目内的业务脚本。 +`gyxx list` 只展示调度工作流。手动补采和维护操作使用 `gyxx scripts run`；已配置的业务日期参数会从 `--date` 自动渲染，仍然只有显式添加 `--execute` 才会真实执行。 -## 运行任意脚本 +主图定时采集只注册为 `product.main_image.weekly`：每周日 08:30 同时启动京东和天猫两个独立分支。每个平台都在各自的采集完成后调用对应插入脚本，分别写入云端 PostgreSQL `main_image_creatives` 和飞书主图表；任一分支失败都不会取消、跳过或回滚另一分支的采集与写入，两个分支结束后工作流再汇总状态，并在失败摘要中标明具体平台和错误。真实写入仍受 `--execute`、凭据、登录态和副作用门禁约束。 + +| 场景 | 当前入口 | +|---|---| +| 内容映射重建 | `gyxx scripts run content.mapping.rebuild --date <日期>` | +| 内容失败任务重试 | `gyxx scripts run content.failed.retry --date <日期>` | +| 内容日报按日期重跑 | `gyxx backfill content.metrics.daily --from <日期> --to <日期>` | +| 京东自营品牌单日回采 | `gyxx scripts run shop.jd_self_operated.collect_brand --date <日期>` | +| 商品历史补采 | `gyxx scripts run product.backfill.run --date <日期>` | +| 商品评价补采 | `gyxx scripts run product.review.orchestrate --date <日期>` | +| 采购单更新 | `gyxx scripts run supply.workflow.run --date <日期>` | + +每次运行生成稳定 `run_id`，并记录图节点状态、日志、产物、资源锁和外部副作用账本。 +业务脚本通过共享适配器取得项目根、数据根、业务日期和服务配置，不应导入其他业务 +模块的内部代码。旧 `module:entry` 脚本 ID 暂时保留为兼容别名。 + +## Python 定时调度 ```powershell -uv run gyxx scripts list -uv run gyxx scripts list --module content_marketing -uv run gyxx scripts run content_marketing:run_all.py --date 2026-07-27 -uv run gyxx scripts run content_marketing:run_all.py --date 2026-07-27 --execute +uv run gyxx schedule run --dry-run --once +uv run gyxx schedule status +uv run gyxx schedule run ``` -脚本 ID 格式为 `:`。Python、BAT/CMD 和 -PowerShell 入口均受统一项目根、数据根、业务日期、run_id 和 shadow 环境约束。 - -## 浏览器与外部系统绑定 - -`config/runtime-bindings.json` 为当前 131 个脚本各自分配固定且唯一的 CDP 端口。 -无论从 workflow、`scripts run` 还是嵌套脚本启动，目标脚本都会重新取得自己的端口和 -`state/browser// + + diff --git a/src/gyxx_flow/workflow/__init__.py b/src/gyxx_flow/workflow/__init__.py index 70d258d..49bf41e 100644 --- a/src/gyxx_flow/workflow/__init__.py +++ b/src/gyxx_flow/workflow/__init__.py @@ -1,6 +1,11 @@ """Workflow definitions, executable steps, and orchestration engine.""" from .engine import WorkflowEngine, WorkflowRunResult +from .graph import ( + WorkflowGraphState, + compile_workflow_graph, + initial_workflow_graph_state, +) from .model import StepDefinition, WorkflowDefinition, WorkflowValidationError from .steps import CommandStep, StepExecution @@ -10,6 +15,9 @@ __all__ = [ "StepExecution", "WorkflowDefinition", "WorkflowEngine", + "WorkflowGraphState", "WorkflowRunResult", "WorkflowValidationError", + "compile_workflow_graph", + "initial_workflow_graph_state", ] diff --git a/src/gyxx_flow/workflow/engine.py b/src/gyxx_flow/workflow/engine.py index aa2af93..d93d54f 100644 --- a/src/gyxx_flow/workflow/engine.py +++ b/src/gyxx_flow/workflow/engine.py @@ -10,6 +10,7 @@ from typing import Callable, Literal from gyxx_flow.core.context import RunContext from gyxx_flow.core.locks import LockManager from gyxx_flow.core.records import RunJournal +from gyxx_flow.core.text import bounded_head_tail from gyxx_flow.ops.effects import ( EffectAlreadyApplied, EffectClaim, @@ -17,12 +18,21 @@ from gyxx_flow.ops.effects import ( EffectStateAmbiguous, ) +from .graph import ( + GraphStepResult, + WorkflowGraphState, + compile_workflow_graph, + initial_workflow_graph_state, +) from .model import StepDefinition, WorkflowDefinition from .steps import StepExecution StepRunStatus = Literal["success", "failed", "skipped"] WorkflowRunStatus = Literal["success", "failed"] +_STEP_FAILURE_DETAIL_MAX_CHARS = 1_000 +_WORKFLOW_ERROR_MAX_CHARS = 4_096 + @dataclass(frozen=True, slots=True) class AttemptResult: @@ -80,60 +90,41 @@ class WorkflowEngine: f"{context.workflow_id!r} != {workflow.workflow_id!r}" ) - results: dict[str, StepRunResult] = {} + def run_step( + step: StepDefinition, + state: WorkflowGraphState, + ) -> GraphStepResult: + prior_results = { + step_id: _step_run_result(result) + for step_id, result in state["step_results"].items() + } + result = self._execute_graph_step( + step, + results=prior_results, + context=context, + journal=journal, + dry_run=dry_run, + ) + return _graph_step_result(result) + + ordered_steps = workflow.ordered_steps() + graph = compile_workflow_graph(workflow, run_step=run_step) with self._lock_manager.acquire( f"workflow:{workflow.workflow_id}", owner=context.run_id ): - for step in workflow.ordered_steps(): - failed_dependency = any( - _blocks_dependents(results[dependency]) - for dependency in step.depends_on - ) - if failed_dependency: - results[step.step_id] = self._record_policy_skip( - step, journal, reason="dependency-failed" - ) - continue - if dry_run: - results[step.step_id] = self._record_policy_skip( - step, journal, reason="dry-run" - ) - continue - if context.shadow and ( - step.production_sink or step.official_notification - ): - results[step.step_id] = self._record_policy_skip( - step, journal, reason="shadow-policy" - ) - continue - claim: EffectClaim | None = None - if step.production_sink and self._effect_ledger is not None: - try: - claim = self._effect_ledger.begin( - workflow_id=context.workflow_id, - business_date=context.business_date.isoformat(), - step_id=step.step_id, - run_id=context.run_id, - ) - except EffectAlreadyApplied: - results[step.step_id] = self._record_policy_skip( - step, journal, reason="idempotency-replay" - ) - continue - except EffectStateAmbiguous: - results[step.step_id] = self._record_policy_failure( - step, journal, reason="effect-state-ambiguous" - ) - continue - result = self._execute_step(step, context=context, journal=journal) - if claim is not None: - if result.status == "success": - self._effect_ledger.mark_applied(claim) - else: - self._effect_ledger.mark_ambiguous( - claim, reason="step-did-not-succeed" - ) - results[step.step_id] = result + final_state = graph.invoke( + initial_workflow_graph_state(context, dry_run=dry_run), + config={"recursion_limit": max(25, len(workflow.steps) + 2)}, + ) + unordered_results = { + step_id: _step_run_result(result) + for step_id, result in final_state["step_results"].items() + } + results = { + step.step_id: unordered_results[step.step_id] + for step in ordered_steps + } + journal.normalize_step_order(tuple(results)) critical_failures = tuple( step_id @@ -148,7 +139,7 @@ class WorkflowEngine: status: WorkflowRunStatus = "failed" if critical_failures else "success" error = None if critical_failures: - error = f"critical steps failed: {', '.join(critical_failures)}" + error = _critical_failure_error(critical_failures, results) journal.finalize(status, error=error) return WorkflowRunResult( workflow_id=workflow.workflow_id, @@ -158,6 +149,66 @@ class WorkflowEngine: warnings=warnings, ) + def _execute_graph_step( + self, + step: StepDefinition, + *, + results: dict[str, StepRunResult], + context: RunContext, + journal: RunJournal, + dry_run: bool, + ) -> StepRunResult: + failed_dependency = any( + _blocks_dependents(results[dependency]) + for dependency in step.depends_on + ) + if failed_dependency and not step.run_after_failure: + return self._record_policy_skip( + step, journal, reason="dependency-failed" + ) + if dry_run: + return self._record_policy_skip(step, journal, reason="dry-run") + if context.shadow and ( + step.production_sink or step.official_notification + ): + return self._record_policy_skip( + step, journal, reason="shadow-policy" + ) + + claim: EffectClaim | None = None + if ( + step.production_sink + and step.replay_policy == "guarded" + and self._effect_ledger is not None + ): + try: + claim = self._effect_ledger.begin( + workflow_id=context.workflow_id, + business_date=context.business_date.isoformat(), + step_id=step.step_id, + run_id=context.run_id, + ) + except EffectAlreadyApplied: + return self._record_policy_skip( + step, journal, reason="idempotency-replay" + ) + except EffectStateAmbiguous: + return self._record_policy_failure( + step, journal, reason="effect-state-ambiguous" + ) + + result = self._execute_step(step, context=context, journal=journal) + if claim is not None: + if result.status == "success": + self._effect_ledger.mark_applied(claim) + elif result.status == "skipped": + self._effect_ledger.cancel(claim) + else: + self._effect_ledger.mark_ambiguous( + claim, reason="step-did-not-succeed" + ) + return result + def _execute_step( self, step: StepDefinition, @@ -320,3 +371,80 @@ def _blocks_dependents(result: StepRunResult) -> bool: def _is_failure(result: StepRunResult) -> bool: return result.status == "failed" or result.reason == "dependency-failed" + + +def _critical_failure_error( + step_ids: tuple[str, ...], + results: dict[str, StepRunResult], +) -> str: + details = "; ".join( + f"{step_id}: {_step_failure_detail(results[step_id])}" + for step_id in step_ids + ) + return _bounded_text( + f"critical steps failed: {details}", + _WORKFLOW_ERROR_MAX_CHARS, + ) + + +def _step_failure_detail(result: StepRunResult) -> str: + attempt = result.attempts[-1] if result.attempts else None + detail = ( + (attempt.error if attempt is not None else None) + or (attempt.reason if attempt is not None else None) + or result.reason + ) + if not detail: + detail = ( + f"exit code {result.final_exit_code}" + if result.final_exit_code is not None + else "unknown failure" + ) + return _bounded_text( + " ".join(str(detail).split()), + _STEP_FAILURE_DETAIL_MAX_CHARS, + ) + + +def _bounded_text(value: str, limit: int) -> str: + return bounded_head_tail(value, limit) + + +def _graph_step_result(result: StepRunResult) -> GraphStepResult: + return { + "step_id": result.step_id, + "status": result.status, + "attempts": [ + { + "attempt": attempt.attempt, + "status": attempt.status, + "exit_code": attempt.exit_code, + "error": attempt.error, + "reason": attempt.reason, + } + for attempt in result.attempts + ], + "final_exit_code": result.final_exit_code, + "critical": result.critical, + "reason": result.reason, + } + + +def _step_run_result(result: GraphStepResult) -> StepRunResult: + return StepRunResult( + step_id=result["step_id"], + status=result["status"], + attempts=tuple( + AttemptResult( + attempt=attempt["attempt"], + status=attempt["status"], + exit_code=attempt["exit_code"], + error=attempt["error"], + reason=attempt["reason"], + ) + for attempt in result["attempts"] + ), + final_exit_code=result["final_exit_code"], + critical=result["critical"], + reason=result["reason"], + ) diff --git a/src/gyxx_flow/workflow/factory.py b/src/gyxx_flow/workflow/factory.py new file mode 100644 index 0000000..b8580ec --- /dev/null +++ b/src/gyxx_flow/workflow/factory.py @@ -0,0 +1,87 @@ +"""Shared catalog-to-LangGraph workflow composition.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from gyxx_flow.adapters.native import DeferredModuleCommandStep, ModuleCommandFactory +from gyxx_flow.catalog import WorkflowEntry + +from .model import StepDefinition, WorkflowDefinition + + +@dataclass(frozen=True, slots=True) +class _ConfiguredStep: + step_id: str + entry: str + args: tuple[str, ...] + depends_on: tuple[str, ...] + run_after_failure: bool = False + timeout_seconds: float | None = None + replay_policy: str | None = None + + +def build_catalog_workflow( + entry: WorkflowEntry, + *, + default_step_id: str, + timeout_seconds: float, + resource: str, + command_factory: ModuleCommandFactory | None = None, + official_notification: bool = False, + independent_step_resources: bool = False, + default_replay_policy: str = "guarded", +) -> WorkflowDefinition: + """Build one executable graph without adding one class per business task.""" + + if entry.steps: + configured_steps = entry.steps + else: + configured_steps = ( + _ConfiguredStep( + step_id=default_step_id, + entry=entry.entry, + args=entry.args, + depends_on=(), + ), + ) + + definitions: list[StepDefinition] = [] + for configured in configured_steps: + action_kwargs = { + "command_entry": configured.entry, + "command_args": configured.args, + } + if command_factory is not None: + action_kwargs["command_factory"] = command_factory + action = DeferredModuleCommandStep(entry, **action_kwargs) + step_resources = ( + (f"{resource}:{configured.step_id}",) + if independent_step_resources + else (resource,) + ) + definitions.append( + StepDefinition( + configured.step_id, + action, + depends_on=configured.depends_on, + timeout_seconds=( + configured.timeout_seconds + if configured.timeout_seconds is not None + else timeout_seconds + ), + max_attempts=1, + resources=step_resources, + production_sink=True, + official_notification=official_notification, + replay_policy=( + getattr(configured, "replay_policy", None) + or default_replay_policy + ), + run_after_failure=getattr(configured, "run_after_failure", False), + ) + ) + return WorkflowDefinition(entry.workflow_id, tuple(definitions)) + + +__all__ = ["build_catalog_workflow"] diff --git a/src/gyxx_flow/workflow/graph.py b/src/gyxx_flow/workflow/graph.py new file mode 100644 index 0000000..31081b4 --- /dev/null +++ b/src/gyxx_flow/workflow/graph.py @@ -0,0 +1,119 @@ +"""LangGraph compilation for deterministic workflow definitions.""" + +from __future__ import annotations + +from typing import Annotated, Callable, Literal, TypedDict + +from langgraph.graph import END, START, StateGraph +from langgraph.graph.state import CompiledStateGraph + +from gyxx_flow.core.context import RunContext + +from .model import StepDefinition, WorkflowDefinition + +GraphStepStatus = Literal["success", "failed", "skipped"] + + +class GraphAttemptResult(TypedDict): + """JSON-serializable attempt result stored in graph state.""" + + attempt: int + status: GraphStepStatus + exit_code: int | None + error: str | None + reason: str | None + + +class GraphStepResult(TypedDict): + """JSON-serializable step result stored in graph state.""" + + step_id: str + status: GraphStepStatus + attempts: list[GraphAttemptResult] + final_exit_code: int | None + critical: bool + reason: str | None + + +class WorkflowGraphState(TypedDict): + """Serializable state passed between one workflow's LangGraph nodes.""" + + workflow_id: str + run_id: str + business_date: str + started_at: str + shadow: bool + dry_run: bool + step_results: Annotated[ + dict[str, GraphStepResult], + _merge_step_results, + ] + + +GraphStepRunner = Callable[ + [StepDefinition, WorkflowGraphState], + GraphStepResult, +] + + +def initial_workflow_graph_state( + context: RunContext, + *, + dry_run: bool, +) -> WorkflowGraphState: + """Create a JSON-serializable initial state from the public run context.""" + + return { + "workflow_id": context.workflow_id, + "run_id": context.run_id, + "business_date": context.business_date.isoformat(), + "started_at": context.started_at.isoformat(), + "shadow": context.shadow, + "dry_run": dry_run, + "step_results": {}, + } + + +def compile_workflow_graph( + workflow: WorkflowDefinition, + *, + run_step: GraphStepRunner, +) -> CompiledStateGraph: + """Compile dependencies into a real LangGraph DAG.""" + + builder = StateGraph(WorkflowGraphState) + ordered_steps = workflow.ordered_steps() + dependent_ids = { + dependency + for step in ordered_steps + for dependency in step.depends_on + } + for step in ordered_steps: + builder.add_node(step.step_id, _step_node(step, run_step)) + if step.depends_on: + builder.add_edge(list(step.depends_on), step.step_id) + else: + builder.add_edge(START, step.step_id) + if step.step_id not in dependent_ids: + builder.add_edge(step.step_id, END) + return builder.compile() + + +def _step_node( + step: StepDefinition, + run_step: GraphStepRunner, +) -> Callable[[WorkflowGraphState], dict[str, object]]: + def execute(state: WorkflowGraphState) -> dict[str, object]: + result = run_step(step, state) + return {"step_results": {step.step_id: result}} + + return execute + + +def _merge_step_results( + current: dict[str, GraphStepResult], + update: dict[str, GraphStepResult], +) -> dict[str, GraphStepResult]: + """Merge branch-local results emitted in the same LangGraph superstep.""" + + return {**current, **update} diff --git a/src/gyxx_flow/workflow/model.py b/src/gyxx_flow/workflow/model.py index 8c9e59d..685f314 100644 --- a/src/gyxx_flow/workflow/model.py +++ b/src/gyxx_flow/workflow/model.py @@ -3,9 +3,10 @@ from __future__ import annotations import heapq +import math import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Literal, Protocol if TYPE_CHECKING: from gyxx_flow.core.context import RunContext @@ -15,6 +16,7 @@ if TYPE_CHECKING: _ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$") _RESOURCE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9:._-]*$") +ReplayPolicy = Literal["guarded", "idempotent"] class WorkflowValidationError(ValueError): @@ -47,6 +49,8 @@ class StepDefinition: resources: tuple[str, ...] = () production_sink: bool = False official_notification: bool = False + replay_policy: ReplayPolicy = "guarded" + run_after_failure: bool = False def __post_init__(self) -> None: object.__setattr__(self, "depends_on", tuple(self.depends_on)) @@ -65,7 +69,10 @@ class StepDefinition: if dependency == self.step_id: raise WorkflowValidationError(f"step {self.step_id!r} depends on itself") if self.timeout_seconds is not None and ( - isinstance(self.timeout_seconds, bool) or self.timeout_seconds <= 0 + isinstance(self.timeout_seconds, bool) + or not isinstance(self.timeout_seconds, (int, float)) + or not math.isfinite(self.timeout_seconds) + or self.timeout_seconds <= 0 ): raise WorkflowValidationError( f"timeout_seconds must be positive for step {self.step_id!r}" @@ -89,6 +96,20 @@ class StepDefinition: raise WorkflowValidationError( f"invalid resource for step {self.step_id!r}: {resource!r}" ) + if not isinstance(self.run_after_failure, bool): + raise WorkflowValidationError( + f"run_after_failure must be boolean for step {self.step_id!r}" + ) + if self.replay_policy not in {"guarded", "idempotent"}: + raise WorkflowValidationError( + f"invalid replay_policy for step {self.step_id!r}: " + f"{self.replay_policy!r}" + ) + if self.replay_policy == "idempotent" and not self.production_sink: + raise WorkflowValidationError( + f"idempotent replay_policy requires production_sink for step " + f"{self.step_id!r}" + ) @dataclass(frozen=True, slots=True) diff --git a/src/gyxx_flow/workflow/steps.py b/src/gyxx_flow/workflow/steps.py index 95d9597..8b8b9c1 100644 --- a/src/gyxx_flow/workflow/steps.py +++ b/src/gyxx_flow/workflow/steps.py @@ -2,12 +2,129 @@ from __future__ import annotations +import os +import signal import subprocess +import tempfile +import time from dataclasses import dataclass from pathlib import Path -from typing import Mapping +from typing import IO, Mapping + +import psutil from gyxx_flow.core.context import RunContext +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.core.text import bounded_head_tail + +_CREATE_SUSPENDED = 0x00000004 +_TERMINATION_GRACE_SECONDS = 0.5 +_TERMINATION_CONFIRM_SECONDS = 5.0 +_FAILURE_OUTPUT_MAX_CHARS = 16_000 + + +class _WindowsJob: + """A private Job Object that contains exactly one command process tree.""" + + def __init__(self) -> None: + if os.name != "nt": + raise OSError("Windows Job Objects are unavailable on this platform") + + import ctypes + from ctypes import wintypes + + class BasicAccountingInformation(ctypes.Structure): + _fields_ = [ + ("TotalUserTime", ctypes.c_longlong), + ("TotalKernelTime", ctypes.c_longlong), + ("ThisPeriodTotalUserTime", ctypes.c_longlong), + ("ThisPeriodTotalKernelTime", ctypes.c_longlong), + ("TotalPageFaultCount", wintypes.DWORD), + ("TotalProcesses", wintypes.DWORD), + ("ActiveProcesses", wintypes.DWORD), + ("TotalTerminatedProcesses", wintypes.DWORD), + ] + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateJobObjectW.argtypes = (wintypes.LPVOID, wintypes.LPCWSTR) + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.OpenProcess.argtypes = ( + wintypes.DWORD, + wintypes.BOOL, + wintypes.DWORD, + ) + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.AssignProcessToJobObject.argtypes = ( + wintypes.HANDLE, + wintypes.HANDLE, + ) + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = (wintypes.HANDLE, wintypes.UINT) + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.QueryInformationJobObject.argtypes = ( + wintypes.HANDLE, + ctypes.c_int, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.LPVOID, + ) + kernel32.QueryInformationJobObject.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + kernel32.CloseHandle.restype = wintypes.BOOL + + handle = kernel32.CreateJobObjectW(None, None) + if not handle: + raise ctypes.WinError(ctypes.get_last_error()) + self._ctypes = ctypes + self._kernel32 = kernel32 + self._accounting_type = BasicAccountingInformation + self._handle = handle + + def assign(self, pid: int) -> None: + process_set_quota = 0x0100 + process_terminate = 0x0001 + process_handle = self._kernel32.OpenProcess( + process_set_quota | process_terminate, + False, + pid, + ) + if not process_handle: + raise self._ctypes.WinError(self._ctypes.get_last_error()) + try: + if not self._kernel32.AssignProcessToJobObject( + self._handle, + process_handle, + ): + raise self._ctypes.WinError(self._ctypes.get_last_error()) + finally: + self._kernel32.CloseHandle(process_handle) + + def terminate(self, exit_code: int) -> None: + if not self._kernel32.TerminateJobObject(self._handle, exit_code): + raise self._ctypes.WinError(self._ctypes.get_last_error()) + + def wait_empty(self, timeout_seconds: float) -> None: + deadline = time.monotonic() + timeout_seconds + while True: + information = self._accounting_type() + if not self._kernel32.QueryInformationJobObject( + self._handle, + 1, + self._ctypes.byref(information), + self._ctypes.sizeof(information), + None, + ): + raise self._ctypes.WinError(self._ctypes.get_last_error()) + if information.ActiveProcesses == 0: + return + if time.monotonic() >= deadline: + raise RuntimeError("command process tree did not terminate") + time.sleep(0.01) + + def close(self) -> None: + if self._handle: + self._kernel32.CloseHandle(self._handle) + self._handle = None @dataclass(frozen=True, slots=True) @@ -59,43 +176,257 @@ class CommandStep: del context if dry_run: return StepExecution(exit_code=0, skipped=True, reason="dry-run") - try: - completed = subprocess.run( - list(self.argv), - cwd=self.cwd, - env=dict(self.env), - shell=False, - capture_output=True, - text=True, - timeout=timeout_seconds, - check=False, - ) - except subprocess.TimeoutExpired as exc: - stdout = _to_text(exc.stdout) - stderr = _to_text(exc.stderr) + with ( + tempfile.TemporaryFile( + mode="w+", encoding="utf-8", errors="replace", newline="" + ) as stdout_file, + tempfile.TemporaryFile( + mode="w+", encoding="utf-8", errors="replace", newline="" + ) as stderr_file, + ): + try: + process, process_scope = _start_process( + list(self.argv), + cwd=self.cwd, + env=dict(self.env), + stdout=stdout_file, + stderr=stderr_file, + ) + except OSError as exc: + return StepExecution(exit_code=127, error=f"cannot start command: {exc}") + + try: + return_code = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired as exc: + _terminate_process_tree(process, process_scope) + stdout = _read_output(stdout_file) + stderr = _read_output(stderr_file) + return StepExecution( + exit_code=124, + stdout=stdout, + stderr=stderr, + error=f"command timed out after {exc.timeout} seconds", + ) + except BaseException: + _terminate_process_tree(process, process_scope) + raise + finally: + if process_scope is not None: + process_scope.close() + + stdout = _read_output(stdout_file) + stderr = _read_output(stderr_file) + + if return_code == COOKIE_SKIP_EXIT_CODE: return StepExecution( - exit_code=124, + exit_code=return_code, stdout=stdout, stderr=stderr, - error=f"command timed out after {exc.timeout} seconds", + skipped=True, + reason="cookie-preflight", ) - except OSError as exc: - return StepExecution(exit_code=127, error=f"cannot start command: {exc}") error = None - if completed.returncode != 0: - error = completed.stderr.strip() or f"command exited with code {completed.returncode}" + if return_code != 0: + error = _command_failure_detail( + stdout, + stderr, + return_code=return_code, + ) return StepExecution( - exit_code=completed.returncode, - stdout=completed.stdout, - stderr=completed.stderr, + exit_code=return_code, + stdout=stdout, + stderr=stderr, error=error, ) -def _to_text(value: str | bytes | None) -> str: - if value is None: - return "" - if isinstance(value, bytes): - return value.decode(errors="replace") - return value +def _start_process( + argv: list[str], + *, + cwd: Path, + env: dict[str, str], + stdout: IO[str], + stderr: IO[str], +) -> tuple[subprocess.Popen[str], _WindowsJob | None]: + common = { + "cwd": cwd, + "env": env, + "shell": False, + "stdout": stdout, + "stderr": stderr, + "text": True, + "encoding": "utf-8", + "errors": "replace", + } + if os.name != "nt": + return subprocess.Popen(argv, start_new_session=True, **common), None + + job = _WindowsJob() + process: subprocess.Popen[str] | None = None + assigned = False + try: + process = subprocess.Popen( + argv, + creationflags=( + _CREATE_SUSPENDED | subprocess.CREATE_NEW_PROCESS_GROUP + ), + **common, + ) + job.assign(process.pid) + assigned = True + psutil.Process(process.pid).resume() + return process, job + except BaseException: + if process is not None: + if assigned: + try: + job.terminate(127) + job.wait_empty(_TERMINATION_CONFIRM_SECONDS) + except (OSError, RuntimeError): + pass + else: + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=_TERMINATION_CONFIRM_SECONDS) + except (OSError, subprocess.TimeoutExpired): + pass + job.close() + raise + + +def start_managed_process( + argv: list[str], + *, + cwd: Path, + env: dict[str, str], + stdout: IO[str], + stderr: IO[str], +) -> tuple[subprocess.Popen[str], _WindowsJob | None]: + """Start a process in a scope that can terminate its complete child tree.""" + + return _start_process( + argv, + cwd=cwd, + env=env, + stdout=stdout, + stderr=stderr, + ) + + +def _terminate_process_tree( + process: subprocess.Popen[str], + process_scope: _WindowsJob | None, +) -> None: + tracked = _process_tree_snapshot(process.pid) + if os.name == "nt": + if process_scope is None: + raise RuntimeError("Windows command process is missing its Job Object") + process_scope.terminate(124) + process_scope.wait_empty(_TERMINATION_CONFIRM_SECONDS) + else: + _terminate_posix_process_group(process.pid) + + try: + process.wait(timeout=_TERMINATION_CONFIRM_SECONDS) + except subprocess.TimeoutExpired as exc: + raise RuntimeError("command root process did not terminate") from exc + _confirm_processes_stopped(tracked) + + +def terminate_managed_process_tree( + process: subprocess.Popen[str], + process_scope: _WindowsJob | None, +) -> None: + """Terminate and confirm the complete tree of a managed process.""" + + _terminate_process_tree(process, process_scope) + + +def _terminate_posix_process_group(process_group_id: int) -> None: + try: + os.killpg(process_group_id, signal.SIGTERM) + except ProcessLookupError: + return + except PermissionError as exc: + raise RuntimeError("cannot terminate command process group") from exc + + deadline = time.monotonic() + _TERMINATION_GRACE_SECONDS + while _posix_group_has_live_processes(process_group_id): + if time.monotonic() >= deadline: + break + time.sleep(0.01) + if _posix_group_has_live_processes(process_group_id): + try: + os.killpg(process_group_id, signal.SIGKILL) + except ProcessLookupError: + return + except PermissionError as exc: + raise RuntimeError("cannot kill command process group") from exc + + deadline = time.monotonic() + _TERMINATION_CONFIRM_SECONDS + while _posix_group_has_live_processes(process_group_id): + if time.monotonic() >= deadline: + raise RuntimeError("command process group did not terminate") + time.sleep(0.01) + + +def _posix_group_has_live_processes(process_group_id: int) -> bool: + for candidate in psutil.process_iter(("pid", "status")): + try: + if ( + os.getpgid(candidate.pid) == process_group_id + and candidate.info["status"] != psutil.STATUS_ZOMBIE + ): + return True + except (OSError, psutil.Error): + continue + return False + + +def _process_tree_snapshot(pid: int) -> tuple[psutil.Process, ...]: + try: + root = psutil.Process(pid) + return (*root.children(recursive=True), root) + except psutil.Error: + return () + + +def _confirm_processes_stopped(processes: tuple[psutil.Process, ...]) -> None: + _, alive = psutil.wait_procs( + list(processes), + timeout=_TERMINATION_CONFIRM_SECONDS, + ) + live = [] + for process in alive: + try: + if process.is_running() and process.status() != psutil.STATUS_ZOMBIE: + live.append(process.pid) + except psutil.Error: + continue + if live: + raise RuntimeError("command descendants did not terminate") + + +def _read_output(handle) -> str: # type: ignore[no-untyped-def] + handle.flush() + handle.seek(0) + return handle.read() + + +def _command_failure_detail( + stdout: str, + stderr: str, + *, + return_code: int, +) -> str: + chunks = [chunk for chunk in (stdout.strip(), stderr.strip()) if chunk] + if not chunks: + return f"command exited with code {return_code}" + return bounded_head_tail( + "\n".join(chunks), + _FAILURE_OUTPUT_MAX_CHARS, + ) diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_analyze_note_no_comments.py b/tests/modules/content_marketing/test_analyze_note_no_comments.py similarity index 80% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_analyze_note_no_comments.py rename to tests/modules/content_marketing/test_analyze_note_no_comments.py index 20cc658..a084285 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_analyze_note_no_comments.py +++ b/tests/modules/content_marketing/test_analyze_note_no_comments.py @@ -4,11 +4,9 @@ import unittest from pathlib import Path from unittest.mock import patch - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -TOOLS_DIR = PROJECT_ROOT / "data" / "tools" - -from gyxx_flow.modules.content_marketing.runtime.data.tools import analyze_note # noqa: E402 +from gyxx_flow.modules.content_marketing.data.tools import ( + analyze_note, # noqa: E402 +) class AnalyzeNoteWithoutCommentsTest(unittest.TestCase): @@ -41,9 +39,11 @@ class AnalyzeNoteWithoutCommentsTest(unittest.TestCase): "--output", str(output), ] - with patch.object(analyze_note, "load_note_from_db", return_value=(info, [])), patch.object( - sys, "argv", argv - ): + with patch.object( + analyze_note, "load_note_from_db", return_value=(info, []) + ), patch.object( + analyze_note, "resolve_layer_output", return_value=output + ), patch.object(sys, "argv", argv): result = analyze_note.main() self.assertEqual(result, 0) diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_batch_rescrape_douyin.py b/tests/modules/content_marketing/test_batch_rescrape_douyin.py similarity index 73% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_batch_rescrape_douyin.py rename to tests/modules/content_marketing/test_batch_rescrape_douyin.py index 57be932..74b74e4 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_batch_rescrape_douyin.py +++ b/tests/modules/content_marketing/test_batch_rescrape_douyin.py @@ -1,4 +1,6 @@ -from gyxx_flow.modules.content_marketing.runtime.data.tools import batch_rescrape_douyin as batch +from gyxx_flow.modules.content_marketing.data.tools import ( + batch_rescrape_douyin as batch, +) def test_result_logged_in_reads_nested_scraper_stats(): diff --git a/tests/modules/content_marketing/test_chanmama_refresh_before_export.py b/tests/modules/content_marketing/test_chanmama_refresh_before_export.py new file mode 100644 index 0000000..dd8bcf2 --- /dev/null +++ b/tests/modules/content_marketing/test_chanmama_refresh_before_export.py @@ -0,0 +1,253 @@ +import pytest + +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.content_marketing import chanmama_scraper as chanmama + + +def test_exposure_state_clicks_text_and_zero_but_keeps_positive_values(): + for value in ("更新数据", " 更新中 ", "暂无数据", "待更新", "0", "0.0", 0): + assert chanmama._classify_exposure_text(value) == "refresh" + for value in (23000, "23,000", "10w+", "8.8万+", ""): + assert chanmama._classify_exposure_text(value) == "ready" + + +def test_refresh_then_export_waits_once_after_all_accounts(monkeypatch): + events = [] + urls = ["account-1", "account-2"] + monkeypatch.setattr(chanmama, "navigate_to_target", lambda _driver, url: events.append(("navigate", url))) + results = iter([ + {"clicked": 2, "pending": 0, "zero_without_button": 0, "pages": 2}, + {"clicked": 0, "pending": 1, "zero_without_button": 0, "pages": 1}, + ]) + monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: next(results)) + monkeypatch.setattr(chanmama.time, "sleep", lambda seconds: events.append(("sleep", seconds))) + monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 123.0) + monkeypatch.setattr(chanmama, "export_video_data", lambda _driver, mtime: events.append(("export", mtime)) or "x.xlsx") + monkeypatch.setattr(chanmama, "parse_chanmama_excel", lambda _path: ([{"title": "video"}], None)) + + records, summaries = chanmama.refresh_then_export_accounts(object(), urls, wait_seconds=600) + + assert len(records) == 2 + assert [item["clicked"] for item in summaries] == [2, 0] + assert events == [ + ("navigate", "account-1"), ("navigate", "account-2"), ("sleep", 600), + ("navigate", "account-1"), ("export", 123.0), + ("navigate", "account-2"), ("export", 123.0), + ] + + +def test_refresh_then_export_skips_wait_when_nothing_needs_refresh(monkeypatch): + sleeps = [] + monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None) + monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: { + "clicked": 0, "pending": 0, "zero_without_button": 0, "pages": 1, + }) + monkeypatch.setattr(chanmama.time, "sleep", sleeps.append) + monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0) + monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None) + + records, _ = chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600) + + assert records == [] + assert sleeps == [] + + +class _RefreshElement: + def __init__(self, text="", css_class="", element_id="node"): + self.text = text + self.css_class = css_class + self.id = element_id + + def is_displayed(self): + return True + + def get_attribute(self, name): + return self.css_class if name == "class" else None + + def click(self): + return None + + +class _RefreshDriver: + def __init__(self, nodes=None): + self.nodes = list(nodes or []) + + def find_elements(self, _by, selector): + return self.nodes if selector == "td .cursor-pointer" else [] + + +def test_next_page_button_keeps_the_source_xpath_fallback(): + expected = "//span[normalize-space()='下一页']/ancestor::button[1]" + button = _RefreshElement("下一页", element_id="next") + + class Driver: + def find_elements(self, _by, selector): + return [button] if selector == expected else [] + + assert chanmama._find_next_page_button(Driver()) is button + + +def test_find_get_data_targets_selects_actual_cursor_pointer_only(): + outer = _RefreshElement("获取数据", "detail-text", "outer") + clickable = _RefreshElement("获取数据", "cp cursor-pointer", "inner") + other = _RefreshElement("查看详情", "cursor-pointer", "other") + assert chanmama._find_get_data_targets(_RefreshDriver([outer, clickable, other])) == [clickable] + + +def test_refresh_zero_exposure_videos_walks_all_pages(monkeypatch): + class Button: + def __init__(self, disabled): + self.disabled = disabled + + class Driver: + page = 0 + + def execute_script(self, _script, _element): + self.page += 1 + + driver = Driver() + page_results = [ + {"clicked": 2, "pending": 1, "zero_without_button": 0}, + {"clicked": 1, "pending": 0, "zero_without_button": 1}, + ] + monkeypatch.setattr( + chanmama, "_page_signature", lambda current: f"page-{current.page}" + ) + monkeypatch.setattr( + chanmama, + "_refresh_current_page", + lambda current: page_results[current.page], + ) + monkeypatch.setattr( + chanmama, + "_find_next_page_button", + lambda current: Button(disabled=current.page == 1), + ) + monkeypatch.setattr( + chanmama, + "_pagination_element_disabled", + lambda button: button.disabled, + ) + monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None) + + assert chanmama.refresh_zero_exposure_videos(driver) == { + "clicked": 3, + "pending": 1, + "zero_without_button": 1, + "unconfirmed": 0, + "pages": 2, + } + + +def test_missing_get_data_button_does_not_trigger_refresh_wait(monkeypatch): + sleeps = [] + monkeypatch.setattr(chanmama, "navigate_to_target", lambda *_args: None) + monkeypatch.setattr(chanmama, "refresh_zero_exposure_videos", lambda _driver: { + "clicked": 0, "pending": 0, "zero_without_button": 1, "pages": 1, + }) + monkeypatch.setattr(chanmama.time, "sleep", sleeps.append) + monkeypatch.setattr(chanmama, "_max_excel_mtime", lambda: 0.0) + monkeypatch.setattr(chanmama, "export_video_data", lambda *_args: None) + chanmama.refresh_then_export_accounts(object(), ["account-1"], wait_seconds=600) + assert sleeps == [] + + +def test_main_reuses_profile_before_cookie_file_or_credentials(monkeypatch): + events = [] + + class Driver: + def quit(self): + events.append("quit") + + monkeypatch.setattr(chanmama, "HAS_SELENIUM", True) + monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"]) + monkeypatch.setattr(chanmama, "create_driver", lambda: Driver()) + monkeypatch.setattr(chanmama, "verify_login", lambda _driver: True) + monkeypatch.setattr(chanmama, "save_cookies", lambda _driver: events.append("saved")) + monkeypatch.setattr( + chanmama, + "load_cookies", + lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")), + ) + monkeypatch.setattr( + chanmama, + "login", + lambda _driver: (_ for _ in ()).throw(AssertionError("profile must win")), + ) + monkeypatch.setattr( + chanmama, + "refresh_then_export_accounts", + lambda *_args, **_kwargs: ([], []), + ) + + assert chanmama.main() == 0 + assert events == ["saved", "quit"] + + +def test_acceptance_skips_when_profile_cookie_and_credentials_fail( + monkeypatch, + tmp_path, +): + evidence = tmp_path / "evidence.jsonl" + + class Driver: + def quit(self): + pass + + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.setattr(chanmama, "HAS_SELENIUM", True) + monkeypatch.setattr(chanmama.sys, "argv", ["chanmama_scraper.py"]) + monkeypatch.setattr(chanmama, "ACCOUNT", "") + monkeypatch.setattr(chanmama, "PASSWORD", "") + monkeypatch.setattr(chanmama, "create_driver", lambda: Driver()) + monkeypatch.setattr(chanmama, "verify_login", lambda _driver: False) + monkeypatch.setattr(chanmama, "load_cookies", lambda _driver: False) + monkeypatch.setattr( + chanmama, + "login", + lambda _driver: (_ for _ in ()).throw(AssertionError("must not login")), + ) + monkeypatch.setattr( + chanmama, + "refresh_then_export_accounts", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("must not scrape") + ), + ) + + assert chanmama.main() == COOKIE_SKIP_EXIT_CODE + assert "profile/cookie session is invalid" in evidence.read_text(encoding="utf-8") + + +def test_acceptance_captcha_is_non_interactive(monkeypatch, tmp_path): + evidence = tmp_path / "evidence.jsonl" + + class Element: + def is_displayed(self): + return True + + class Driver: + current_url = "https://www.chanmama.com/login.html" + page_source = "" + + def find_element(self, *_args): + return Element() + + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.setattr(chanmama.time, "sleep", lambda _seconds: None) + + assert chanmama.handle_captcha(Driver()) is False + assert "captcha or interactive verification" in evidence.read_text( + encoding="utf-8" + ) + + +def test_cli_preserves_cookie_skip_exit_code(monkeypatch): + monkeypatch.setattr(chanmama, "main", lambda: COOKIE_SKIP_EXIT_CODE) + + with pytest.raises(SystemExit) as raised: + chanmama.cli() + + assert raised.value.code == COOKIE_SKIP_EXIT_CODE diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_collection_completeness.py b/tests/modules/content_marketing/test_collection_completeness.py similarity index 98% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_collection_completeness.py rename to tests/modules/content_marketing/test_collection_completeness.py index c65fbe2..0790d26 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_collection_completeness.py +++ b/tests/modules/content_marketing/test_collection_completeness.py @@ -3,12 +3,12 @@ from pathlib import Path import pytest -from gyxx_flow.modules.content_marketing.runtime import collection_completeness as cc -import bilibili_scraper as bili -import pgy_xhs_scraper_v2 as pgy -import run_all -import self_douyin_scraper as self_dy -import xingtu_scraper_v2 as xingtu +from gyxx_flow.modules.content_marketing import bilibili_scraper as bili +from gyxx_flow.modules.content_marketing import collection_completeness as cc +from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy +from gyxx_flow.modules.content_marketing import run_all +from gyxx_flow.modules.content_marketing import self_douyin_scraper as self_dy +from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu def test_normalize_title_handles_nfkc_case_and_zero_width(): diff --git a/tests/modules/content_marketing/test_content_acceptance_feishu_boundaries.py b/tests/modules/content_marketing/test_content_acceptance_feishu_boundaries.py new file mode 100644 index 0000000..ba348f0 --- /dev/null +++ b/tests/modules/content_marketing/test_content_acceptance_feishu_boundaries.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID +from gyxx_flow.modules.content_marketing import chanmama_scraper, weekly_summary_all +from gyxx_flow.modules.content_marketing.data.tools import ( + analyze_comments, + analyze_note, + daily_report_card, + friday_relogin_parallel, +) + + +def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path: + evidence = tmp_path / "content-evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False) + return evidence + + +def test_content_write_back_skips_before_lark_cli( + monkeypatch, + tmp_path: Path, +) -> None: + evidence = _enable_acceptance(monkeypatch, tmp_path) + + def fail_if_called(*args, **kwargs): + raise AssertionError("lark-cli wrapper must not be called") + + monkeypatch.setattr(chanmama_scraper, "_call_lark_json", fail_if_called) + assert chanmama_scraper._write_back( + "base_secret", "tbl_test", "rec_test", "fld_test", 123 + ) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["operation"].endswith("record-upsert") + + +def test_weekly_summary_skips_table_upsert_but_reports_success( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + monkeypatch.setattr( + weekly_summary_all, + "PATHS", + SimpleNamespace(tmp_root=tmp_path), + ) + + def fail_if_called(*args, **kwargs): + raise AssertionError("weekly lark-cli wrapper must not be called") + + monkeypatch.setattr(weekly_summary_all, "call_lark_json", fail_if_called) + assert weekly_summary_all.write_to_target_table( + "base_secret", + "tbl_test", + "fld_time", + "fld_summary", + "style-test", + "https://example.test/doc", + False, + ) + + +def test_daily_card_sends_only_to_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"message_id": "om_test"}), + stderr="", + ) + + monkeypatch.setattr(daily_report_card.subprocess, "run", fake_run) + results = daily_report_card.send_card_to_recipients( + {"schema": "2.0", "body": {"elements": []}}, + ("ou_other_a", "ou_other_b"), + "2026-08-01", + ) + + assert len(results) == 1 + assert len(commands) == 1 + command = commands[0] + assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + + +def test_comment_summary_sends_only_to_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + commands = [] + + def fake_run(command, **kwargs): + commands.append(command) + return SimpleNamespace(returncode=0, stdout="{}", stderr="") + + monkeypatch.setattr(analyze_comments.subprocess, "run", fake_run) + analyze_comments.send_feishu_summary("summary", "ou_other") + + command = commands[0] + assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + + +def test_note_report_sends_only_to_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + commands = [] + monkeypatch.setattr(analyze_note.sys, "platform", "linux") + monkeypatch.setattr( + analyze_note, + "build_card_payload", + lambda *args, **kwargs: {"schema": "2.0"}, + ) + + def fake_run(command, **kwargs): + commands.append(command) + return SimpleNamespace(returncode=0, stdout="{}", stderr="") + + monkeypatch.setattr(analyze_note.subprocess, "run", fake_run) + analyze_note.send_feishu_report( + {}, + {}, + {}, + [], + [], + "analysis", + "ou_other", + tmp_path / "report.md", + ) + + command = commands[0] + assert command[command.index("--user-id") + 1] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + + +def test_relogin_notifications_resolve_only_to_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + + assert ( + friday_relogin_parallel.recipient_for_platform("pgy", "ou_other") + == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + ) diff --git a/tests/modules/content_marketing/test_content_nested_child_bindings.py b/tests/modules/content_marketing/test_content_nested_child_bindings.py new file mode 100644 index 0000000..2f2a97c --- /dev/null +++ b/tests/modules/content_marketing/test_content_nested_child_bindings.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from pathlib import Path + +from gyxx_flow.modules.content_marketing import run_all + + +def test_run_all_rebinds_browser_environment_to_child_entry(monkeypatch) -> None: + observed: dict[str, object] = {} + + def fake_rebind(target, base_environment): + observed["target"] = Path(target) + observed["base"] = dict(base_environment) + return { + **base_environment, + "GYXX_SCRIPT_ID": "content_marketing:pgy_xhs_scraper_v2.py", + "GYXX_BROWSER_CDP_PORT": "child-port", + "GYXX_BROWSER_PROFILE_DIR": "child-profile", + "GYXX_BROWSER_COOKIE_FILE": "child-cookie", + "GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage", + } + + monkeypatch.setattr(run_all, "environment_for_child_script", fake_rebind) + target = run_all.BASE_DIR / "pgy_xhs_scraper_v2.py" + + environment = run_all.build_child_environment( + target, + { + "KEEP": "yes", + "GYXX_BROWSER_CDP_PORT": "parent-port", + "GYXX_BROWSER_PROFILE_DIR": "parent-profile", + }, + ) + + assert observed["target"] == target + assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "parent-port" + assert environment["KEEP"] == "yes" + assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port" + assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile" + assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie" + assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage" + assert environment["PYTHONIOENCODING"] == "utf-8" + assert environment["PYTHONUNBUFFERED"] == "1" diff --git a/tests/modules/content_marketing/test_daily_creator_exposure_scope.py b/tests/modules/content_marketing/test_daily_creator_exposure_scope.py new file mode 100644 index 0000000..efdaf12 --- /dev/null +++ b/tests/modules/content_marketing/test_daily_creator_exposure_scope.py @@ -0,0 +1,100 @@ +from argparse import Namespace +from datetime import date + +import pytest + +from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy +from gyxx_flow.modules.content_marketing import run_all +from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu +from gyxx_flow.modules.content_marketing.daily_creator_exposure_scope import ( + DEFAULT_MAX_AGE_DAYS, + DEFAULT_PUBLISHED_FROM, + EXCLUDED_STYLE_NAMES, + classify_publish_scope, + select_daily_styles, +) + + +def test_daily_scope_excludes_requested_styles(): + styles = [ + {"index": 1, "name": "保留款"}, + {"index": 2, "name": "盖亚微单"}, + {"index": 3, "name": "逐星GT"}, + {"index": 4, "name": "晨星2"}, + {"index": 5, "name": "觅光"}, + ] + assert [style["name"] for style in select_daily_styles(styles)] == ["保留款"] + assert EXCLUDED_STYLE_NAMES == frozenset({"盖亚微单", "逐星GT", "晨星2", "觅光"}) + + +def test_publish_scope_starts_on_july_first_and_stops_at_30_days(): + collected_on = date(2026, 7, 31) + assert classify_publish_scope( + date(2026, 6, 30), + collected_on, + published_from=DEFAULT_PUBLISHED_FROM, + max_age_days=DEFAULT_MAX_AGE_DAYS, + ) == "before_publish_cutoff" + assert classify_publish_scope( + date(2026, 7, 1), + collected_on, + published_from=DEFAULT_PUBLISHED_FROM, + max_age_days=DEFAULT_MAX_AGE_DAYS, + ) == "collection_window_complete" + assert classify_publish_scope( + date(2026, 7, 2), + collected_on, + published_from=DEFAULT_PUBLISHED_FROM, + max_age_days=DEFAULT_MAX_AGE_DAYS, + ) is None + + +def test_run_all_daily_scope_passes_filters_and_no_retry_to_children(): + args = Namespace(dry_run=False, style=[1, 2], daily_scope=True, no_retry=False) + command = run_all.build_process_command("pgy", args) + assert "--skip-field-prepare" in command + assert command[command.index("--published-from") + 1] == "2026-07-01" + assert command[command.index("--max-age-days") + 1] == "30" + assert "--no-retry" in command + + +@pytest.mark.parametrize( + ("module", "platform", "url"), + [ + (pgy, "小红书", "https://www.xiaohongshu.com/explore/eligible"), + (xingtu, "抖音", "https://www.douyin.com/video/eligible"), + ], +) +def test_platform_task_extraction_applies_daily_publish_window( + module, platform, url, monkeypatch +): + style = { + "index": 1, + "name": "保留款", + "base_token": "base", + "table_id": "table", + "field_map": { + "creator_name": {"field_id": "creator"}, + "note_title": {"field_id": "title"}, + "publish_time": {"field_id": "published"}, + "note_url": {"field_id": "url"}, + "platform": {"field_id": "platform"}, + "daily_exposure": {"field_id": "daily", "field_name": "2026-07-29曝光量"}, + }, + } + base = {"creator": "达人", "title": "标题", "url": url, "platform": platform} + records = [ + {"record_id": "old", **base, "published": "2026-06-30"}, + {"record_id": "eligible", **base, "published": "2026-07-02"}, + {"record_id": "missing", **base, "published": None}, + ] + monkeypatch.setattr(module, "list_records_by_table", lambda *_args: records) + tasks = module.extract_target_tasks( + style, + published_from=date(2026, 7, 1), + max_age_days=30, + collection_date=date(2026, 7, 29), + ) + assert [task["record_id"] for task in tasks] == ["eligible"] + assert style["_extract_stats"]["skipped_publish_scope"] == 1 + assert style["_extract_stats"]["skipped_incomplete"] == 1 diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_dashboard_analytics.py b/tests/modules/content_marketing/test_daily_dashboard_analytics.py similarity index 98% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_dashboard_analytics.py rename to tests/modules/content_marketing/test_daily_dashboard_analytics.py index 7bfb452..f710974 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_dashboard_analytics.py +++ b/tests/modules/content_marketing/test_daily_dashboard_analytics.py @@ -1,7 +1,9 @@ import unittest from datetime import date -from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_dashboard_analytics import build_dashboard_facts +from gyxx_flow.modules.content_marketing.data.tools.daily_dashboard_analytics import ( + build_dashboard_facts, +) class DailyDashboardAnalyticsTests(unittest.TestCase): diff --git a/tests/modules/content_marketing/test_daily_exposure_fields.py b/tests/modules/content_marketing/test_daily_exposure_fields.py new file mode 100644 index 0000000..e3ac76b --- /dev/null +++ b/tests/modules/content_marketing/test_daily_exposure_fields.py @@ -0,0 +1,127 @@ +import copy +import json +from datetime import date, datetime +from pathlib import Path + +from gyxx_flow.modules.content_marketing import bilibili_scraper as bili +from gyxx_flow.modules.content_marketing import feishu_mapping +from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy +from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu + +TARGET_DATE = date(2026, 7, 28) +TARGET_NAME = "2026-07-28曝光量" + + +def _mapping(): + return {"tables": [{"index": 1, "name": "款式A", "base_token": "base-a", "table_id": "table-a", "field_map": {}, "all_field_names": []}]} + + +def test_daily_exposure_field_name_uses_collection_date(): + assert feishu_mapping.daily_exposure_field_name(TARGET_DATE) == TARGET_NAME + + +def test_build_field_map_only_maps_exact_requested_daily_field(): + fmap, names = feishu_mapping.build_field_map([ + {"id": "old", "name": "2026-07-27曝光量"}, {"id": "today", "name": TARGET_NAME}, + ], target_date=TARGET_DATE) + assert fmap["daily_exposure"] == {"field_id": "today", "field_name": TARGET_NAME} + assert names == ["2026-07-27曝光量", TARGET_NAME] + + +def test_ensure_daily_field_reuses_existing_field(monkeypatch): + mapping = _mapping() + monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: [{"id": "field-today", "name": TARGET_NAME}]) + calls = [] + monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True}) + result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False) + assert result["ok"] is True and result["created"] == [] and calls == [] + assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"] == "field-today" + + +def test_ensure_daily_field_creates_number_field_then_refreshes(monkeypatch): + mapping = _mapping() + fields = iter([[], [{"id": "new-field", "name": TARGET_NAME}]]) + monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: next(fields)) + calls = [] + monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda args: calls.append(args) or {"ok": True}) + result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, write=False) + assert result["ok"] is True and result["created"] == ["款式A"] + payload = calls[0][calls[0].index("--json") + 1] + assert TARGET_NAME in payload and '"type": "number"' in payload + + +def test_ensure_daily_field_dry_run_does_not_call_feishu(monkeypatch): + mapping = _mapping() + monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: (_ for _ in ()).throw(AssertionError("must not list"))) + monkeypatch.setattr(feishu_mapping, "call_lark_json", lambda *_: (_ for _ in ()).throw(AssertionError("must not create"))) + result = feishu_mapping.ensure_daily_exposure_fields(mapping, target_date=TARGET_DATE, dry_run=True, write=False) + assert result["ok"] is True + assert mapping["tables"][0]["field_map"]["daily_exposure"]["field_id"].startswith("dryrun:") + + +def test_acceptance_skips_field_create_and_keeps_sentinel_in_memory( + monkeypatch, + tmp_path: Path, +): + evidence = tmp_path / "evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + mapping = _mapping() + monkeypatch.setattr(feishu_mapping, "_field_list", lambda *_: []) + monkeypatch.setattr( + feishu_mapping, + "call_lark_json", + lambda *_: (_ for _ in ()).throw(AssertionError("must not create")), + ) + + result = feishu_mapping.ensure_daily_exposure_fields( + mapping, + target_date=TARGET_DATE, + write=False, + ) + + assert result["ok"] is True + assert result["created"] == [] + assert result["skipped_creations"] == ["款式A"] + assert mapping["tables"][0]["field_map"]["daily_exposure"][ + "field_id" + ].startswith("acceptance-skipped:") + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["operation"] == "content.mapping.daily-exposure-field-create" + + +def test_acceptance_hydrates_cached_mapping_without_persisting_sentinel( + monkeypatch, + tmp_path: Path, +): + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + cache = tmp_path / "mapping.json" + cache.write_text(json.dumps(_mapping(), ensure_ascii=False), encoding="utf-8") + + hydrated = feishu_mapping._hydrate_acceptance_daily_field( + json.loads(cache.read_text(encoding="utf-8")), + self_operated=False, + ) + + assert hydrated["tables"][0]["field_map"]["daily_exposure"][ + "field_id" + ].startswith("acceptance-skipped:") + persisted = json.loads(cache.read_text(encoding="utf-8")) + assert "daily_exposure" not in persisted["tables"][0]["field_map"] + + +def test_collaboration_slot_pickers_use_daily_field_without_publish_time(): + fmap = {"daily_exposure": {"field_id": "today-field", "field_name": TARGET_NAME}} + expected = ("daily_exposure", "today-field", TARGET_NAME) + assert pgy.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected + assert xingtu.pick_read_field(fmap, None, datetime(2026, 7, 28)) == expected + + +def test_self_operated_mapping_removes_daily_field(monkeypatch): + source = {"tables": [{"field_map": { + "daily_exposure": {"field_id": "daily", "field_name": TARGET_NAME}, + "read_count_7d": {"field_id": "s7", "field_name": "7天曝光量"}, + }}]} + monkeypatch.setattr(feishu_mapping, "load_mapping", lambda *_args, **_kwargs: copy.deepcopy(source)) + for mapping in (pgy.load_mapping(True), xingtu.load_mapping(True), bili.load_mapping(bili.DEFAULT_DATA_DIR, True)): + assert "daily_exposure" not in mapping["tables"][0]["field_map"] diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_marketing_report.py b/tests/modules/content_marketing/test_daily_marketing_report.py similarity index 97% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_marketing_report.py rename to tests/modules/content_marketing/test_daily_marketing_report.py index aeb6a1c..d19cebe 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_marketing_report.py +++ b/tests/modules/content_marketing/test_daily_marketing_report.py @@ -2,10 +2,23 @@ import unittest from datetime import date from unittest.mock import patch -import daily_marketing_report as daily +from gyxx_flow.modules.content_marketing import daily_marketing_report as daily +from gyxx_flow.modules.content_marketing.data.tools import analyze_comments class DailyMarketingReportTests(unittest.TestCase): + def test_hermes_url_accepts_api_base_or_full_completion_path(self): + full_url = "http://127.0.0.1:8642/v1/chat/completions" + + self.assertEqual( + analyze_comments._chat_completions_url("http://127.0.0.1:8642/v1"), + full_url, + ) + self.assertEqual( + analyze_comments._chat_completions_url(full_url), + full_url, + ) + def test_sales_forecast_uses_seven_day_total_and_recent_three_day_velocity(self): forecast = daily.estimate_sales_forecast([10, 12, 14, 16, 18, 20, 22]) diff --git a/tests/modules/content_marketing/test_daily_marketing_report_schedule.py b/tests/modules/content_marketing/test_daily_marketing_report_schedule.py new file mode 100644 index 0000000..5e142c7 --- /dev/null +++ b/tests/modules/content_marketing/test_daily_marketing_report_schedule.py @@ -0,0 +1,26 @@ +import unittest +from pathlib import Path + +from gyxx_flow.catalog import WorkflowCatalog + +PROJECT_ROOT = Path(__file__).resolve().parents[3] + + +class DailyMarketingReportScheduleTests(unittest.TestCase): + def test_python_scheduler_runs_latest_daily_report(self): + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + workflow = next( + item + for item in catalog.workflows + if item.workflow_id == "content.marketing_report.daily" + ) + schedule = catalog.schedule_for(workflow.workflow_id) + + self.assertEqual(workflow.steps[0].entry, "daily_marketing_report.py") + self.assertEqual(workflow.steps[0].args, ("--send",)) + self.assertEqual(schedule.kind, "daily") + self.assertEqual(schedule.at, "10:00") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_card.py b/tests/modules/content_marketing/test_daily_report_card.py similarity index 91% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_card.py rename to tests/modules/content_marketing/test_daily_report_card.py index 503f86e..d9894c6 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_card.py +++ b/tests/modules/content_marketing/test_daily_report_card.py @@ -6,10 +6,10 @@ from unittest.mock import patch from PIL import Image -from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_report_card import ( +from gyxx_flow.modules.content_marketing.data.tools.daily_report_card import ( build_analysis_card, - build_dashboard_card, build_daily_report_card, + build_dashboard_card, prepare_feishu_image, send_card_to_recipients, send_daily_report_cards, @@ -124,7 +124,10 @@ class DailyReportCardTests(unittest.TestCase): self.assertIn("总销量下降15%", card_text) self.assertIn("48小时内排查", card_text) - @patch("data.tools.daily_report_card.subprocess.run") + @patch( + "gyxx_flow.modules.content_marketing.data.tools." + "daily_report_card.subprocess.run" + ) def test_upload_dashboard_uses_named_image_file_field(self, run): run.return_value = type("Result", (), { "returncode": 0, "stdout": json.dumps({"image_key": "img_v3_test"}), "stderr": "" @@ -134,7 +137,10 @@ class DailyReportCardTests(unittest.TestCase): command = run.call_args.args[0] self.assertEqual(command[command.index("--file") + 1], "image=./dashboard.png") - @patch("data.tools.daily_report_card.subprocess.run") + @patch( + "gyxx_flow.modules.content_marketing.data.tools." + "daily_report_card.subprocess.run" + ) def test_send_card_to_all_recipients(self, run): run.side_effect = [ type("Result", (), {"returncode": 0, "stdout": json.dumps({"message_id": f"om_{i}"}), "stderr": ""})() @@ -152,9 +158,16 @@ class DailyReportCardTests(unittest.TestCase): self.assertTrue(all(key.startswith("daily-2026-07-15-") for key in keys)) self.assertTrue(all(len(key) <= 50 for key in keys)) - @patch("data.tools.daily_report_card.send_card_to_recipients") - @patch("data.tools.daily_report_card.upload_dashboard_image", return_value="img_v3_test") - @patch("data.tools.daily_report_card.prepare_feishu_image") + @patch( + "gyxx_flow.modules.content_marketing.data.tools.daily_report_card.send_card_to_recipients" + ) + @patch( + "gyxx_flow.modules.content_marketing.data.tools.daily_report_card.upload_dashboard_image", + return_value="img_v3_test", + ) + @patch( + "gyxx_flow.modules.content_marketing.data.tools.daily_report_card.prepare_feishu_image" + ) def test_daily_report_sends_dashboard_then_analysis_as_two_messages_per_recipient( self, prepare, upload, send ): diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_charts.py b/tests/modules/content_marketing/test_daily_report_charts.py similarity index 98% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_charts.py rename to tests/modules/content_marketing/test_daily_report_charts.py index 1e66011..6631e2e 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_daily_report_charts.py +++ b/tests/modules/content_marketing/test_daily_report_charts.py @@ -4,7 +4,7 @@ from pathlib import Path from PIL import Image -from gyxx_flow.modules.content_marketing.runtime.data.tools.daily_report_charts import ( +from gyxx_flow.modules.content_marketing.data.tools.daily_report_charts import ( _split_style_groups, build_overview_cards, classify_style_action, diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_friday_relogin_parallel.py b/tests/modules/content_marketing/test_friday_relogin_parallel.py similarity index 92% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_friday_relogin_parallel.py rename to tests/modules/content_marketing/test_friday_relogin_parallel.py index f71321b..5641149 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_friday_relogin_parallel.py +++ b/tests/modules/content_marketing/test_friday_relogin_parallel.py @@ -1,5 +1,7 @@ -from gyxx_flow.modules.content_marketing.runtime.data.tools import friday_relogin_parallel as relogin -from gyxx_flow.modules.content_marketing.runtime.data.tools import relogin_transaction +from gyxx_flow.modules.content_marketing.data.tools import ( + friday_relogin_parallel as relogin, +) +from gyxx_flow.modules.content_marketing.data.tools import relogin_transaction def test_five_expected_platforms_are_configured(): @@ -10,6 +12,11 @@ def test_five_expected_platforms_are_configured(): assert relogin.PLATFORMS["douyin"].required_cookie == "sessionid" +def test_manual_relogin_can_select_only_xingtu(): + assert relogin.resolve_platform_names(["xingtu"]) == ["xingtu"] + assert relogin.resolve_platform_names(None) == list(relogin.PLATFORMS) + + def test_default_screenshot_recipients_are_routed_by_platform(): assert relogin.DEFAULT_RECIPIENTS == { "pgy": "ou_24cc944d6e43c69c59d6560ad4e2ae6e", diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_global_creator_scraping.py b/tests/modules/content_marketing/test_global_creator_scraping.py similarity index 72% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_global_creator_scraping.py rename to tests/modules/content_marketing/test_global_creator_scraping.py index 7754c3e..cb56b98 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_global_creator_scraping.py +++ b/tests/modules/content_marketing/test_global_creator_scraping.py @@ -3,10 +3,20 @@ from pathlib import Path import pytest -from gyxx_flow.modules.content_marketing.runtime.creator_task_grouping import chunk_creator_groups, group_tasks_by_creator +from gyxx_flow.catalog import WorkflowCatalog +from gyxx_flow.modules.content_marketing import pgy_xhs_scraper_v2 as pgy +from gyxx_flow.modules.content_marketing import xingtu_scraper_v2 as xingtu +from gyxx_flow.modules.content_marketing.creator_task_grouping import ( + chunk_creator_groups, + group_tasks_by_creator, +) -import pgy_xhs_scraper_v2 as pgy -import xingtu_scraper_v2 as xingtu +PROJECT_ROOT = Path(__file__).resolve().parents[3] + + +def _workflow(workflow_id): + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + return next(item for item in catalog.workflows if item.workflow_id == workflow_id) def test_group_tasks_uses_creator_id_and_merges_missing_id_by_unique_name(): @@ -270,36 +280,88 @@ def test_xingtu_new_tab_creation_failure_rebuilds_browser_session(message): assert xingtu.is_browser_session_lost(RuntimeError(message)) is True -def test_daily_retry_does_not_search_xingtu_again_same_day(): - batch = Path("data/tools/daily_run.bat").read_text(encoding="utf-8") +def test_scheduled_collection_has_no_automatic_retry_and_syncs_once(): + steps = _workflow("content.metrics.daily").steps + entries = [step.entry for step in steps] - retry_lines = [line for line in batch.splitlines() if "retry_failed.py" in line and "call" in line] + assert "data/tools/retry_failed.py" not in entries + assert entries.count("data/tools/sync_metrics_to_cmt_notes.py") == 1 + assert entries.index("run_all.py") < entries.index( + "data/tools/sync_metrics_to_cmt_notes.py" + ) - assert len(retry_lines) == 1 - assert "--platform bili,pgy" in retry_lines[0] + +def test_daily_collection_uses_dedicated_self_operated_pipeline(): + steps = _workflow("content.metrics.daily").steps + entries = [step.entry for step in steps] + run_step = next(step for step in steps if step.entry == "run_all.py") + + assert "--daily-scope" in run_step.args + assert "--include-self-operated" not in run_step.args + assert "data/tools/refresh_self_mapping.py" in entries + assert "self_bilibili_scraper.py" in entries + assert "chanmama_scraper.py" in entries + refresh_index = entries.index("data/tools/refresh_self_mapping.py") + bili_index = entries.index("self_bilibili_scraper.py") + douyin_index = entries.index("chanmama_scraper.py") + assert refresh_index < bili_index < douyin_index @pytest.mark.parametrize( - "batch_path", + ("module", "entry_url", "error_type", "operation"), [ - Path("data/tools/daily_run.bat"), - Path("data/tools/daily_run_with_backfill.bat"), + (pgy, pgy.HOME_URL, pgy.AcceptanceCookieSkip, "content.pugongying.login"), + ( + xingtu, + xingtu.MARKET_URL, + xingtu.AcceptanceCookieSkip, + "content.xingtu.login", + ), ], ) -def test_scheduled_collection_syncs_pg_before_and_after_retry(batch_path): - batch = batch_path.read_text(encoding="utf-8") - calls = [ - line.strip().lower() - for line in batch.splitlines() - if line.strip().lower().startswith("call %python%") - ] +def test_acceptance_invalid_session_skips_before_qr_login( + module, + entry_url, + error_type, + operation, + monkeypatch, + tmp_path, +): + evidence_file = tmp_path / "evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_FEISHU_TABLE_WRITE_DISABLED", "1") + monkeypatch.setenv("GYXX_COOKIE_INVALID_SKIP", "1") + monkeypatch.setenv( + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", + "ou_8ee224968aa26a74c7d30ba27fed5eeb", + ) + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence_file)) + monkeypatch.setattr(module, "is_logged_in", lambda _page: False) + monkeypatch.setattr( + module, + "wait_for_scan_login", + lambda *_args, **_kwargs: pytest.fail("QR login must remain disabled"), + ) - run_index = next(i for i, line in enumerate(calls) if "run_all.py" in line) - retry_index = next(i for i, line in enumerate(calls) if "retry_failed.py" in line) - sync_indices = [ - i for i, line in enumerate(calls) - if "sync_metrics_to_cmt_notes.py" in line - ] + class Page: + def __init__(self): + self.urls = [] - assert len(sync_indices) == 2 - assert run_index < sync_indices[0] < retry_index < sync_indices[1] + def goto(self, url, **_kwargs): + self.urls.append(url) + + def wait_for_timeout(self, _milliseconds): + pass + + page = Page() + with pytest.raises(error_type): + module.ensure_login(page, login_timeout=30) + + assert page.urls == [entry_url] + records = [ + json.loads(line) + for line in evidence_file.read_text(encoding="utf-8").splitlines() + ] + assert records[-1]["event"] == "cookie_skipped" + assert records[-1]["operation"] == operation + assert records[-1]["details"]["status"] == "SKIPPED_COOKIE" diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_relogin_rollback.py b/tests/modules/content_marketing/test_relogin_rollback.py similarity index 66% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_relogin_rollback.py rename to tests/modules/content_marketing/test_relogin_rollback.py index cde40aa..7454e6e 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_relogin_rollback.py +++ b/tests/modules/content_marketing/test_relogin_rollback.py @@ -1,10 +1,15 @@ -from pathlib import Path import pytest -from gyxx_flow.modules.content_marketing.runtime.data.tools import relogin_douyin, relogin_pgy, relogin_xingtu -import xingtu_scraper_v2 -import pgy_xhs_scraper_v2 +from gyxx_flow.modules.content_marketing import ( + pgy_xhs_scraper_v2, + xingtu_scraper_v2, +) +from gyxx_flow.modules.content_marketing.data.tools import ( + relogin_douyin, + relogin_pgy, + relogin_xingtu, +) @pytest.mark.parametrize("module", [relogin_pgy, relogin_xingtu]) @@ -122,6 +127,50 @@ def test_xingtu_index_page_is_recognized_as_logged_in(): assert xingtu_scraper_v2.is_logged_in(Page()) is True +def test_xingtu_public_homepage_is_not_logged_in_even_with_stale_cookie(): + class Context: + def cookies(self): + return [{"name": "sessionid", "value": "stale"}] + + class Page: + url = "https://www.xingtu.cn/?redirect_uri=/ad/creator/market" + context = Context() + + def get_by_text(self, *_args, **_kwargs): + raise AssertionError("public homepage must be rejected before marker checks") + + assert xingtu_scraper_v2.is_logged_in(Page()) is False + + +def test_xingtu_market_page_without_business_ui_rejects_stale_cookie(): + class Locator: + @property + def first(self): + return self + + def wait_for(self, **_kwargs): + raise xingtu_scraper_v2.PlaywrightTimeoutError("missing") + + def count(self): + return 0 + + class Context: + def cookies(self): + return [{"name": "sessionid", "value": "stale"}] + + class Page: + url = "https://www.xingtu.cn/ad/creator/market" + context = Context() + + def get_by_text(self, *_args, **_kwargs): + return Locator() + + def locator(self, *_args, **_kwargs): + return Locator() + + assert xingtu_scraper_v2.is_logged_in(Page()) is False + + def test_xingtu_cookie_validation_requires_real_session_cookie(tmp_path): cookie = tmp_path / "xingtu.json" cookie.write_text('[{"name":"other","value":"1"}]', encoding="utf-8") @@ -155,6 +204,84 @@ def test_xingtu_login_only_does_not_report_success_when_page_action_was_swallowe assert xingtu_scraper_v2.login_only(1) == 1 +def test_xingtu_login_only_verifies_persisted_profile_after_oauth_context_closes( + tmp_path, monkeypatch +): + cookie = tmp_path / "xingtu.json" + + class Session: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def fetch(self, *_args, **_kwargs): + raise RuntimeError("Target page, context or browser has been closed") + + monkeypatch.setattr(xingtu_scraper_v2, "COOKIE_FILE", cookie) + monkeypatch.setattr(xingtu_scraper_v2, "DynamicSession", lambda **_kwargs: Session()) + monkeypatch.setattr(xingtu_scraper_v2, "_force_cleanup_session", lambda _session: None) + + def verify(): + cookie.write_text('[{"name":"sessionid","value":"fresh"}]', encoding="utf-8") + return True + + monkeypatch.setattr(xingtu_scraper_v2, "verify_persisted_profile_login", verify) + + assert xingtu_scraper_v2.login_only(1) == 0 + + +def test_xingtu_scan_login_follows_replacement_page_after_oauth_tab_closes(monkeypatch): + class Context: + def __init__(self): + self.pages = [] + + def new_page(self): + raise AssertionError("a surviving replacement page should be reused") + + class Page: + def __init__(self, context, url, *, closes_on_wait=False): + self.context = context + self.url = url + self.closed = False + self.closes_on_wait = closes_on_wait + + def is_closed(self): + return self.closed + + def wait_for_timeout(self, _milliseconds): + if self.closes_on_wait: + self.closed = True + + def goto(self, url, **_kwargs): + self.url = url + + context = Context() + qr_page = Page( + context, + "https://open.douyin.com/platform/oauth/pc/auth", + closes_on_wait=True, + ) + market_page = Page(context, xingtu_scraper_v2.MARKET_URL) + context.pages = [qr_page, market_page] + saved = [] + + monkeypatch.setattr(xingtu_scraper_v2, "open_scan_login", lambda _page: None) + monkeypatch.setattr( + xingtu_scraper_v2, "is_scan_qr_page", lambda page: page is qr_page + ) + monkeypatch.setattr( + xingtu_scraper_v2, "is_logged_in", lambda page: page is market_page + ) + monkeypatch.setattr(xingtu_scraper_v2, "save_state", lambda page: saved.append(page)) + + result = xingtu_scraper_v2.wait_for_scan_login(qr_page, 2) + + assert result is market_page + assert saved == [market_page] + + def test_xingtu_interrupted_transaction_is_recovered(tmp_path, monkeypatch): cookie = tmp_path / "xingtu_cookies.json" profile = tmp_path / "profile" diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_remote_database_config.py b/tests/modules/content_marketing/test_remote_database_config.py similarity index 52% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_remote_database_config.py rename to tests/modules/content_marketing/test_remote_database_config.py index 8a1c9b8..6892331 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_remote_database_config.py +++ b/tests/modules/content_marketing/test_remote_database_config.py @@ -1,32 +1,38 @@ import io -import os import unittest from pathlib import Path -from unittest.mock import patch -from gyxx_flow.modules.content_marketing.runtime.data.tools import db -from gyxx_flow.modules.content_marketing.runtime.data.tools import sync_metrics_to_cmt_notes as sync_metrics +from gyxx_flow.adapters import RuntimeServicePolicy +from gyxx_flow.modules.content_marketing.data.tools import db +from gyxx_flow.modules.content_marketing.data.tools import ( + sync_metrics_to_cmt_notes as sync_metrics, +) + +ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "content_marketing" +) -ROOT = Path(__file__).resolve().parents[1] - - -class RemoteDatabaseConfigTests(unittest.TestCase): - def test_config_requires_every_connection_value_without_local_fallback(self): +class LocalDatabaseConfigTests(unittest.TestCase): + def test_config_requires_every_final_connection_value(self): complete = { - "PG_HOST": "db.example.com", + "PG_HOST": "127.0.0.1", "PG_PORT": "5432", - "PG_DB": "analytics", - "PG_USER": "reporter", + "PG_DB": "gyxx_super_data", + "PG_USER": "gyxx_flow", "PG_PASSWORD": "secret", } self.assertEqual( db.database_config_from_env(complete), { - "host": "db.example.com", + "host": "127.0.0.1", "port": 5432, - "dbname": "analytics", - "user": "reporter", + "dbname": "gyxx_super_data", + "user": "gyxx_flow", "password": "secret", }, ) @@ -35,14 +41,22 @@ class RemoteDatabaseConfigTests(unittest.TestCase): with self.subTest(missing=missing), self.assertRaisesRegex(RuntimeError, missing): db.database_config_from_env(values) - def test_runtime_config_points_to_remote_data_hub(self): - config = db.get_db_config() - self.assertEqual(config["host"], "8.148.185.119") + def test_runtime_policy_uses_explicit_cloud_postgres(self): + environment = RuntimeServicePolicy().apply( + { + "PG_HOST": "db.example.com", + "PG_PORT": "5432", + "PG_DB": "data_hub", + "PG_USER": "data_hub", + "GYXX_POSTGRES_PASSWORD": "secret", + } + ) + config = db.database_config_from_env(environment) + self.assertEqual(config["host"], "db.example.com") self.assertEqual(config["dbname"], "data_hub") self.assertEqual(config["user"], "data_hub") - self.assertNotIn(config["host"], {"localhost", "127.0.0.1"}) - def test_database_scripts_do_not_define_local_connection_defaults(self): + def test_database_scripts_do_not_define_remote_connection_defaults(self): paths = ( ROOT / "data" / "tools" / "db.py", ROOT / "data" / "tools" / "sync_metrics_to_cmt_notes.py", @@ -52,10 +66,8 @@ class RemoteDatabaseConfigTests(unittest.TestCase): for path in paths: source = path.read_text(encoding="utf-8-sig") with self.subTest(path=path.name): - self.assertNotIn('"localhost"', source) - self.assertNotIn("'localhost'", source) - self.assertNotIn('"gyxx_super_data"', source) - self.assertNotIn("'gyxx_super_data'", source) + self.assertNotIn("8.148.185.119", source) + self.assertNotIn("data_hub", source) def test_sync_console_replaces_characters_unsupported_by_gbk(self): raw = io.BytesIO() diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_retry_failed_completeness.py b/tests/modules/content_marketing/test_retry_failed_completeness.py similarity index 93% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_retry_failed_completeness.py rename to tests/modules/content_marketing/test_retry_failed_completeness.py index aa75a53..c7f4e80 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_retry_failed_completeness.py +++ b/tests/modules/content_marketing/test_retry_failed_completeness.py @@ -4,8 +4,8 @@ import time from pathlib import Path from types import SimpleNamespace -import gyxx_flow.modules.content_marketing.runtime.data.tools.retry_failed as retry -import run_all +import gyxx_flow.modules.content_marketing.data.tools.retry_failed as retry +from gyxx_flow.modules.content_marketing import run_all def test_requested_records_must_be_successful_even_when_process_exits_zero(): @@ -96,7 +96,8 @@ def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_p result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False) calls = [] - def fake_call(_cmd, cwd): + def fake_call(_cmd, cwd, env=None): + del env calls.append(cwd) _write_json(result_path, { "index": 1, @@ -112,6 +113,7 @@ def test_run_one_style_rechecks_payload_and_retries_exit_zero(monkeypatch, tmp_p return 0 monkeypatch.setattr(retry.subprocess, "call", fake_call) + monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {}) monkeypatch.setattr(retry.time, "sleep", lambda _seconds: None) ok = retry.run_one_style( @@ -127,7 +129,8 @@ def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(mo style = {"index": 1, "name": "款式A"} result_path = tmp_path / retry.canonical_result_filename(style, "pgy", False) - def fake_call(_cmd, cwd): + def fake_call(_cmd, cwd, env=None): + del env _write_json(result_path, { "index": 1, "total": 1, @@ -143,6 +146,7 @@ def test_run_one_style_accepts_exit_zero_only_after_requested_record_succeeds(mo return 0 monkeypatch.setattr(retry.subprocess, "call", fake_call) + monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {}) assert retry.run_one_style( "pgy", 1, max_attempts=1, record_ids=["ok"], @@ -155,7 +159,8 @@ def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_pa result_path = tmp_path / retry.canonical_result_filename(style, "xt", True) observed = {} - def fake_call(cmd, cwd): + def fake_call(cmd, cwd, env=None): + del env observed["cmd"] = cmd _write_json(result_path, { "index": 1, @@ -165,6 +170,7 @@ def test_self_operated_xingtu_retry_uses_self_douyin_scraper(monkeypatch, tmp_pa return 0 monkeypatch.setattr(retry.subprocess, "call", fake_call) + monkeypatch.setattr(retry, "environment_for_child_script", lambda *_a, **_k: {}) assert retry.run_one_style( "xt", 1, max_attempts=1, self_operated=True, @@ -383,3 +389,10 @@ def test_run_round_marks_exit_zero_platform_failed_when_summary_is_invalid(monke assert result["_non_zero_platforms"] == ["pgy"] assert observed["expected_indices"] == {1} assert isinstance(observed["started_at"], float) + + +def test_run_all_preserves_cookie_skip_exit_code(): + assert run_all.aggregate_exit_code(True, [75, 75]) == 75 + assert run_all.aggregate_exit_code(True, [75, 1]) == 1 + assert run_all.aggregate_exit_code(True, []) == 1 + assert run_all.aggregate_exit_code(False, []) == 0 diff --git a/tests/modules/content_marketing/test_self_operated_pipeline.py b/tests/modules/content_marketing/test_self_operated_pipeline.py new file mode 100644 index 0000000..f0e0f16 --- /dev/null +++ b/tests/modules/content_marketing/test_self_operated_pipeline.py @@ -0,0 +1,10 @@ +from gyxx_flow.modules.content_marketing import self_douyin_scraper + + +def test_self_douyin_accepts_feishu_select_and_multiselect_values(): + assert self_douyin_scraper.is_douyin_platform("抖音") is True + assert self_douyin_scraper.is_douyin_platform(["抖音"]) is True + assert self_douyin_scraper.is_douyin_platform("Douyin") is True + assert self_douyin_scraper.is_douyin_platform(["小红书", "抖音"]) is True + assert self_douyin_scraper.is_douyin_platform("小红书") is False + assert self_douyin_scraper.is_douyin_platform(None) is False diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_sync_style_categories.py b/tests/modules/content_marketing/test_sync_style_categories.py similarity index 93% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_sync_style_categories.py rename to tests/modules/content_marketing/test_sync_style_categories.py index 94b27a4..c830ed2 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_sync_style_categories.py +++ b/tests/modules/content_marketing/test_sync_style_categories.py @@ -1,6 +1,9 @@ import unittest -from gyxx_flow.modules.content_marketing.runtime.data.tools.sync_style_categories import extract_style_mapping, normalize_style_name +from gyxx_flow.modules.content_marketing.data.tools.sync_style_categories import ( + extract_style_mapping, + normalize_style_name, +) class SyncStyleCategoriesTests(unittest.TestCase): diff --git a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_weekly_tmall_persona.py b/tests/modules/content_marketing/test_weekly_tmall_persona.py similarity index 96% rename from src/gyxx_flow/modules/content_marketing/runtime/tests/test_weekly_tmall_persona.py rename to tests/modules/content_marketing/test_weekly_tmall_persona.py index d111b35..732bba0 100644 --- a/src/gyxx_flow/modules/content_marketing/runtime/tests/test_weekly_tmall_persona.py +++ b/tests/modules/content_marketing/test_weekly_tmall_persona.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from gyxx_flow.modules.content_marketing.runtime import weekly_summary_all as weekly +from gyxx_flow.modules.content_marketing import weekly_summary_all as weekly class WeeklyTmallPersonaTests(unittest.TestCase): diff --git a/tests/modules/product_commerce/conftest.py b/tests/modules/product_commerce/conftest.py new file mode 100644 index 0000000..14aa56d --- /dev/null +++ b/tests/modules/product_commerce/conftest.py @@ -0,0 +1,18 @@ +"""Test bootstrap for the flattened product-commerce module.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +MODULE_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" +) + +# The migrated scripts intentionally retain their original sibling imports. +if str(MODULE_ROOT) not in sys.path: + sys.path.insert(0, str(MODULE_ROOT)) diff --git a/tests/modules/product_commerce/test_alert_command.py b/tests/modules/product_commerce/test_alert_command.py new file mode 100644 index 0000000..7393ffc --- /dev/null +++ b/tests/modules/product_commerce/test_alert_command.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +COMMAND_PATH = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" + / "commands" + / "run_alerts.py" +) + + +def _load_command( + monkeypatch: pytest.MonkeyPatch, + observed: dict[str, object], +) -> ModuleType: + legacy = ModuleType("run_alerts_with_retry") + + def fake_main() -> int: + observed["argv"] = tuple(sys.argv) + return 0 + + legacy.main = fake_main # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "run_alerts_with_retry", legacy) + spec = importlib.util.spec_from_file_location("test_run_alerts_command", COMMAND_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_alert_command_uses_canonical_acceptance_recipient( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, object] = {} + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01") + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False) + monkeypatch.delenv("GYXX_ALERT_RECIPIENT_OPEN_ID", raising=False) + module = _load_command(monkeypatch, observed) + + assert module.main() == 0 + assert observed["argv"] == ( + sys.argv[0], + "--end-date", + "2026-08-01", + "--openid", + WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID, + ) + + +def test_alert_command_accepts_canonical_production_recipient( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, object] = {} + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-01") + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + monkeypatch.setenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", "ou_configured") + monkeypatch.setenv("GYXX_ALERT_RECIPIENT_OPEN_ID", "ou_legacy") + module = _load_command(monkeypatch, observed) + + assert module.main() == 0 + assert observed["argv"][-1] == "ou_configured" diff --git a/tests/modules/product_commerce/test_alert_integrity.py b/tests/modules/product_commerce/test_alert_integrity.py new file mode 100644 index 0000000..f51e2a8 --- /dev/null +++ b/tests/modules/product_commerce/test_alert_integrity.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import subprocess +import sys +from datetime import date +from pathlib import Path +from types import ModuleType + +import check_nine_day_decline as decline +import pytest +import run_alerts_with_retry as retry + +PAIR = ("tm", "测试款") + + +def _segment(sales: int, *, days: int = 3) -> dict: + return { + "sales": sales, + "present_days": days, + "erp_style_codes": ["STYLE-1"], + } + + +def _event() -> dict: + return { + "style_name": "测试款", + "platform": "tm", + "erp_style_codes": ["STYLE-1"], + "window_start": "2026-07-24", + "window_end": "2026-08-01", + "seg1_sales": 300, + "seg2_sales": 210, + "seg3_sales": 90, + "drop_pct_1to2": 30.0, + "drop_pct_2to3": 57.1, + "drop_pct_1to3": 70.0, + } + + +def _prepare_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "check_nine_day_decline.py", + "--end-date", + "2026-08-01", + "--openid", + "ou_owner", + "--data-root", + str(tmp_path), + ], + ) + monkeypatch.setattr(decline, "managed_data_path", lambda value: Path(value)) + monkeypatch.setattr(decline, "append_log", lambda line: None) + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + + +def test_detect_decline_uses_three_inclusive_postgres_sum_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: list[list[tuple[date, date]]] = [] + + def fake_load(windows): + observed.append(windows) + return [ + {PAIR: _segment(300)}, + {PAIR: _segment(210)}, + {PAIR: _segment(90)}, + ] + + monkeypatch.setattr(decline, "_load_segment_snapshots", fake_load) + + events = decline.detect_decline("2026-08-01", 9) + + assert observed == [ + [ + (date(2026, 7, 24), date(2026, 7, 26)), + (date(2026, 7, 27), date(2026, 7, 29)), + (date(2026, 7, 30), date(2026, 8, 1)), + ] + ] + assert len(events) == 1 + assert events[0]["window_start"] == "2026-07-24" + assert events[0]["window_end"] == "2026-08-01" + assert [events[0][f"seg{index}_sales"] for index in range(1, 4)] == [ + 300, + 210, + 90, + ] + assert events[0]["segment_windows"] == [ + {"start": "2026-07-24", "end": "2026-07-26", "days": 3}, + {"start": "2026-07-27", "end": "2026-07-29", "days": 3}, + {"start": "2026-07-30", "end": "2026-08-01", "days": 3}, + ] + + +def test_detect_decline_rejects_incomplete_three_day_segment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + decline, + "_load_segment_snapshots", + lambda windows: [ + {PAIR: _segment(300)}, + {PAIR: _segment(210, days=2)}, + {PAIR: _segment(90)}, + ], + ) + + assert decline.detect_decline("2026-08-01", 9) == [] + + +def test_segment_loader_uses_postgres_range_sum_on_one_connection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_db = ModuleType("db") + connection = object() + observed: list[tuple[object, str, str]] = [] + + class ConnectionContext: + def __enter__(self): + return connection + + def __exit__(self, *args): + return False + + def fake_fetch(conn, start, end): + observed.append((conn, start, end)) + return {} + + fake_db.get_conn = ConnectionContext # type: ignore[attr-defined] + fake_db.fetch_daily_for_range = fake_fetch # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "db", fake_db) + windows = decline._segment_windows("2026-08-01", 9) + + assert decline._load_segment_snapshots(windows) == [{}, {}, {}] + assert observed == [ + (connection, "2026-07-24", "2026-07-26"), + (connection, "2026-07-27", "2026-07-29"), + (connection, "2026-07-30", "2026-08-01"), + ] + + +@pytest.mark.parametrize( + ("response", "expected"), + [ + ({"id": "chatcmpl-not-a-receipt"}, None), + ({"message_id": "om_direct123"}, "om_direct123"), + ({"data": {"message_id": "om_nested123"}}, "om_nested123"), + ( + {"choices": [{"message": {"content": "飞书回执 om_content123"}}]}, + "om_content123", + ), + ( + {"choices": [{"message": {"content": "发送成功，但没有回执"}}]}, + None, + ), + ], +) +def test_extract_feishu_message_id_requires_real_receipt_shape( + response: dict, + expected: str | None, +) -> None: + assert decline.extract_feishu_message_id(response) == expected + + +def test_main_fails_when_postgres_detection_read_fails( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _prepare_main(monkeypatch, tmp_path) + monkeypatch.setattr( + decline, + "detect_decline", + lambda *args: (_ for _ in ()).throw(RuntimeError("postgres unavailable")), + ) + + assert decline.main() == 1 + + +def test_main_zero_events_has_explicit_postcondition( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _prepare_main(monkeypatch, tmp_path) + monkeypatch.setattr(decline, "detect_decline", lambda *args: []) + + assert decline.main() == 0 + output = capsys.readouterr().out + assert "detection_source=postgresql events=0" in output + assert "notification=not_required persistence=not_required" in output + + +def test_main_dedup_query_failure_is_fail_closed( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _prepare_main(monkeypatch, tmp_path) + monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()]) + monkeypatch.setattr( + decline, + "_already_notified", + lambda *args: (_ for _ in ()).throw(RuntimeError("db timeout")), + ) + notify = pytest.fail + monkeypatch.setattr(decline, "notify_hermes", notify) + + assert decline.main() == 5 + + +def test_main_does_not_accept_arbitrary_non_error_hermes_text( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _prepare_main(monkeypatch, tmp_path) + monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()]) + monkeypatch.setattr(decline, "_already_notified", lambda *args: False) + monkeypatch.setattr( + decline, + "notify_hermes", + lambda *args, **kwargs: { + "id": "chatcmpl-123", + "choices": [{"message": {"content": "发送成功"}}], + }, + ) + persist = pytest.fail + monkeypatch.setattr(decline, "_persist_events", persist) + + assert decline.main() == 6 + + +def test_main_returns_nonzero_when_receipt_persistence_fails( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _prepare_main(monkeypatch, tmp_path) + monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()]) + monkeypatch.setattr(decline, "_already_notified", lambda *args: False) + monkeypatch.setattr( + decline, + "notify_hermes", + lambda *args, **kwargs: {"message_id": "om_real123"}, + ) + monkeypatch.setattr( + decline, + "_persist_events", + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("write failed")), + ) + + assert decline.main() == 4 + + +def test_main_success_requires_and_persists_feishu_receipt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _prepare_main(monkeypatch, tmp_path) + observed: dict[str, object] = {} + monkeypatch.setattr(decline, "detect_decline", lambda *args: [_event()]) + monkeypatch.setattr(decline, "_already_notified", lambda *args: False) + monkeypatch.setattr( + decline, + "notify_hermes", + lambda *args, **kwargs: {"message_id": "om_real123"}, + ) + + def fake_persist(*args, **kwargs): + observed.update(kwargs) + return 1 + + monkeypatch.setattr(decline, "_persist_events", fake_persist) + + assert decline.main() == 0 + assert observed["message_id"] == "om_real123" + assert observed["openid"] == "ou_owner" + output = capsys.readouterr().out + assert "notification=sent feishu_message_id=om_real123" in output + assert "persistence=committed history_rows=1" in output + + +class _FailedCollector: + def __init__(self, *args): + self.pages: list[tuple] = [] + + def add_page(self, *args, **kwargs): + self.pages.append((args, kwargs)) + + def write(self): + return None + + def summary_line(self): + return "failure" + + +def test_retry_wrapper_does_not_repeat_after_possible_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr( + sys, + "argv", + [ + "run_alerts_with_retry.py", + "--end-date", + "2026-08-01", + "--openid", + "ou_owner", + ], + ) + monkeypatch.setattr(retry, "FailedCollector", _FailedCollector) + monkeypatch.setattr(retry, "consolidate_failed_logs", lambda value: None) + monkeypatch.setattr(retry, "build_child_environment", lambda *args: {}) + monkeypatch.setattr( + retry.time, + "sleep", + lambda seconds: pytest.fail("non-retryable result must not sleep"), + ) + + def fake_run(command, **kwargs): + calls.append(command) + return subprocess.CompletedProcess(command, 4) + + monkeypatch.setattr(retry.subprocess, "run", fake_run) + + assert retry.main() == 4 + assert len(calls) == 1 + + +def test_retry_wrapper_can_retry_fail_closed_dedup_read( + monkeypatch: pytest.MonkeyPatch, +) -> None: + return_codes = iter((5, 0)) + sleeps: list[int] = [] + monkeypatch.setattr( + sys, + "argv", + [ + "run_alerts_with_retry.py", + "--end-date", + "2026-08-01", + "--openid", + "ou_owner", + ], + ) + monkeypatch.setattr(retry, "FailedCollector", _FailedCollector) + monkeypatch.setattr(retry, "consolidate_failed_logs", lambda value: None) + monkeypatch.setattr(retry, "build_child_environment", lambda *args: {}) + monkeypatch.setattr(retry.time, "sleep", sleeps.append) + monkeypatch.setattr( + retry.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, + next(return_codes), + ), + ) + + assert retry.main() == 0 + assert sleeps == [retry.RUN_SLEEP_SECONDS] diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_backfill_poseidon_sales.py b/tests/modules/product_commerce/test_backfill_poseidon_sales.py similarity index 99% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_backfill_poseidon_sales.py rename to tests/modules/product_commerce/test_backfill_poseidon_sales.py index f7b2ffe..fe30119 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_backfill_poseidon_sales.py +++ b/tests/modules/product_commerce/test_backfill_poseidon_sales.py @@ -1,9 +1,7 @@ from __future__ import annotations -import pandas as pd - import backfill_poseidon_sales as backfill - +import pandas as pd def test_backfill_uses_layered_runtime_roots(): diff --git a/tests/modules/product_commerce/test_browser_profile_reuse.py b/tests/modules/product_commerce/test_browser_profile_reuse.py new file mode 100644 index 0000000..046fff8 --- /dev/null +++ b/tests/modules/product_commerce/test_browser_profile_reuse.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import inspect +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import collect_erp_yesterday_metrics as erp +import collect_jd_persona_to_bitable as jd_persona +from runtime_paths import playwright_launch_options + + +class _Chromium: + def __init__(self) -> None: + self.kwargs = None + + def launch_persistent_context(self, **kwargs): + self.kwargs = kwargs + return object() + + +def test_jd_persona_launches_the_managed_persistent_profile( + monkeypatch, + tmp_path: Path, +) -> None: + profile = tmp_path / "jd-profile" + monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile)) + chromium = _Chromium() + + context = jd_persona._launch_jd_context(chromium, headless=True) + + assert context is not None + assert chromium.kwargs["user_data_dir"] == str(profile.resolve()) + assert chromium.kwargs["headless"] is True + + +def test_jd_persona_reuses_logged_in_page_without_credentials() -> None: + calls: list[tuple[str, str]] = [] + + class Page: + url = "https://shop.jd.com/jdm/home" + + def goto(self, *_args, **_kwargs): + return None + + def wait_for_timeout(self, _milliseconds): + return None + + def locator(self, _selector): + return SimpleNamespace( + inner_text=lambda **_kwargs: "京东商家后台 商品明细", + ) + + jd_persona._ensure_jd_session( + Page(), + shop="", + password="", + step_login=lambda _page, shop, password: calls.append((shop, password)), + ) + + assert calls == [] + + +def test_erp_uses_managed_profile_when_bound_cdp_is_not_running( + monkeypatch, + tmp_path: Path, +) -> None: + profile = tmp_path / "erp-profile" + monkeypatch.setattr(erp, "_cdp_is_listening", lambda _url: False) + + cdp_url, user_data_dir = erp.resolve_browser_session( + {"cdp_url": "http://127.0.0.1:9222"}, + { + "GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22063", + "GYXX_BROWSER_PROFILE_DIR": str(profile), + }, + ) + + assert cdp_url is None + assert user_data_dir == str(profile) + + +def test_erp_attaches_only_to_the_live_managed_cdp(monkeypatch) -> None: + monkeypatch.setattr(erp, "_cdp_is_listening", lambda _url: True) + + cdp_url, user_data_dir = erp.resolve_browser_session( + {"cdp_url": "http://127.0.0.1:9222", "user_data_dir": "legacy"}, + { + "GYXX_BROWSER_CDP_URL": "http://127.0.0.1:22063", + "GYXX_BROWSER_PROFILE_DIR": "managed", + }, + ) + + assert cdp_url == "http://127.0.0.1:22063" + assert user_data_dir is None + + +def test_erp_main_fails_before_browser_when_no_style_is_collectable(monkeypatch) -> None: + args = SimpleNamespace( + config="unused.json", + target_date=date(2026, 8, 3), + mode="daily", + refresh_config=False, + input="unused-styles.json", + ) + monkeypatch.setattr(erp, "parse_args", lambda: args) + monkeypatch.setattr( + erp, + "load_config", + lambda _path: {"login": {"url": "https://erp.test", "username": "placeholder", "password": "placeholder"}}, + ) + monkeypatch.setattr( + erp, + "load_erp_styles", + lambda *_args: ([], {"source": "lark", "collectable": [], "skipped": []}), + ) + monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None) + monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None) + monkeypatch.setattr( + erp.DynamicFetcher, + "fetch", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("browser must not start") + ), + ) + + assert erp.main() == 1 + + +def test_erp_main_fails_when_collector_summary_contains_failures( + monkeypatch, + tmp_path: Path, +) -> None: + args = SimpleNamespace( + config="unused.json", + target_date=date(2026, 8, 3), + mode="daily", + refresh_config=False, + input="unused-styles.json", + data_dir=str(tmp_path), + platform="all", + no_cdp=False, + timeout_ms=1000, + limit=0, + report_timeout=1, + report_attempts=1, + include_slow_codes=False, + hold_seconds=0, + headless=True, + ) + styles = [{"style_name": "style-a", "erp_style_codes": ["erp-a"], "brand": erp.DEFAULT_BRAND}] + monkeypatch.setattr(erp, "parse_args", lambda: args) + monkeypatch.setattr( + erp, + "load_config", + lambda _path: {"login": {"url": "https://erp.test", "username": "placeholder", "password": "placeholder"}}, + ) + monkeypatch.setattr( + erp, + "load_erp_styles", + lambda *_args: (styles, {"source": "lark", "collectable": ["style-a"], "skipped": []}), + ) + monkeypatch.setattr(erp, "print_skip_summary", lambda _report: None) + monkeypatch.setattr(erp, "write_erp_skip_report", lambda *_args: None) + monkeypatch.setattr(erp, "managed_data_path", lambda _path: tmp_path) + monkeypatch.setattr(erp, "resolve_browser_session", lambda _config: (None, None)) + + def fake_login_action(**kwargs): + state = kwargs["state"] + + def action(_page): + state["ok"] = True + + return action + + monkeypatch.setattr(erp, "make_page_action", fake_login_action) + monkeypatch.setattr( + erp, + "collect_in_page", + lambda *_args, **_kwargs: { + "reports": 1, + "valid_codes": 1, + "has_failures": True, + }, + ) + monkeypatch.setattr( + erp.DynamicFetcher, + "fetch", + lambda _url, **kwargs: kwargs["page_action"](object()), + ) + + assert erp.main() == 1 + + +def test_erp_date_inputs_are_required_and_read_back() -> None: + source = inspect.getsource(erp.set_platform_and_code) + + assert "const begin = await waitFor('#order_date_begin')" in source + assert "const end = await waitFor('#order_date_end')" in source + assert "date range did not persist" in source + + +def test_playwright_channel_is_optional_and_environment_driven() -> None: + assert playwright_launch_options({}) == {} + assert playwright_launch_options({"GYXX_PLAYWRIGHT_CHANNEL": "chrome"}) == { + "channel": "chrome" + } diff --git a/tests/modules/product_commerce/test_daily_insert_integrity.py b/tests/modules/product_commerce/test_daily_insert_integrity.py new file mode 100644 index 0000000..aa874f3 --- /dev/null +++ b/tests/modules/product_commerce/test_daily_insert_integrity.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import insert_bitable_records as insert +import pytest + + +def test_record_id_parser_accepts_nested_upsert_receipt() -> None: + envelope = { + "ok": True, + "data": {"record": {"record_id_list": ["rec-confirmed"]}}, + } + + assert insert._record_id_from_envelope(envelope) == "rec-confirmed" + + +def test_lark_upsert_rejects_success_without_record_id(monkeypatch) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + monkeypatch.setattr(insert.shutil, "which", lambda _name: "lark-cli.cmd") + monkeypatch.setattr( + insert.subprocess, + "run", + lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, + stdout='{"ok": true, "data": {}}', + stderr="", + ), + ) + + with pytest.raises(RuntimeError, match="without a confirmed record_id"): + insert.run_lark_upsert("base", "table", {"field": "value"}) + + +def _daily_record(style: str, source_id: str) -> dict: + return { + "数据日期": "2026-08-03", + "平台": "京东", + "平台代码": "jd", + "款式": style, + "SourceID": source_id, + "metric": 1, + } + + +def test_any_feishu_insert_failure_makes_whole_command_fail(monkeypatch) -> None: + style_map = { + style: { + "base_token": "base", + "table_id": "table", + "enabled": True, + "field_map": {"metric": "metric", "SourceID": "SourceID"}, + } + for style in ("style-a", "style-b") + } + monkeypatch.setattr(insert, "get_bitable_style_map", lambda: style_map) + monkeypatch.setattr( + insert, + "load_records", + lambda *_args, **_kwargs: [ + _daily_record("style-a", "source-a"), + _daily_record("style-b", "source-b"), + ], + ) + monkeypatch.setattr(insert, "_find_record_id_by_source_id", lambda *_args, **_kwargs: None) + + def upsert(_base, _table, fields, **_kwargs): + if fields["SourceID"] == "source-b": + raise RuntimeError("write failed") + return "rec-a" + + monkeypatch.setattr(insert, "run_lark_upsert", upsert) + + assert insert.main(["--date", "2026-08-03"]) == 1 + + +def test_empty_feishu_export_is_not_success(monkeypatch) -> None: + monkeypatch.setattr(insert, "get_bitable_style_map", lambda: {"style-a": {}}) + monkeypatch.setattr(insert, "load_records", lambda *_args, **_kwargs: []) + + assert insert.main(["--date", "2026-08-03"]) == 1 diff --git a/tests/modules/product_commerce/test_daily_pipeline_paths.py b/tests/modules/product_commerce/test_daily_pipeline_paths.py new file mode 100644 index 0000000..2698a87 --- /dev/null +++ b/tests/modules/product_commerce/test_daily_pipeline_paths.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import aggregate_daily_final as aggregate +import export_bitable_records as export + + +def test_daily_aggregate_and_export_share_curated_summary_root() -> None: + assert export.SUMMARY_ROOT == aggregate.CURATED_DATA_ROOT + + +def test_daily_export_keeps_platform_details_in_raw_root() -> None: + assert export.PLATFORM_ROOT == export.DATA_ROOT + assert export.PLATFORM_ROOT != export.SUMMARY_ROOT diff --git a/tests/modules/product_commerce/test_daily_result_integrity.py b/tests/modules/product_commerce/test_daily_result_integrity.py new file mode 100644 index 0000000..662bcdc --- /dev/null +++ b/tests/modules/product_commerce/test_daily_result_integrity.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace + +import aggregate_daily_final as aggregate +import db as product_db +import import_product_daily as product_import +import orchestrate_daily_collection as daily +import pytest +from commands import import_daily as import_daily_command + + +class _Context: + def __init__(self, value): + self.value = value + + def __enter__(self): + return self.value + + def __exit__(self, *_args): + return False + + +class _Cursor: + def __init__(self, rows): + self.rows = rows + self.execute_args = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, *args): + self.execute_args = args + + def fetchone(self): + return self.rows[0] if self.rows else None + + def fetchall(self): + return list(self.rows) + + +class _Connection: + def __init__(self, rows): + self.cursor_instance = _Cursor(rows) + + def cursor(self): + return self.cursor_instance + + +def _record() -> dict: + return { + "platform": "dy", + "stat_date": datetime(2026, 8, 3).date(), + "product_id": "product-1", + "source_file": "report.xlsx", + } + + +def _patch_valid_import(monkeypatch, *, upserted: int = 1) -> None: + target_dir = Path("2026-08-03") + monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir]) + monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: [Path("report.xlsx")]) + monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: [_record()]) + monkeypatch.setattr(product_import, "get_conn", lambda: _Context(object())) + monkeypatch.setattr( + product_import, + "upsert_product_daily_metrics", + lambda _conn, _records: upserted, + ) + + +def test_selected_platform_collector_failure_stops_daily_aggregation(monkeypatch) -> None: + results = iter( + [ + {"name": "jd", "code": 0}, + {"name": "dy", "code": 1, "error": "collector failed"}, + ] + ) + monkeypatch.setattr(daily, "run_step_with_retry", lambda *_args, **_kwargs: next(results)) + monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1) + + with pytest.raises(RuntimeError, match="required platform collector.*dy"): + daily.run_platforms( + parallel=False, + platforms=["jd", "dy"], + target_date="2026-08-03", + ) + +def test_platform_selection_keeps_omitted_collectors_optional(monkeypatch) -> None: + calls: list[str] = [] + + def fake_run(name, *_args, **_kwargs): + calls.append(name) + return {"name": name, "code": 0} + + monkeypatch.setattr(daily, "run_step_with_retry", fake_run) + monkeypatch.setattr(daily, "verify_platform_artifacts", lambda *_args: 1) + + results = daily.run_platforms( + parallel=False, + platforms=["jd"], + target_date="2026-08-03", + ) + + assert calls == ["jd"] + assert results == [{"name": "jd", "code": 0}] + + +def test_successful_collector_exit_still_requires_fresh_artifact(monkeypatch) -> None: + monkeypatch.setattr( + daily, + "run_step_with_retry", + lambda *_args, **_kwargs: {"name": "jd", "code": 0}, + ) + monkeypatch.setattr( + daily, + "verify_platform_artifacts", + lambda *_args: (_ for _ in ()).throw(RuntimeError("missing-or-stale")), + ) + + with pytest.raises(RuntimeError, match="missing-or-stale"): + daily.run_platforms( + parallel=False, + platforms=["jd"], + target_date="2026-08-03", + ) + + +@pytest.mark.parametrize( + ("platform", "payload"), + [ + ( + "jd", + { + "采集日期": "2026-08-03", + "matched_row_count": 0, + "download_file": "jd.xlsx", + }, + ), + ( + "dy", + { + "date_range": "2026-08-03 ~ 2026-08-03", + "matched_row_count": 0, + "download_file": "dy.xlsx", + }, + ), + ( + "tm", + { + "date": "2026-08-03", + "source_file": "tm.xlsx", + "style": {"匹配商品ID数": 0}, + }, + ), + ], +) +def test_zero_matched_product_is_not_a_valid_artifact(platform, payload) -> None: + assert daily._artifact_has_valid_product(platform, payload, "2026-08-03") is False + + +def test_platform_artifact_verifier_rejects_missing_configured_style( + monkeypatch, + tmp_path, +) -> None: + class Loader: + def get_daily_styles(self): + return {"jd": {"collectable": ["style-a", "style-b"], "skipped": []}} + + monkeypatch.setattr(daily, "StyleConfigLoader", Loader) + monkeypatch.setattr(daily, "DATA_ROOT", tmp_path) + marker_ns = time.time_ns() + path = daily._platform_artifact_path("jd", "style-a", "2026-08-03") + path.parent.mkdir(parents=True) + path.write_text( + json.dumps({ + "采集日期": "2026-08-03", + "matched_row_count": 1, + "download_file": "_downloads/2026-08-03/report.xlsx", + }), + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="style-b:missing-or-stale"): + daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) + + +def test_platform_artifact_verifier_accepts_fresh_valid_product( + monkeypatch, + tmp_path, +) -> None: + class Loader: + def get_daily_styles(self): + return {"jd": {"collectable": ["style-a"], "skipped": []}} + + monkeypatch.setattr(daily, "StyleConfigLoader", Loader) + monkeypatch.setattr(daily, "DATA_ROOT", tmp_path) + marker_ns = time.time_ns() + path = daily._platform_artifact_path("jd", "style-a", "2026-08-03") + path.parent.mkdir(parents=True) + path.write_text( + json.dumps({ + "采集日期": "2026-08-03", + "matched_row_count": 2, + "download_file": "_downloads/2026-08-03/report.xlsx", + }), + encoding="utf-8", + ) + + assert daily.verify_platform_artifacts("jd", "2026-08-03", marker_ns) == 1 + + +def test_review_import_remains_explicitly_noncritical(monkeypatch) -> None: + args = SimpleNamespace( + skip_erp=True, + skip_platforms=False, + serial_platforms=False, + dry_run=False, + skip_reviews_import=False, + ) + monkeypatch.setattr(daily, "run_platforms", lambda **_kwargs: []) + calls = [] + monkeypatch.setattr( + daily, + "run_step_with_retry", + lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 1}, + ) + + daily.run_collect(args, "2026-08-03", ["jd"], failed=None) + + assert len(calls) == 1 + assert calls[0][0][0] == "import_product_reviews" + assert calls[0][1]["critical"] is False + + +def test_formal_aggregation_fails_when_postgres_write_is_not_fresh(monkeypatch) -> None: + marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc) + args = SimpleNamespace(dry_run=False) + monkeypatch.setattr(daily, "database_write_marker", lambda: marker) + monkeypatch.setattr( + daily, + "run_step_with_retry", + lambda *_args, **_kwargs: {"name": "aggregate_daily_final", "code": 0}, + ) + monkeypatch.setattr( + daily, + "verify_daily_metrics_write", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("daily aggregation produced no fresh PostgreSQL rows") + ), + ) + + with pytest.raises(RuntimeError, match="no fresh PostgreSQL rows"): + daily.run_analyze(args, "2026-08-03", failed=None) + + +def test_dry_run_aggregation_does_not_require_postgres(monkeypatch) -> None: + args = SimpleNamespace(dry_run=True) + monkeypatch.setattr( + daily, + "database_write_marker", + lambda: pytest.fail("dry-run must not connect to PostgreSQL"), + ) + monkeypatch.setattr( + daily, + "verify_daily_metrics_write", + lambda *_args: pytest.fail("dry-run must not verify PostgreSQL"), + ) + calls = [] + monkeypatch.setattr( + daily, + "run_step_with_retry", + lambda *args, **kwargs: calls.append((args, kwargs)) or {"name": args[0], "code": 0}, + ) + + daily.run_analyze(args, "2026-08-03", failed=None) + + assert calls[0][1]["dry_run"] is True + + +def test_aggregate_command_rejects_zero_style_outputs(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(aggregate, "collect_style_names", lambda *_args: []) + monkeypatch.setattr(aggregate, "managed_data_path", lambda _path: tmp_path) + + assert aggregate.main([ + "--date", + "2026-08-03", + "--data-root", + str(tmp_path), + ]) == 1 + + +def test_aggregate_command_rejects_partial_postgres_write(monkeypatch, tmp_path) -> None: + payload = { + "date": "2026-08-03", + "style_name": "style-a", + "erp": {"style_codes": ["erp-a"]}, + "jd": { + "sales": 1, + "refund_orders": 0, + "visitors": 2, + "cart_users": 1, + "source": "jd.json", + }, + "dy": {}, + "tm": {}, + } + monkeypatch.setattr(aggregate, "collect_style_names", lambda *_args: ["style-a"]) + monkeypatch.setattr(aggregate, "managed_data_path", lambda _path: tmp_path) + monkeypatch.setattr(aggregate, "build_payload", lambda *_args: payload) + monkeypatch.setattr(aggregate, "write_outputs", lambda *_args: (tmp_path / "a", tmp_path / "b")) + monkeypatch.setattr(product_db, "get_conn", lambda: _Context(object())) + monkeypatch.setattr(product_db, "upsert_daily_metrics", lambda *_args: 0) + + assert aggregate.main([ + "--date", + "2026-08-03", + "--data-root", + str(tmp_path), + ]) == 1 + + +def test_postgres_verifier_rejects_stale_or_missing_rows(monkeypatch) -> None: + connection = _Connection([]) + monkeypatch.setattr(daily, "get_conn", lambda: _Context(connection)) + marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc) + + with pytest.raises(RuntimeError, match="missing fresh PostgreSQL rows"): + daily.verify_daily_metrics_write( + "2026-08-03", + marker, + required_platforms=["jd"], + expected_pairs={("jd", "style-a")}, + ) + + assert connection.cursor_instance.execute_args[1] == ( + "2026-08-03", + marker, + ["jd"], + ) + + +def test_postgres_verifier_requires_every_expected_platform_style(monkeypatch) -> None: + connection = _Connection([("jd", "style-a"), ("dy", "style-a")]) + monkeypatch.setattr(daily, "get_conn", lambda: _Context(connection)) + marker = datetime(2026, 8, 4, 3, 0, tzinfo=timezone.utc) + + with pytest.raises(RuntimeError, match="tm/style-a"): + daily.verify_daily_metrics_write( + "2026-08-03", + marker, + required_platforms=["jd", "dy", "tm"], + expected_pairs={ + ("jd", "style-a"), + ("dy", "style-a"), + ("tm", "style-a"), + }, + ) + + +def test_import_fails_when_selected_platform_has_no_target_directory(monkeypatch, capsys) -> None: + monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: []) + + code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"]) + + assert code == 2 + assert "dy: 没有匹配所选日期范围的报表目录" in capsys.readouterr().err + + +def test_import_fails_when_target_directory_has_no_input_file(monkeypatch, capsys) -> None: + target_dir = Path("2026-08-03") + monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir]) + monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: []) + monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: []) + + code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"]) + + assert code == 2 + assert "目标目录中没有可导入的报表文件" in capsys.readouterr().err + + +def test_import_fails_when_input_has_no_valid_rows(monkeypatch, capsys) -> None: + target_dir = Path("2026-08-03") + monkeypatch.setattr(product_import, "list_date_dirs", lambda _platform: [target_dir]) + monkeypatch.setattr(product_import, "find_target_files", lambda _dir, _platform: [Path("report.xlsx")]) + monkeypatch.setattr(product_import, "parse_records_for_date", lambda _platform, _dir: []) + + code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"]) + + assert code == 2 + assert "报表中没有有效商品明细行" in capsys.readouterr().err + + +def test_import_fails_when_postgres_reports_zero_written_rows(monkeypatch, capsys) -> None: + _patch_valid_import(monkeypatch, upserted=0) + + code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"]) + + assert code == 2 + assert "PostgreSQL 未写入任何商品明细行" in capsys.readouterr().err + + +def test_valid_selected_platform_input_still_imports_successfully(monkeypatch) -> None: + _patch_valid_import(monkeypatch, upserted=1) + + code = product_import.main(["--date", "2026-08-03", "--platforms", "dy"]) + + assert code == 0 + + +def test_import_command_passes_target_date_and_propagates_failure(monkeypatch) -> None: + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-04") + invoked = [] + monkeypatch.setattr( + import_daily_command, + "import_main", + lambda argv: invoked.append(argv) or 2, + ) + + assert import_daily_command.main() == 2 + assert invoked == [["--date", "2026-08-03"]] diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_db_config.py b/tests/modules/product_commerce/test_db_config.py similarity index 93% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_db_config.py rename to tests/modules/product_commerce/test_db_config.py index a4a5eb9..44fabc7 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_db_config.py +++ b/tests/modules/product_commerce/test_db_config.py @@ -26,7 +26,7 @@ class DatabaseConfigTests(unittest.TestCase): }, ) - def test_missing_config_does_not_fall_back_to_local_database(self): + def test_missing_final_config_is_rejected(self): with self.assertRaisesRegex(RuntimeError, "PG_HOST"): build_pg_config( { diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_market_rank.py b/tests/modules/product_commerce/test_dy_market_rank.py similarity index 84% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_market_rank.py rename to tests/modules/product_commerce/test_dy_market_rank.py index 9ebcc9b..8fe5508 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_market_rank.py +++ b/tests/modules/product_commerce/test_dy_market_rank.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import patch +from unittest.mock import MagicMock, patch from collect_dy_market_rank import ( TARGET_DY_MARKET_CATEGORIES, @@ -8,12 +8,14 @@ from collect_dy_market_rank import ( build_dy_feishu_batch_xml, build_dy_feishu_intro_xml, is_excluded_product_title, + navigate_to_product_rank, parse_market_product_rank_html, parse_money_range_upper, + payment_metric_header_index, + require_payment_metric_header, ) from collect_sycm_market_rank import build_feishu_batch_xml - SAMPLE_HTML = """ @@ -90,6 +92,51 @@ class DyMarketRankTests(unittest.TestCase): self.assertEqual(parse_money_range_upper("¥2,500万-¥5,000万"), 50_000_000) self.assertEqual(parse_money_range_upper("¥750万"), 7_500_000) + def test_payment_metric_header_uses_exact_supported_aliases(self): + self.assertEqual( + payment_metric_header_index(["排名", "支付金额（元）", "成交人数"]), + 1, + ) + self.assertEqual( + payment_metric_header_index(["排名", "成交金额", "成交人数"]), + 1, + ) + self.assertIsNone( + payment_metric_header_index(["排名", "用户支付金额同比", "成交人数"]) + ) + + def test_missing_payment_metric_reports_current_headers(self): + page = MagicMock() + page.locator.return_value.all_inner_texts.return_value = [ + "排名", + "商品", + "成交人数", + ] + + with self.assertRaisesRegex( + RuntimeError, + "允许的精确候选.*当前表头.*成交人数", + ): + require_payment_metric_header(page) + + def test_login_probe_can_open_rank_page_without_waiting_for_metrics(self): + page = MagicMock() + + with patch("collect_dy_market_rank.wait_for_product_rank_ready") as ready: + navigate_to_product_rank(page, wait_until_ready=False) + + page.goto.assert_called_once() + ready.assert_not_called() + + def test_parser_accepts_supported_payment_metric_alias(self): + rows = parse_market_product_rank_html( + SAMPLE_HTML.replace("用户支付金额", "支付金额（元）"), + category_name="旅行箱", + category_path="鞋靴箱包 > 箱包 > 功能箱包 > 旅行箱/拉杆箱", + ) + + self.assertEqual(len(rows), 2) + def test_parse_rank_html_extracts_requested_fields(self): rows = parse_market_product_rank_html( SAMPLE_HTML, diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_session_reuse.py b/tests/modules/product_commerce/test_dy_session_reuse.py similarity index 93% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_session_reuse.py rename to tests/modules/product_commerce/test_dy_session_reuse.py index 61eea50..cc3cfbf 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_dy_session_reuse.py +++ b/tests/modules/product_commerce/test_dy_session_reuse.py @@ -2,8 +2,15 @@ import json import sys from pathlib import Path - -VENDOR_ROOT = Path(__file__).resolve().parents[1] / "vendors" / "dy-data-flow" +VENDOR_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" + / "vendors" + / "dy-data-flow" +) sys.path.insert(0, str(VENDOR_ROOT)) from dy_store_competitor_store_scraping import ( # noqa: E402 diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_metric_overrides.py b/tests/modules/product_commerce/test_erp_metric_overrides.py similarity index 100% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_metric_overrides.py rename to tests/modules/product_commerce/test_erp_metric_overrides.py diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_no_data_freshness.py b/tests/modules/product_commerce/test_erp_no_data_freshness.py similarity index 100% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_no_data_freshness.py rename to tests/modules/product_commerce/test_erp_no_data_freshness.py diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_slow_skip_codes.py b/tests/modules/product_commerce/test_erp_slow_skip_codes.py similarity index 88% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_slow_skip_codes.py rename to tests/modules/product_commerce/test_erp_slow_skip_codes.py index 4077339..3266a40 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_erp_slow_skip_codes.py +++ b/tests/modules/product_commerce/test_erp_slow_skip_codes.py @@ -2,8 +2,13 @@ import ast import unittest from pathlib import Path - -PROJECT_ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" +) ERP_COLLECTOR = PROJECT_ROOT / "collect_erp_yesterday_metrics.py" diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_feishu_doc_native.py b/tests/modules/product_commerce/test_feishu_doc_native.py similarity index 99% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_feishu_doc_native.py rename to tests/modules/product_commerce/test_feishu_doc_native.py index f47f896..83699fe 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_feishu_doc_native.py +++ b/tests/modules/product_commerce/test_feishu_doc_native.py @@ -1,5 +1,4 @@ import pytest - from feishu_doc_native import ( FeishuNativeDocClient, FeishuNativeDocError, diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_guanghe_metadata.py b/tests/modules/product_commerce/test_guanghe_metadata.py similarity index 100% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_guanghe_metadata.py rename to tests/modules/product_commerce/test_guanghe_metadata.py diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_guanghe_store_routing.py b/tests/modules/product_commerce/test_guanghe_store_routing.py similarity index 100% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_guanghe_store_routing.py rename to tests/modules/product_commerce/test_guanghe_store_routing.py diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_import_product_daily.py b/tests/modules/product_commerce/test_import_product_daily.py similarity index 99% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_import_product_daily.py rename to tests/modules/product_commerce/test_import_product_daily.py index a9628c1..eb15c41 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_import_product_daily.py +++ b/tests/modules/product_commerce/test_import_product_daily.py @@ -5,7 +5,6 @@ from unittest.mock import patch import import_product_daily - TM_31_HEADERS = [ "统计日期", "商品ID", "商品名称", "货号", "商品状态", "商品访客数", "商品浏览量", "平均停留时长", "商品详情页跳出率", "商品收藏人数", diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_enter_shangzhi.py b/tests/modules/product_commerce/test_jd_enter_shangzhi.py similarity index 98% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_enter_shangzhi.py rename to tests/modules/product_commerce/test_jd_enter_shangzhi.py index 44368cf..3eda327 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_enter_shangzhi.py +++ b/tests/modules/product_commerce/test_jd_enter_shangzhi.py @@ -4,8 +4,13 @@ import unittest from pathlib import Path from unittest.mock import patch - -PROJECT_ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" +) JD_VENDOR_DIR = PROJECT_ROOT / "vendors" / "jd-data-flow" MODULE_PATH = JD_VENDOR_DIR / "jd_data_collector.py" diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_market_rank.py b/tests/modules/product_commerce/test_jd_market_rank.py similarity index 75% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_market_rank.py rename to tests/modules/product_commerce/test_jd_market_rank.py index d3eb7d1..8bd825c 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_jd_market_rank.py +++ b/tests/modules/product_commerce/test_jd_market_rank.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import MagicMock, patch from collect_jd_market_rank import ( NEXT_PAGE_SELECTORS, @@ -6,15 +7,17 @@ from collect_jd_market_rank import ( build_jd_feishu_batch_xml, build_jd_feishu_category_xml, build_jd_feishu_intro_xml, - high_resolution_jd_image_url, + category_selection_matches, + exact_category_label_pattern, group_products_by_category, + high_resolution_jd_image_url, inspect_feishu_table_images, parse_jd_market_rank_html, parse_money_range_upper, resize_feishu_table_xml, + select_category_paths, ) - SAMPLE_HTML = """
@@ -77,11 +80,56 @@ class JdMarketRankTests(unittest.TestCase): set(specs), {"旅行箱", "运动包", "双肩包", "斜挎包", "胸包", "手机包", "电脑包"}, ) - self.assertEqual(specs["旅行箱"]["path"], ("功能箱包", "行李箱")) - self.assertEqual(specs["运动包"]["path"], ("功能箱包", "休闲运动包")) - self.assertEqual(specs["双肩包"]["path"], ("男包", "男士双肩包")) - self.assertEqual(specs["手机包"]["path"], ("男包", "男士手机包")) - self.assertEqual(specs["电脑包"]["path"], ("功能箱包", "电脑包")) + self.assertEqual(specs["旅行箱"]["paths"], (("功能箱包", "行李箱"),)) + self.assertEqual(specs["运动包"]["paths"], (("功能箱包", "休闲运动包"),)) + self.assertEqual( + specs["双肩包"]["paths"], + (("男包", "双肩包"), ("男包", "男士双肩包")), + ) + self.assertEqual(specs["手机包"]["paths"], (("男包", "男士手机包"),)) + self.assertEqual(specs["电脑包"]["paths"], (("功能箱包", "电脑包"),)) + + def test_category_label_matching_is_exact_not_contains(self): + pattern = exact_category_label_pattern("双肩包") + + self.assertIsNotNone(pattern.fullmatch(" 双肩包 ")) + self.assertIsNone(pattern.fullmatch("男士双肩包")) + self.assertIsNone(pattern.fullmatch("双肩包配件")) + + def test_selected_category_verification_accepts_only_exact_display_forms(self): + path = ("男包", "双肩包") + + self.assertTrue(category_selection_matches(("双肩包",), path)) + self.assertTrue(category_selection_matches(("男包 > 双肩包",), path)) + self.assertFalse(category_selection_matches(("男士双肩包",), path)) + self.assertFalse(category_selection_matches(("双肩包配件",), path)) + + def test_category_candidates_fall_back_only_after_explicit_failure(self): + page = MagicMock() + paths = (("男包", "双肩包"), ("男包", "男士双肩包")) + with patch( + "collect_jd_market_rank.select_category_path", + side_effect=[RuntimeError("当前页面无此精确类目"), paths[1]], + ) as select_one: + selected = select_category_paths(page, paths) + + self.assertEqual(selected, paths[1]) + self.assertEqual( + [call.args[1] for call in select_one.call_args_list], + list(paths), + ) + + def test_all_missing_category_candidates_fail_with_diagnostics(self): + page = MagicMock() + paths = (("男包", "双肩包"), ("男包", "男士双肩包")) + with ( + patch( + "collect_jd_market_rank.select_category_path", + side_effect=[RuntimeError("无双肩包"), RuntimeError("无男士双肩包")], + ), + self.assertRaisesRegex(RuntimeError, "候选类目路径.*无双肩包.*无男士双肩包"), + ): + select_category_paths(page, paths) def test_parse_money_range_upper(self): self.assertEqual(parse_money_range_upper("¥50万~¥75万"), 750_000) diff --git a/tests/modules/product_commerce/test_main_image_concurrency.py b/tests/modules/product_commerce/test_main_image_concurrency.py new file mode 100644 index 0000000..060da26 --- /dev/null +++ b/tests/modules/product_commerce/test_main_image_concurrency.py @@ -0,0 +1,449 @@ +import io +import json +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import db as product_db +import run_weekly_jd_main_image as jd_runner +import run_weekly_main_image as tmall_runner +from main_image_db import persist_main_image_records, validate_main_image_schema +from main_image_paths import collect_images_dir, resolve_images_dir +from scripts import insert_jd_main_image_records as jd_insert +from scripts import insert_main_image_records as tmall_insert +from taobao_wanxiang_ai_creative_report import cleanup_images_after_visual_merge + + +class MainImagePathTests(unittest.TestCase): + def test_collectors_use_platform_owned_directories(self): + style_dir = Path("style") + self.assertEqual(collect_images_dir(style_dir, "tm"), style_dir / "tm_images") + self.assertEqual(collect_images_dir(style_dir, "jd"), style_dir / "jd_images") + + def test_insert_prefers_platform_directory_and_supports_legacy_data(self): + with tempfile.TemporaryDirectory() as tmp: + style_dir = Path(tmp) + legacy = style_dir / "images" + legacy.mkdir() + self.assertEqual(resolve_images_dir(style_dir, "jd"), legacy) + + current = style_dir / "jd_images" + current.mkdir() + self.assertEqual(resolve_images_dir(style_dir, "jd"), current) + + +class MainImageWorkflowPreflightTests(unittest.TestCase): + def test_zero_valid_styles_is_a_formal_failure(self): + for runner in (tmall_runner, jd_runner): + with self.subTest(runner=runner.__name__): + with self.assertRaisesRegex(RuntimeError, "拒绝按成功退出"): + runner.require_valid_output(0, "2026-08-04") + + def test_dynamic_style_read_failure_is_not_silently_ignored(self): + with patch.object( + tmall_runner, + "get_main_image_styles", + side_effect=RuntimeError("Feishu unavailable"), + ): + with self.assertRaisesRegex(RuntimeError, "款式读取失败"): + tmall_runner.load_dynamic_styles() + + with patch.object( + jd_runner, + "get_jd_spu_groups", + side_effect=RuntimeError("Feishu unavailable"), + ): + with self.assertRaisesRegex(RuntimeError, "款式读取失败"): + jd_runner.load_dynamic_styles() + + def test_empty_dynamic_style_set_is_a_formal_failure(self): + with patch.object(tmall_runner, "get_main_image_styles", return_value=[]): + with self.assertRaisesRegex(RuntimeError, "未返回任何天猫"): + tmall_runner.load_dynamic_styles() + + with patch.object(jd_runner, "get_jd_spu_groups", return_value=[]): + with self.assertRaisesRegex(RuntimeError, "未返回任何京东"): + jd_runner.load_dynamic_styles() + + +class TmallCleanupTests(unittest.TestCase): + def test_locked_duplicate_is_retried_without_console_encoding_failure(self): + with tempfile.TemporaryDirectory() as tmp: + images_dir = Path(tmp) + keep = images_dir / "keep.png" + locked = images_dir / "locked.png" + keep.write_bytes(b"keep") + locked.write_bytes(b"duplicate") + creatives = [{"local_image_path": str(keep), "local_image_paths": [str(keep), str(locked)]}] + + original_unlink = Path.unlink + attempts = 0 + + def flaky_unlink(path, *args, **kwargs): + nonlocal attempts + if path == locked: + attempts += 1 + if attempts < 3: + raise PermissionError("temporarily locked") + return original_unlink(path, *args, **kwargs) + + raw = io.BytesIO() + strict_gbk_stdout = io.TextIOWrapper(raw, encoding="gbk", errors="strict") + with patch.object(Path, "unlink", new=flaky_unlink): + with redirect_stdout(strict_gbk_stdout): + cleanup_images_after_visual_merge( + images_dir, + creatives, + retry_attempts=3, + retry_delay=0, + ) + strict_gbk_stdout.flush() + + self.assertEqual(attempts, 3) + self.assertFalse(locked.exists()) + self.assertTrue(keep.exists()) + + +class MainImageDatabaseTests(unittest.TestCase): + @staticmethod + def _schema_connection(columns): + class Cursor: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, _sql): + return None + + def fetchone(self): + return (list(columns),) + + class Connection: + @staticmethod + def cursor(): + return Cursor() + + class ConnectionContext: + def __enter__(self): + return Connection() + + def __exit__(self, exc_type, exc, tb): + return False + + return ConnectionContext + + def test_schema_preflight_accepts_platform_aware_primary_key(self): + columns = ("collect_date", "style_name", "platform", "image_key") + self.assertEqual( + validate_main_image_schema(get_conn_fn=self._schema_connection(columns)), + columns, + ) + + def test_schema_preflight_rejects_legacy_primary_key_before_external_writes(self): + legacy = ("collect_date", "style_name", "image_key") + with self.assertRaisesRegex(RuntimeError, "primary key mismatch"): + validate_main_image_schema(get_conn_fn=self._schema_connection(legacy)) + + def test_empty_batch_does_not_open_database_connection(self): + def unexpected_connection(): + raise AssertionError("database should not be opened") + + self.assertEqual( + persist_main_image_records([], get_conn_fn=unexpected_connection), + 0, + ) + + def test_database_connection_is_opened_only_when_records_are_persisted(self): + events = [] + connection = object() + + class ConnectionContext: + def __enter__(self): + events.append("connect") + return connection + + def __exit__(self, exc_type, exc, tb): + events.append("close") + + def upsert(conn, records): + self.assertIs(conn, connection) + events.append(("upsert", len(records))) + return len(records) + + self.assertEqual( + persist_main_image_records( + [{"image_key": "a"}, {"image_key": "b"}], + get_conn_fn=ConnectionContext, + upsert_fn=upsert, + ), + 2, + ) + self.assertEqual(events, ["connect", ("upsert", 2), "close"]) + + def test_upsert_identity_contains_platform_and_preserves_both_platform_rows(self): + class CursorContext: + def __enter__(self): + return object() + + def __exit__(self, exc_type, exc, tb): + return False + + class Connection: + @staticmethod + def cursor(): + return CursorContext() + + records = [ + { + "collect_date": "2026-08-02", + "style_name": "同款", + "platform": platform, + "image_key": "main.jpg", + } + for platform in ("jd", "tm") + ] + with patch.object(product_db, "execute_values") as execute_values: + count = product_db.upsert_main_image_creatives(Connection(), records) + + self.assertEqual(count, 2) + sql = execute_values.call_args.args[1] + rows = execute_values.call_args.args[2] + self.assertIn( + "ON CONFLICT (collect_date, style_name, platform, image_key)", + sql, + ) + self.assertEqual([row[2] for row in rows], ["jd", "tm"]) + + def test_schema_migrates_existing_primary_key_to_platform_identity(self): + schema = (Path(product_db.__file__).with_name("schema.sql")).read_text( + encoding="utf-8" + ) + + expected = "PRIMARY KEY (collect_date, style_name, platform, image_key)" + self.assertGreaterEqual(schema.count(expected), 2) + self.assertIn("current_pk_columns IS DISTINCT FROM", schema) + self.assertIn("DROP CONSTRAINT", schema) + + +class MainImageSinkIsolationTests(unittest.TestCase): + @staticmethod + def _write_source( + root: Path, + *, + platform: str, + date_str: str = "2026-08-02", + style: str = "同款", + ) -> tuple[Path, Path]: + style_dir = root / date_str / style + images_dir = style_dir / f"{platform}_images" + images_dir.mkdir(parents=True) + image_path = images_dir / "main.jpg" + image_path.write_bytes(b"image") + if platform == "tm": + payload = { + "creatives": [ + { + "image_key": image_path.name, + "impressions": 10, + "clicks": 2, + } + ] + } + filename = "wanxiang_creative_main_images.json" + else: + payload = { + "records": [ + { + "image_key": image_path.name, + "impressions": 20, + "clicks": 3, + } + ] + } + filename = "jd_main_images.json" + (style_dir / filename).write_text( + json.dumps(payload, ensure_ascii=False), encoding="utf-8" + ) + return style_dir, image_path + + def test_tmall_pg_payload_survives_feishu_discovery_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._write_source(root, platform="tm") + pg_records = [] + with ( + patch.object( + tmall_insert, + "artifact_relative_path", + return_value="raw/product_commerce/main.jpg", + ), + patch.object( + tmall_insert, + "discover_fields", + side_effect=RuntimeError("Feishu unavailable"), + ), + ): + result = tmall_insert.process_product( + "同款", + {"base_token": "base", "table_id": "table"}, + "2026-08-02", + root, + False, + pg_records, + ) + + self.assertTrue(result["errors"]) + self.assertEqual(len(pg_records), 1) + self.assertEqual(pg_records[0]["platform"], "tm") + self.assertFalse(pg_records[0]["uploaded_to_bitable"]) + + def test_jd_pg_payload_survives_feishu_discovery_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._write_source(root, platform="jd") + pg_records = [] + with ( + patch.object( + jd_insert, + "artifact_relative_path", + return_value="raw/product_commerce/main.jpg", + ), + patch.object( + jd_insert, + "discover_fields", + side_effect=RuntimeError("Feishu unavailable"), + ), + ): + result = jd_insert.process_style( + "同款", + {"base_token": "base", "table_id": "table"}, + "2026-08-02", + root, + False, + pg_records, + ) + + self.assertTrue(result["errors"]) + self.assertEqual(len(pg_records), 1) + self.assertEqual(pg_records[0]["platform"], "jd") + self.assertFalse(pg_records[0]["uploaded_to_bitable"]) + + def test_tmall_pg_payload_survives_feishu_record_lookup_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._write_source(root, platform="tm") + pg_records = [] + fields = { + "时间": "time", + "平台": "platform", + "曝光量": "impressions", + "点击量": "clicks", + "主图": "image", + } + with ( + patch.object( + tmall_insert, + "artifact_relative_path", + return_value="raw/product_commerce/main.jpg", + ), + patch.object(tmall_insert, "discover_fields", return_value=fields), + patch.object( + tmall_insert, + "list_existing_records", + side_effect=RuntimeError("Feishu record-list unavailable"), + ), + ): + result = tmall_insert.process_product( + "同款", + {"base_token": "base", "table_id": "table"}, + "2026-08-02", + root, + False, + pg_records, + ) + + self.assertIn("record-list 失败", result["errors"][0]) + self.assertEqual(len(pg_records), 1) + self.assertEqual(pg_records[0]["platform"], "tm") + + def test_jd_pg_payload_survives_feishu_record_lookup_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self._write_source(root, platform="jd") + pg_records = [] + fields = { + "时间": "time", + "平台": "platform", + "曝光量": "impressions", + "点击量": "clicks", + "主图": "image", + } + with ( + patch.object( + jd_insert, + "artifact_relative_path", + return_value="raw/product_commerce/main.jpg", + ), + patch.object(jd_insert, "discover_fields", return_value=fields), + patch.object( + jd_insert, + "list_existing_records", + side_effect=RuntimeError("Feishu record-list unavailable"), + ), + ): + result = jd_insert.process_style( + "同款", + {"base_token": "base", "table_id": "table"}, + "2026-08-02", + root, + False, + pg_records, + ) + + self.assertIn("record-list 失败", result["errors"][0]) + self.assertEqual(len(pg_records), 1) + self.assertEqual(pg_records[0]["platform"], "jd") + + def test_feishu_existing_record_lookup_is_platform_scoped(self): + fields = { + "时间": "time", + "平台": "platform", + "曝光量": "impressions", + "点击量": "clicks", + "主图": "image", + } + payload = { + "data": { + "field_id_list": list(fields.values()), + "record_id_list": ["tm-record", "jd-record"], + "data": [ + ["本周", ["天猫"], 10, 2, [{"name": "main.jpg"}]], + ["本周", ["京东"], 20, 3, [{"name": "main.jpg"}]], + ], + } + } + with patch.object( + tmall_insert, + "run_lark", + return_value=(0, json.dumps(payload, ensure_ascii=False), ""), + ) as run_lark: + by_image, metric_tuples, duplicates = ( + tmall_insert.list_existing_records( + "base", "table", fields, "本周", "京东" + ) + ) + + command = run_lark.call_args.args[0] + filter_json = json.loads(command[command.index("--filter-json") + 1]) + self.assertIn(["平台", "==", "京东"], filter_json["conditions"]) + self.assertEqual(by_image["main.jpg"]["record_id"], "jd-record") + self.assertEqual(metric_tuples, {(20, 3)}) + self.assertEqual(duplicates, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_hermes_notification.py b/tests/modules/product_commerce/test_market_rank_hermes_notification.py similarity index 92% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_hermes_notification.py rename to tests/modules/product_commerce/test_market_rank_hermes_notification.py index 0d076ca..20c48fd 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_hermes_notification.py +++ b/tests/modules/product_commerce/test_market_rank_hermes_notification.py @@ -6,7 +6,6 @@ from unittest.mock import MagicMock, patch import market_rank_hermes_notification as notification - REPORTS = [ { "platform": "tm", @@ -82,6 +81,22 @@ def test_extracts_real_feishu_message_id_from_hermes_completion(): assert notification.hermes_send_succeeded(response) is True +def test_hermes_success_text_without_real_message_id_is_not_success(): + response = { + "choices": [{"message": {"content": "发送成功，但没有返回消息回执"}}] + } + + assert notification.extract_feishu_message_id(response) is None + assert notification.hermes_send_succeeded(response) is False + + +def test_extracts_nested_feishu_message_id_and_accepts_safe_suffixes(): + response = {"data": {"message_id": "om_real-id_123"}} + + assert notification.extract_feishu_message_id(response) == "om_real-id_123" + assert notification.hermes_send_succeeded(response) is True + + def test_notify_skips_when_same_day_recipient_was_already_sent(): conn = MagicMock() with ( diff --git a/tests/modules/product_commerce/test_market_rank_limits.py b/tests/modules/product_commerce/test_market_rank_limits.py new file mode 100644 index 0000000..97e90e4 --- /dev/null +++ b/tests/modules/product_commerce/test_market_rank_limits.py @@ -0,0 +1,61 @@ +import pytest + +from gyxx_flow.modules.product_commerce.market_rank_limits import ( + category_rank_limit, + remaining_category_rank_limit, + require_valid_market_rank_products, +) + + +def test_tmall_and_douyin_collect_top_200_per_category(): + assert category_rank_limit("tm", "旅行箱") == 200 + assert category_rank_limit("tm", "双肩包") == 200 + assert category_rank_limit("dy", "旅行箱") == 200 + assert category_rank_limit("dy", "电脑包") == 200 + + +def test_jd_collects_top_150_for_luggage_and_top_100_for_other_categories(): + assert category_rank_limit("jd", "旅行箱") == 150 + assert category_rank_limit("jd", "运动包") == 100 + assert category_rank_limit("jd", "双肩包") == 100 + assert category_rank_limit("jd", "电脑包") == 100 + + +def test_remaining_limit_is_per_category_and_respects_optional_global_limit(): + assert remaining_category_rank_limit("tm", "双肩包", category_count=80) == 120 + assert remaining_category_rank_limit("jd", "旅行箱", category_count=120) == 30 + assert remaining_category_rank_limit("jd", "电脑包", category_count=100) == 0 + assert remaining_category_rank_limit( + "dy", + "双肩包", + category_count=20, + total_count=490, + global_limit=500, + ) == 10 + + +@pytest.mark.parametrize( + ("platform", "rows"), + [ + ("tm", ["not-a-product-row"]), + ("jd", [{"name": "", "rank": 1, "sales_amount_upper": 10}]), + ("dy", [{"name": "商品", "rank": 1, "sales_amount_upper": 0}]), + ], +) +def test_zero_valid_platform_results_are_never_success(platform, rows): + with pytest.raises(RuntimeError, match="0 条有效商品.*拒绝标记成功"): + require_valid_market_rank_products(platform, rows) + + +@pytest.mark.parametrize( + ("platform", "metric_name"), + [ + ("tm", "buyer_max"), + ("jd", "sales_amount_upper"), + ("dy", "sales_amount_upper"), + ], +) +def test_valid_platform_result_passes_gate(platform, metric_name): + row = {"name": "有效商品", "rank": 1, metric_name: 100} + + assert require_valid_market_rank_products(platform, [row]) == 1 diff --git a/tests/modules/product_commerce/test_market_rank_product_import.py b/tests/modules/product_commerce/test_market_rank_product_import.py new file mode 100644 index 0000000..c2f49da --- /dev/null +++ b/tests/modules/product_commerce/test_market_rank_product_import.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from decimal import Decimal +from pathlib import Path + +from gyxx_flow.modules.product_commerce.market_rank_product_import import ( + build_product_snapshot, + extract_document_labels, + normalize_scene_name, + normalize_style_name, +) + +RUNTIME_ROOT = ( + Path(__file__).resolve().parents[3] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" +) + + +def test_known_label_aliases_are_normalized_without_guessing_unknown_styles(): + assert normalize_style_name("都是新旅") == "都市新旅" + assert normalize_style_name("智性通勤") == "智性通勤" + assert normalize_style_name("都市运动") == "都市运动" + assert normalize_style_name("都市通勤") is None + assert normalize_scene_name("上班通勤") == "通勤上班" + assert normalize_scene_name("摄影爱好") == "摄影爱好" + + +def test_tmall_snapshot_uses_buyer_upper_times_cached_unit_price(): + snapshot = build_product_snapshot( + "tm", + { + "rank": 1, + "name": "旅行箱", + "buyer_range": "100 ~ 250", + "buyer_max": 250, + "product_url": "https://detail.tmall.com/item.htm?id=123", + "category_name": "旅行箱", + "category_path": "箱包 > 旅行箱", + }, + price_cache={"123": "¥418"}, + ) + + assert snapshot["platform_product_id"] == "123" + assert snapshot["unit_price_min"] == Decimal("418") + assert snapshot["unit_price_max"] == Decimal("418") + assert snapshot["estimated_gmv"] == Decimal("104500") + assert snapshot["estimate_method"] == "buyer_upper_x_unit_price" + assert snapshot["quality_status"] == "complete" + + +def test_douyin_and_jd_snapshots_use_reported_sales_upper(): + for platform in ("dy", "jd"): + snapshot = build_product_snapshot( + platform, + { + "product_id": "9988", + "name": "商品", + "sales_amount_range": "¥100万-¥250万", + "sales_amount_upper": 2_500_000, + "price_range": "¥398-¥617", + "product_url": "https://example.com/9988", + "category_name": "双肩包", + }, + ) + + assert snapshot["estimated_gmv"] == Decimal("2500000") + assert snapshot["estimate_method"] == "reported_sales_upper" + assert snapshot["quality_status"] == "complete" + + +def test_missing_tmall_price_keeps_product_but_excludes_it_from_amount(): + snapshot = build_product_snapshot( + "tm", + { + "buyer_max": 100, + "name": "没有价格的商品", + "product_url": "https://detail.tmall.com/item.htm?id=456", + "category_name": "相机包", + }, + price_cache={}, + ) + + assert snapshot["estimated_gmv"] is None + assert snapshot["quality_status"] == "missing_price" + + +def test_feishu_document_labels_keep_raw_values_and_normalized_values(): + content = """ + 京东榜单 +

旅行箱（1条）

+
+ + + + + + + + + + +
风格（运营手动区分）场景用途（运营手动区分）品类排名商品图商品名称销售额商品价格商品链接
都是新旅上班通勤旅行箱1测试商品￥50万~￥75万￥399打开商品
+ """ + + labels = extract_document_labels(content, "jd") + + assert labels["123456"]["raw_style_name"] == "都是新旅" + assert labels["123456"]["style_name"] == "都市新旅" + assert labels["123456"]["raw_scene_name"] == "上班通勤" + assert labels["123456"]["scene_name"] == "通勤上班" + assert labels["123456"]["unit_price_text"] == "￥399" + + +def test_collector_schema_contains_idempotent_market_snapshot_tables(): + schema = (RUNTIME_ROOT / "db" / "schema.sql").read_text(encoding="utf-8") + + assert "CREATE TABLE IF NOT EXISTS market_product" in schema + assert "CREATE TABLE IF NOT EXISTS fact_market_product_snapshot" in schema + assert "CREATE TABLE IF NOT EXISTS market_product_classification" in schema + assert "UNIQUE (report_id, market_product_id, category_name)" in schema diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_report_archive.py b/tests/modules/product_commerce/test_market_rank_report_archive.py similarity index 100% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_report_archive.py rename to tests/modules/product_commerce/test_market_rank_report_archive.py diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_workflow.py b/tests/modules/product_commerce/test_market_rank_workflow.py similarity index 63% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_workflow.py rename to tests/modules/product_commerce/test_market_rank_workflow.py index 4b9aa6b..28349bd 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_market_rank_workflow.py +++ b/tests/modules/product_commerce/test_market_rank_workflow.py @@ -2,26 +2,70 @@ from __future__ import annotations import json import sys +from datetime import date, timedelta from pathlib import Path from threading import Barrier import orchestrate_market_rank_collection as workflow +import pytest - -PROJECT_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +PROJECT_ROOT = ( + REPOSITORY_ROOT / "src" / "gyxx_flow" / "modules" / "product_commerce" +) +HISTORY_ROOT = REPOSITORY_ROOT / "docs" / "history" / "product-commerce" def test_default_platform_order_and_commands(): assert workflow.DEFAULT_PLATFORMS == ("tm", "jd", "dy") + report_date = date(2026, 8, 4) - tm_command = workflow.build_platform_command("tm", python_executable=sys.executable) - jd_command = workflow.build_platform_command("jd", python_executable=sys.executable) - dy_command = workflow.build_platform_command("dy", python_executable=sys.executable) + tm_command = workflow.build_platform_command( + "tm", report_date=report_date, python_executable=sys.executable + ) + jd_command = workflow.build_platform_command( + "jd", report_date=report_date, python_executable=sys.executable + ) + dy_command = workflow.build_platform_command( + "dy", report_date=report_date, python_executable=sys.executable + ) - assert tm_command[:2] == [sys.executable, "collect_sycm_market_rank.py"] + assert tm_command[0] == sys.executable + assert Path(tm_command[1]).name == "collect_sycm_market_rank.py" assert tm_command[-2:] == ["--risk-mode", "manual"] - assert jd_command == [sys.executable, "collect_jd_market_rank.py"] - assert dy_command == [sys.executable, "collect_dy_market_rank.py"] + assert jd_command[0] == sys.executable + assert Path(jd_command[1]).name == "collect_jd_market_rank.py" + assert dy_command[0] == sys.executable + assert Path(dy_command[1]).name == "collect_dy_market_rank.py" + for command in (tm_command, jd_command, dy_command): + assert command[command.index("--report-date") + 1] == "2026-08-04" + + +def test_historical_live_snapshot_date_is_rejected(): + today = date(2026, 8, 4) + + with pytest.raises(ValueError, match="拒绝错标"): + workflow.validate_live_snapshot_date( + today - timedelta(days=1), + current_date=today, + ) + + +def test_notification_recipient_comes_only_from_runtime_environment(): + assert workflow.notification_recipient_from_env({}) == ("", "") + assert workflow.notification_recipient_from_env( + { + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_runtime", + "GYXX_NOTIFICATION_RECIPIENT_NAME": "runtime-owner", + } + ) == ("ou_runtime", "runtime-owner") + assert workflow.notification_recipient_from_env( + { + "MARKET_RANK_NOTIFY_OPEN_ID": "ou_market", + "MARKET_RANK_NOTIFY_NAME": "market-owner", + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_generic", + } + ) == ("ou_market", "market-owner") def test_child_environment_forces_utf8_unbuffered_output(): @@ -131,6 +175,7 @@ def test_three_platforms_start_in_parallel(tmp_path): def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path): all_started = Barrier(3) notifications: list[dict] = [] + report_date = date.today() def synchronized_executor(*, platform, command, log_path, dry_run): all_started.wait(timeout=1) @@ -155,13 +200,20 @@ def test_workflow_notifies_after_all_parallel_steps_finish(tmp_path): run_id="notify-run", log_dir=tmp_path, executor=synchronized_executor, + report_date=report_date, notify_open_id="ou_wangyunlong", + notify_name="runtime-owner", notifier=fake_notifier, ) assert len(notifications) == 1 + assert notifications[0]["report_date"] == report_date assert notifications[0]["recipient_open_id"] == "ou_wangyunlong" + assert notifications[0]["recipient_name"] == "runtime-owner" assert notifications[0]["generated_after"] is not None + for step in summary["steps"]: + command = step["command"] + assert command[command.index("--report-date") + 1] == report_date.isoformat() assert summary["notification"]["status"] == "sent" saved = json.loads((tmp_path / "notify-run_summary.json").read_text(encoding="utf-8")) assert saved["notification"]["status"] == "sent" @@ -201,8 +253,46 @@ def test_platform_parser_deduplicates_while_preserving_order(): assert workflow.parse_platforms("dy,tm,dy,jd") == ("dy", "tm", "jd") +def test_main_requires_runtime_recipient_unless_notification_disabled(monkeypatch): + monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "") + monkeypatch.setattr(sys, "argv", ["orchestrate_market_rank_collection.py", "--dry-run"]) + + with pytest.raises(SystemExit) as exc_info: + workflow.main() + + assert exc_info.value.code == 2 + + +def test_main_allows_explicit_no_notify_without_recipient(monkeypatch, tmp_path): + monkeypatch.setattr(workflow, "DEFAULT_NOTIFY_OPEN_ID", "") + observed = {} + + def fake_execute(*args, **kwargs): + observed.update(kwargs) + return {"exit_code": 0} + + monkeypatch.setattr(workflow, "execute_workflow", fake_execute) + monkeypatch.setattr(workflow, "managed_data_path", Path) + monkeypatch.setattr( + sys, + "argv", + [ + "orchestrate_market_rank_collection.py", + "--dry-run", + "--no-notify", + "--log-dir", + str(tmp_path), + ], + ) + + assert workflow.main() == 0 + assert observed["notify_open_id"] is None + + def test_weekly_launcher_uses_market_rank_orchestrator(): - launcher = (PROJECT_ROOT / "run_weekly_market_rank.bat").read_text(encoding="utf-8") + launcher = ( + HISTORY_ROOT / "launchers_reference" / "run_weekly_market_rank.bat" + ).read_text(encoding="utf-8") assert ".venv\\Scripts\\python.exe" in launcher assert "orchestrate_market_rank_collection.py" in launcher diff --git a/tests/modules/product_commerce/test_persona_integrity.py b/tests/modules/product_commerce/test_persona_integrity.py new file mode 100644 index 0000000..e831f81 --- /dev/null +++ b/tests/modules/product_commerce/test_persona_integrity.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import json +import sys +from datetime import date +from pathlib import Path +from types import SimpleNamespace + +import collect_dy_persona_to_bitable as dy_worker +import collect_jd_persona_to_bitable as jd_worker +import collect_persona_to_bitable as tm_worker +import dy_audience_profile_collect as dy_collector +import pytest +import taobao_dmp_item_crowd_insight_screenshots as dmp + +from gyxx_flow.adapters import acceptance_policy + +WORKERS = (tm_worker, dy_worker, jd_worker) +BUSINESS_DATE = date(2026, 8, 4) + + +@pytest.mark.parametrize("worker", WORKERS) +def test_platform_collector_failure_forces_nonzero(worker) -> None: + assert worker.combined_result_exit_code( + [{"status": "ok"}], + collection_ok=False, + ) == 2 + + +@pytest.mark.parametrize("worker", WORKERS) +def test_formal_base_upsert_requires_response_record_id(monkeypatch, worker) -> None: + monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False) + monkeypatch.setattr( + worker.subprocess, + "run", + lambda *_a, **_k: SimpleNamespace( + returncode=0, + stdout=json.dumps({"ok": True, "data": {}}), + stderr="", + ), + ) + + ok, message = worker.upsert_record( + "bas_test", + "tbl_test", + {"时间": "8.4", "男性比例": 0.5}, + ) + + assert ok is False + assert "record_id" in message + + +@pytest.mark.parametrize("worker", WORKERS) +def test_formal_base_upsert_accepts_verified_response_record_id(monkeypatch, worker) -> None: + monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: False) + monkeypatch.setattr( + worker.subprocess, + "run", + lambda *_a, **_k: SimpleNamespace( + returncode=0, + stdout=json.dumps( + { + "ok": True, + "data": {"record": {"record_id_list": ["rec12345678"]}}, + } + ), + stderr="", + ), + ) + + ok, message = worker.upsert_record( + "bas_test", + "tbl_test", + {"时间": "8.4", "男性比例": 0.5}, + ) + + assert ok is True + assert message == "record_id=rec12345678" + + +@pytest.mark.parametrize("worker", WORKERS) +def test_acceptance_skip_is_not_formal_success(monkeypatch, worker) -> None: + monkeypatch.setattr(acceptance_policy, "skip_feishu_table_write", lambda *_a, **_k: True) + monkeypatch.setattr( + worker.subprocess, + "run", + lambda *_a, **_k: (_ for _ in ()).throw( + AssertionError("lark subprocess must not run after acceptance skip") + ), + ) + + ok, message = worker.upsert_record( + "bas_test", + "tbl_test", + {"时间": "8.4", "男性比例": 0.5}, + ) + + assert ok is False + assert "acceptance-skipped" in message + + +def test_tm_payload_must_match_requested_business_date(monkeypatch, tmp_path: Path) -> None: + style = "款A" + style_dir = tmp_path / BUSINESS_DATE.isoformat() / style + style_dir.mkdir(parents=True) + (style_dir / f"{style}_123_chart_values.json").write_text( + json.dumps( + { + "style_name": style, + "item_id": "123", + "business_date": "2026-08-03", + "chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + monkeypatch.setattr(tm_worker, "DMP_OUTPUT_ROOT", tmp_path) + + assert tm_worker.load_latest_chart_values(style, BUSINESS_DATE) is None + + +@pytest.mark.parametrize("worker", (dy_worker, jd_worker)) +def test_profile_payload_must_match_requested_business_date( + monkeypatch, + tmp_path: Path, + worker, +) -> None: + style = "款A" + style_dir = tmp_path / BUSINESS_DATE.isoformat() / style + style_dir.mkdir(parents=True) + (style_dir / f"{style}_123_profile.json").write_text( + json.dumps( + { + "style_name": style, + "product_id": "123", + "business_date": "2026-08-03", + "profile": {"gender_distribution": [{"name": "男", "value": 50}]}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path) + + assert worker.load_latest_profile(style, BUSINESS_DATE) is None + + +@pytest.mark.parametrize("worker", (tm_worker, dy_worker)) +def test_current_run_does_not_reuse_old_artifact(monkeypatch, tmp_path: Path, worker) -> None: + style = "款A" + style_dir = tmp_path / BUSINESS_DATE.isoformat() / style + style_dir.mkdir(parents=True) + if worker is tm_worker: + path = style_dir / f"{style}_123_chart_values.json" + payload = { + "style_name": style, + "item_id": "123", + "business_date": BUSINESS_DATE.isoformat(), + "chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]}, + } + monkeypatch.setattr(worker, "DMP_OUTPUT_ROOT", tmp_path) + load = worker.load_latest_chart_values + else: + path = style_dir / f"{style}_123_profile.json" + payload = { + "style_name": style, + "product_id": "123", + "business_date": BUSINESS_DATE.isoformat(), + "profile": {"gender_distribution": [{"name": "男", "value": 50}]}, + } + monkeypatch.setattr(worker, "OUTPUT_ROOT", tmp_path) + load = worker.load_latest_profile + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + + assert load( + style, + BUSINESS_DATE, + artifact_not_before=path.stat().st_mtime + 10, + ) is None + + +@pytest.mark.parametrize("worker", WORKERS) +def test_database_failure_stops_before_base_write(monkeypatch, worker) -> None: + if worker is tm_worker: + monkeypatch.setattr( + worker, + "load_latest_chart_values", + lambda *_a, **_k: { + "style_name": "款A", + "item_id": "123", + "business_date": BUSINESS_DATE.isoformat(), + "chart_values": {"用户性别": [{"category": "男性用户", "分析人群占比": 50}]}, + }, + ) + monkeypatch.setattr( + worker, + "chart_values_to_fields", + lambda *_a, **_k: {"时间": "8.4", "男性比例": 0.5}, + ) + normalize_name = "normalize_tm_payload" + elif worker is dy_worker: + monkeypatch.setattr( + worker, + "load_latest_profile", + lambda *_a, **_k: { + "style_name": "款A", + "product_id": "123", + "business_date": BUSINESS_DATE.isoformat(), + "profile": {"gender_distribution": [{"name": "男", "value": 50}]}, + }, + ) + monkeypatch.setattr(worker, "profile_has_positive_distribution", lambda _profile: True) + monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: []) + monkeypatch.setattr(worker, "profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5}) + normalize_name = "normalize_dy_payload" + else: + monkeypatch.setattr( + worker, + "load_latest_profile", + lambda *_a, **_k: { + "style_name": "款A", + "product_id": "123", + "business_date": BUSINESS_DATE.isoformat(), + "profile": {"性别": [{"name": "男", "value": 50}]}, + }, + ) + monkeypatch.setattr(worker, "get_subtable_fields", lambda *_a, **_k: []) + monkeypatch.setattr(worker, "map_profile_to_fields", lambda *_a, **_k: {"男性比例": 0.5}) + normalize_name = "normalize_jd_payload" + + fake_db = SimpleNamespace( + get_conn=lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")), + upsert_persona_metrics=lambda *_a, **_k: 1, + **{normalize_name: lambda _payload: {"valid": True}}, + ) + monkeypatch.setitem(sys.modules, "db", fake_db) + monkeypatch.setattr( + worker, + "upsert_record", + lambda *_a, **_k: (_ for _ in ()).throw( + AssertionError("Base write must not run after DB failure") + ), + ) + + result = worker.upsert_one_style( + "款A", + {"base_token": "bas_test", "table_id": "tbl_test"}, + "8.4", + "user", + False, + BUSINESS_DATE, + ) + + assert result["status"] == "db_failed" + + +def test_zero_distributions_are_not_valid_persona() -> None: + assert dy_worker.profile_has_positive_distribution( + { + "gender_distribution": [{"name": "男", "value": 0}], + "age_distribution": [], + "strategy_crowd_distribution": [{"name": "Z世代", "value": "0%"}], + } + ) is False + + +def test_dmp_skips_are_not_valid_style_results() -> None: + styles = {"款A": ["123"]} + assert dmp.records_exit_code( + styles, + [{"style_name": "款A", "status": "skipped", "skip_reason": "无图表"}], + ) == 2 + assert dmp.records_exit_code( + styles, + [ + { + "style_name": "款A", + "status": "ok", + "chart_values": { + "用户性别": [{"category": "男性用户", "分析人群占比": 50}] + }, + } + ], + ) == 0 + + +@pytest.mark.parametrize( + ("detector", "url", "body", "expected"), + ( + (dmp.blocking_page_reason, "https://dmp.taobao.com/", "暂无权限", "权限"), + (dy_collector.blocking_page_reason, "https://compass.test/", "请登录", "登录"), + (jd_worker.blocking_page_reason, "https://passport.jd.com/login", "", "登录"), + ), +) +def test_login_and_permission_pages_have_explicit_diagnostics( + detector, + url: str, + body: str, + expected: str, +) -> None: + assert expected in detector(url, body) + + +def test_persona_search_selectors_are_finite_and_evidence_based() -> None: + assert 1 <= len(dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS) <= 6 + assert 1 <= len(jd_worker.JD_SEARCH_INPUT_SELECTORS) <= 6 + assert all("input" in selector for selector in dy_collector.PRODUCT_SEARCH_INPUT_SELECTORS) + assert all("input" in selector for selector in jd_worker.JD_SEARCH_INPUT_SELECTORS) + + +def test_tm_collector_command_redacts_credentials() -> None: + rendered = tm_worker.redacted_command( + ["python", "collector.py", "--account", "secret-user", "--password", "secret-pass"] + ) + + assert "secret-user" not in rendered + assert "secret-pass" not in rendered + assert rendered.count("***") == 2 diff --git a/tests/modules/product_commerce/test_persona_launcher.py b/tests/modules/product_commerce/test_persona_launcher.py new file mode 100644 index 0000000..59fdd7a --- /dev/null +++ b/tests/modules/product_commerce/test_persona_launcher.py @@ -0,0 +1,57 @@ +import unittest +from datetime import date +from pathlib import Path + +import collect_dy_persona_to_bitable as dy_worker +import collect_jd_persona_to_bitable as jd_worker +import collect_persona_to_bitable as tm_worker +import run_daily_persona + + +class PersonaLauncherTests(unittest.TestCase): + def test_any_platform_failure_makes_launcher_fail(self): + self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 2, "jd": 0}), 2) + + def test_all_platforms_success_makes_launcher_success(self): + self.assertEqual(run_daily_persona.combined_exit_code({"tm": 0, "dy": 0, "jd": 0}), 0) + + def test_launcher_passes_business_date_to_each_worker(self): + command = run_daily_persona.build_child_command( + Path("collect_persona_to_bitable.py"), + date(2026, 8, 4), + ) + + self.assertEqual(command[-2:], ["--date", "2026-08-04"]) + + def test_workers_fail_when_permission_is_skipped(self): + results = [{"status": "ok"}, {"status": "permission_skipped"}] + + for worker in (tm_worker, dy_worker, jd_worker): + with self.subTest(worker=worker.__name__): + self.assertEqual(worker.combined_result_exit_code(results), 2) + + def test_workers_fail_when_no_styles_were_processed(self): + for worker in (tm_worker, dy_worker, jd_worker): + with self.subTest(worker=worker.__name__): + self.assertEqual(worker.combined_result_exit_code([]), 2) + + def test_workers_succeed_only_when_all_styles_are_successful(self): + results = [{"status": "ok"}, {"status": "dry-run"}] + + for worker in (tm_worker, dy_worker, jd_worker): + with self.subTest(worker=worker.__name__): + self.assertEqual(worker.combined_result_exit_code(results), 0) + + def test_workers_fail_when_collector_itself_failed(self): + results = [{"status": "ok"}] + + for worker in (tm_worker, dy_worker, jd_worker): + with self.subTest(worker=worker.__name__): + self.assertEqual( + worker.combined_result_exit_code(results, collection_ok=False), + 2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/modules/product_commerce/test_product_acceptance_feishu_boundaries.py b/tests/modules/product_commerce/test_product_acceptance_feishu_boundaries.py new file mode 100644 index 0000000..45e8819 --- /dev/null +++ b/tests/modules/product_commerce/test_product_acceptance_feishu_boundaries.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import io +import json +from pathlib import Path +from types import SimpleNamespace + +import check_nine_day_decline as decline +import insert_bitable_records as insert_records +import lark_cli_runtime +import market_rank_hermes_notification as market_notification +import pytest + +from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + + +def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path: + evidence = tmp_path / "product-evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False) + return evidence + + +@pytest.mark.parametrize( + "arguments", + [ + ["base", "+record-upsert"], + ["base", "+record-delete"], + ["sheets", "+values-batch-update"], + ], +) +def test_product_shared_lark_cli_skips_table_mutations( + monkeypatch, + tmp_path: Path, + arguments: list[str], +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + + def fail_if_spawned(*args, **kwargs): + raise AssertionError("lark-cli subprocess must not start") + + monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fail_if_spawned) + result = lark_cli_runtime.run_lark_cli(arguments) + + assert result["acceptance_skipped"] is True + + +def test_legacy_product_upsert_bypass_is_physically_skipped( + monkeypatch, + tmp_path: Path, +) -> None: + evidence = _enable_acceptance(monkeypatch, tmp_path) + + def fail_if_spawned(*args, **kwargs): + raise AssertionError("legacy lark-cli subprocess must not start") + + monkeypatch.setattr(insert_records.subprocess, "run", fail_if_spawned) + insert_records.run_lark_upsert("base_secret", "tbl_test", {"field": 1}) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["operation"].endswith("record-upsert") + assert payload["details"]["base_token"] == "" + + +def test_product_shared_lark_cli_keeps_production_write_behavior(monkeypatch) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="") + + monkeypatch.setattr(lark_cli_runtime, "_node_cli_entry", lambda: ("node", "run.js")) + monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fake_run) + result = lark_cli_runtime.run_lark_cli(["base", "+record-upsert"]) + + assert result == {"ok": True} + assert len(calls) == 1 + assert calls[0][:4] == ["node", "run.js", "--profile", "hermes-analyzer"] + + +def test_product_shared_lark_cli_allows_safe_profile_override(monkeypatch) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="") + + monkeypatch.setattr(lark_cli_runtime, "_node_cli_entry", lambda: ("node", "run.js")) + monkeypatch.setattr(lark_cli_runtime.subprocess, "run", fake_run) + lark_cli_runtime.run_lark_cli( + ["auth", "status", "--json"], + env={"GYXX_LARK_CLI_PROFILE": "product-analyzer.test"}, + ) + + assert calls[0][:4] == [ + "node", + "run.js", + "--profile", + "product-analyzer.test", + ] + + +def test_decline_hermes_prompt_forces_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + monkeypatch.setattr(decline, "HERMES_API_KEY", "test-only") + captured = {} + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode() + + def fake_urlopen(request, timeout): + captured["body"] = json.load(io.BytesIO(request.data)) + return Response() + + monkeypatch.setattr(decline.urllib.request, "urlopen", fake_urlopen) + decline.notify_hermes( + [{"style_name": "test", "platform": "tm"}], + "2026-08-01", + 9, + openid="ou_someone_else", + ) + + encoded = json.dumps(captured["body"], ensure_ascii=False) + assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded + assert "ou_someone_else" not in encoded + + +def test_market_rank_notification_forces_wang_yunlong( + monkeypatch, + tmp_path: Path, +) -> None: + _enable_acceptance(monkeypatch, tmp_path) + captured = {} + + class ConnectionContext: + def __enter__(self): + return object() + + def __exit__(self, *args): + return False + + monkeypatch.setattr(market_notification, "get_conn", ConnectionContext) + monkeypatch.setattr( + market_notification, + "ensure_market_rank_notification_table", + lambda conn: None, + ) + monkeypatch.setattr( + market_notification, + "get_market_rank_reports_for_date", + lambda *args, **kwargs: [ + { + "platform": "tm", + "feishu_doc_url": "https://example.test/tm", + "product_count": 1, + } + ], + ) + monkeypatch.setattr( + market_notification, + "market_rank_notification_was_sent", + lambda *args, **kwargs: False, + ) + + def fake_post(payload): + captured["payload"] = payload + return {"message_id": "om_test"} + + def fake_archive(conn, record): + captured["record"] = record + + monkeypatch.setattr(market_notification, "post_hermes", fake_post) + monkeypatch.setattr( + market_notification, + "upsert_market_rank_notification", + fake_archive, + ) + + result = market_notification.notify_market_rank_reports( + report_date=decline.date(2026, 8, 1), + recipient_open_id="ou_someone_else", + ) + + encoded = json.dumps(captured["payload"], ensure_ascii=False) + assert WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID in encoded + assert "ou_someone_else" not in encoded + assert result["recipient_open_id"] == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + assert ( + captured["record"]["recipient_open_id"] + == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + ) diff --git a/tests/modules/product_commerce/test_product_nested_child_bindings.py b/tests/modules/product_commerce/test_product_nested_child_bindings.py new file mode 100644 index 0000000..9bad766 --- /dev/null +++ b/tests/modules/product_commerce/test_product_nested_child_bindings.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import subprocess +from datetime import date +from pathlib import Path + +import backfill_collect +import backfill_one_day +import collect_dy_persona_to_bitable as dy_persona_worker +import collect_persona_to_bitable as tm_persona_worker +import orchestrate_daily_collection as daily +import orchestrate_market_rank_collection as market_rank +import orchestrate_review_collection as reviews +import run_alerts_with_retry as alerts +import run_daily_persona as persona +import run_weekly_jd_main_image as jd_main_image +import run_weekly_main_image as tm_main_image + + +def _fake_rebind(observed: dict[str, object]): + def rebind(target, base_environment): + observed["target"] = Path(target) + observed["base"] = dict(base_environment) + return { + **base_environment, + "GYXX_SCRIPT_ID": f"product_commerce:{Path(target).name}", + "GYXX_BROWSER_CDP_PORT": "child-port", + "GYXX_BROWSER_PROFILE_DIR": "child-profile", + "GYXX_BROWSER_COOKIE_FILE": "child-cookie", + "GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage", + } + + return rebind + + +def _assert_child_binding(environment: dict[str, str]) -> None: + assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port" + assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile" + assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie" + assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage" + + +def test_persona_builds_one_target_bound_environment_per_collector(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr(persona, "environment_for_child_script", _fake_rebind(observed)) + target = persona.PROJECT_ROOT / "collect_persona_to_bitable.py" + + environment = persona.build_child_environment(target, {"KEEP": "yes"}) + + assert observed["target"] == target + assert environment["KEEP"] == "yes" + _assert_child_binding(environment) + + +def test_persona_workers_rebind_their_nested_browser_collectors(monkeypatch) -> None: + cases = ( + ( + tm_persona_worker, + tm_persona_worker.DMP_SCRIPT, + lambda: tm_persona_worker.run_dmp_collect_all( + "parent-profile", + False, + "", + "", + date(2026, 8, 4), + ), + ), + ( + dy_persona_worker, + dy_persona_worker.COLLECT_SCRIPT, + lambda: dy_persona_worker.run_collect_all( + "parent-profile", + False, + date(2026, 8, 4), + ), + ), + ) + for worker, target, invoke in cases: + observed: dict[str, object] = {} + monkeypatch.setattr( + worker, + "environment_for_child_script", + _fake_rebind(observed), + ) + + def fake_run(command, **kwargs): + observed["command"] = command + observed["environment"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(worker.subprocess, "run", fake_run) + + assert invoke() is True + assert observed["target"] == target + command = observed["command"] + assert command[command.index("--date") + 1] == "2026-08-04" + _assert_child_binding(observed["environment"]) + + +def test_daily_run_step_rebinds_to_command_target(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr(daily, "environment_for_child_script", _fake_rebind(observed)) + + def fake_run(command, **kwargs): + observed["environment"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(daily.subprocess, "run", fake_run) + target = daily.PROJECT_ROOT / "dy_product_scraping.py" + + result = daily.run_step("dy", ["python", str(target)], critical=True) + + assert result["code"] == 0 + assert observed["target"] == target + _assert_child_binding(observed["environment"]) + + +def test_market_rank_rebinds_to_each_platform_target(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr( + market_rank, + "environment_for_child_script", + _fake_rebind(observed), + ) + target = market_rank.PROJECT_ROOT / "collect_jd_market_rank.py" + + environment = market_rank.build_subprocess_env( + {"KEEP": "yes"}, + target=target, + ) + + assert observed["target"] == target + assert environment["KEEP"] == "yes" + assert environment["PYTHONIOENCODING"] == "utf-8" + assert environment["PYTHONUNBUFFERED"] == "1" + _assert_child_binding(environment) + + +def test_review_run_step_rebinds_to_command_target(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr(reviews, "environment_for_child_script", _fake_rebind(observed)) + + def fake_run(command, **kwargs): + observed["environment"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(reviews.subprocess, "run", fake_run) + target = reviews.PROJECT_ROOT / "dy_product_scraping.py" + + result = reviews.run_step("dy", ["python", str(target)], critical=True) + + assert result["code"] == 0 + assert observed["target"] == target + _assert_child_binding(observed["environment"]) + + +def test_main_image_launchers_rebind_to_command_target(monkeypatch) -> None: + for launcher in (tm_main_image, jd_main_image): + observed: dict[str, object] = {} + monkeypatch.setattr( + launcher, + "environment_for_child_script", + _fake_rebind(observed), + ) + + def fake_run(command, **kwargs): + observed["environment"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(launcher.subprocess, "run", fake_run) + target = launcher.PROJECT_ROOT / "collector.py" + + result = launcher.run_step("collect", ["python", str(target)]) + + assert result["code"] == 0 + assert observed["target"] == target + _assert_child_binding(observed["environment"]) + + +def test_alert_wrapper_exposes_target_bound_environment(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr(alerts, "environment_for_child_script", _fake_rebind(observed)) + target = alerts.PROJECT_ROOT / "check_nine_day_decline.py" + + environment = alerts.build_child_environment(target, {"KEEP": "yes"}) + + assert observed["target"] == target + assert environment["KEEP"] == "yes" + _assert_child_binding(environment) + + +def test_backfill_range_rebinds_daily_orchestrator_after_overrides(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr( + backfill_collect, + "environment_for_child_script", + _fake_rebind(observed), + ) + target = backfill_collect.PROJECT_ROOT / "orchestrate_daily_collection.py" + + environment = backfill_collect._subprocess_env(target) + + assert observed["target"] == target + assert observed["base"]["KEEP_BROWSER_OPEN"] == "0" + assert environment["KEEP_BROWSER_OPEN"] == "0" + _assert_child_binding(environment) + + +def test_backfill_day_rebinds_each_stage_target(monkeypatch) -> None: + observed: dict[str, object] = {} + monkeypatch.setattr( + backfill_one_day, + "environment_for_child_script", + _fake_rebind(observed), + ) + + def fake_run(command, **kwargs): + observed["environment"] = kwargs["env"] + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(backfill_one_day.subprocess, "run", fake_run) + target = backfill_one_day.PROJECT_ROOT / "dy_product_scraping.py" + + code, _elapsed = backfill_one_day.run_subprocess( + "dy", + ["python", str(target)], + "2026-07-31", + ) + + assert code == 0 + assert observed["target"] == target + _assert_child_binding(observed["environment"]) diff --git a/tests/modules/product_commerce/test_product_workflow_business_dates.py b/tests/modules/product_commerce/test_product_workflow_business_dates.py new file mode 100644 index 0000000..0480bb5 --- /dev/null +++ b/tests/modules/product_commerce/test_product_workflow_business_dates.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] + + +def _workflow(workflow_id: str) -> dict: + catalog = json.loads( + (REPOSITORY_ROOT / "config" / "workflows.json").read_text(encoding="utf-8") + ) + return next(item for item in catalog["workflows"] if item["id"] == workflow_id) + + +def test_product_workflows_forward_the_selected_business_date() -> None: + persona = _workflow("product.persona.daily")["execution"]["steps"][0] + style = _workflow("product.style_analysis.interval")["execution"]["steps"][0] + main_image = _workflow("product.main_image.weekly")["execution"]["steps"] + market_rank = _workflow("product.market_rank")["execution"]["steps"][0] + monthly_sales = _workflow("product.sales_sheet.daily")["execution"]["steps"][0] + + assert persona["args"] == ["--date", "{business_date}"] + assert style["args"][-2:] == ["--end-date", "{business_date}"] + assert all( + step["args"][:2] == ["--date", "{business_date}"] + for step in main_image + ) + assert market_rank["args"] == ["--report-date", "{business_date}"] + assert monthly_sales["args"] == [ + "--month", + "{business_date}", + "--execute", + "--allow-missing-erp-as-zero", + ] diff --git a/tests/modules/product_commerce/test_style_analysis_orchestration.py b/tests/modules/product_commerce/test_style_analysis_orchestration.py new file mode 100644 index 0000000..402390f --- /dev/null +++ b/tests/modules/product_commerce/test_style_analysis_orchestration.py @@ -0,0 +1,695 @@ +import json +import os +import tempfile +import unittest +from datetime import date +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import analyze_style_with_hermes as analysis +import orchestrate_daily_collection as orchestrator +from config.style_config_loader import StyleConfigLoader + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +HISTORY_ROOT = REPOSITORY_ROOT / "docs" / "history" / "product-commerce" + + +class StyleAnalysisConfigTests(unittest.TestCase): + def test_platform_analysis_url_is_aggregated_by_style(self): + loader = StyleConfigLoader() + payload = loader._aggregate([ + { + "platform": "天猫", + "style": "盖亚斜挎", + "style_analysis_bitable_url": ( + "[分析表](https://example.feishu.cn/base/base_token" + "?table=table_id&view=view_id)" + ), + } + ]) + + self.assertEqual( + payload["styles"]["盖亚斜挎"]["style_analysis_bitable"], + {"base_token": "base_token", "table_id": "table_id", "view_id": "view_id"}, + ) + + +class StyleAnalysisWindowTests(unittest.TestCase): + def test_main_image_analysis_remains_tmall_scoped_after_jd_merge(self): + class Cursor: + sql = "" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql, params): + self.sql = sql + self.params = params + + @staticmethod + def fetchall(): + return [] + + class Connection: + def __init__(self, cursor): + self._cursor = cursor + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self._cursor + + cursor = Cursor() + with patch.object(analysis, "get_conn", return_value=Connection(cursor)): + rows = analysis.pull_main_image( + "同款", date(2026, 7, 1), date(2026, 7, 3) + ) + + self.assertEqual(rows, []) + self.assertIn("platform='tm'", cursor.sql) + + def test_three_day_label_uses_compact_end_date(self): + self.assertEqual( + analysis.analysis_period_label(date(2026, 7, 13), date(2026, 7, 15)), + "2026-07-13~07-15", + ) + + def test_due_styles_wait_three_full_days(self): + styles = ["A", "B", "C"] + latest = {"A": date(2026, 7, 15), "B": date(2026, 7, 12)} + + due = analysis.select_due_styles(styles, latest, date(2026, 7, 15), 3) + + self.assertEqual(due, ["B", "C"]) + + def test_prompt_keeps_all_dimensions_inside_current_three_day_window(self): + payload = { + "total": {"sales": 30, "visitors": 1000, "cart": 80}, + "platforms": {"tm": {"sales": 20}, "jd": {"sales": 10}}, + "daily": [{"date": "2026-07-14", "plat": "tm", "sales": 10}], + "persona": [{"date": "2026-07-14", "plat": "tm", "gender": []}], + "product_reviews": {"tm|negative": {"n": 2, "samples": ["肩带偏硬"]}}, + "main_image": [{"date": "2026-07-14", "key": "hero", "ctr": "2.1"}], + "marketing": { + "notes_by_platform_all": [{"platform": "xiaohongshu", "views": 1000}], + "note_comments_sample": [{"content": "求尺寸"}], + }, + } + + prompt = analysis.build_user_prompt( + "盖亚斜挎", date(2026, 7, 14), date(2026, 7, 16), payload, + ) + + self.assertIn("2026-07-14 ~ 2026-07-16", prompt) + self.assertNotIn("对比基期", prompt) + self.assertNotIn("紧邻上期", prompt) + self.assertNotIn("最近 6 周", prompt) + self.assertNotIn("近 60d", prompt) + self.assertIn("逐日经营数据", prompt) + self.assertIn("人物画像", prompt) + self.assertIn("商品评价", prompt) + self.assertIn("主图表现", prompt) + self.assertIn("营销笔记、曝光互动与评论", prompt) + self.assertIn("肩带偏硬", prompt) + self.assertIn("求尺寸", prompt) + self.assertIn('"sales": 30', prompt) + + def test_system_prompt_requires_each_business_dimension(self): + for dimension in ("访客", "加购", "人物画像", "营销笔记", "笔记评论", "商品评价", "主图"): + self.assertIn(dimension, analysis.SYSTEM) + + def test_system_prompt_requires_fixed_seven_section_report(self): + headings = [ + "## 一、访客分析", + "## 二、加购率分析", + "## 三、点击率分析", + "## 四、转化率分析", + "## 五、退货率分析", + "## 六、销量分析", + "## 七、总评", + ] + + for heading in headings: + self.assertIn(heading, analysis.SYSTEM) + + def test_plain_seven_section_titles_are_normalized_to_markdown(self): + report = """盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13) + +一、访客分析 +内容 +二、加购率分析 +内容 +三、点击率分析 +内容 +四、转化率分析 +内容 +五、退货率分析 +内容 +六、销量分析 +内容 +七、总评 +内容 +""" + + normalized = analysis.normalize_report_markdown(report) + + self.assertTrue(normalized.startswith("# 盖亚斜挎")) + self.assertTrue(analysis.has_exact_report_headings(normalized)) + + def test_report_gets_clear_subheadings_and_key_emphasis(self): + blocks = ["盖亚斜挎 单款式多维度分析报告 (2026-07-11~2026-07-13)"] + for heading in analysis.REPORT_HEADINGS[:-1]: + blocks.extend([ + heading.removeprefix("## "), + "数据快照：三日访客2692，转化率0.71%。其余数据。", + "问题判断：天猫访客高但转化偏低。需要重点处理。", + "可执行建议：P0，24小时内完成核查，目标转化率1.00%。", + ]) + blocks.extend([ + "七、总评", + "核心结论：当前核心问题是高访客、低转化。需要优先修复。", + "链路表现为：曝光到访客再到成交。", + "P0，48小时内完成详情页整改，目标转化率1.00%。", + ]) + + normalized = analysis.normalize_report_markdown("\n\n".join(blocks)) + + self.assertTrue(analysis.has_clear_report_hierarchy(normalized)) + self.assertTrue(analysis.has_key_emphasis(normalized)) + self.assertIn("### 数据快照", normalized) + self.assertIn("### 核心判断", normalized) + self.assertIn("### 优先级行动", normalized) + self.assertIn("**P0**", normalized) + self.assertIn("**24小时内**", normalized) + + def test_extra_helpful_h3_does_not_reject_complete_required_hierarchy(self): + required = "\n".join(analysis.REPORT_SUBHEADINGS) + report = required.replace( + "### 核心判断\n", + "### 平台明细\n补充内容\n### 核心判断\n", + 1, + ) + + self.assertTrue(analysis.has_clear_report_hierarchy(report)) + + def test_missing_calendar_day_is_reported(self): + rows = [ + {"date": "2026-07-14", "plat": "tm"}, + {"date": "2026-07-16", "plat": "jd"}, + ] + + missing = analysis.missing_window_dates( + rows, date(2026, 7, 14), date(2026, 7, 16), + ) + + self.assertEqual(missing, [date(2026, 7, 15)]) + + +class LarkEnvironmentTests(unittest.TestCase): + def test_legacy_lark_config_is_used_when_default_is_missing(self): + with tempfile.TemporaryDirectory() as temp_dir: + home = Path(temp_dir) + legacy = home / ".lark-cli" / "hermes" + legacy.mkdir(parents=True) + (legacy / "config.json").write_text("{}", encoding="utf-8") + with patch.dict(os.environ, {}, clear=True), patch.object(Path, "home", return_value=home): + env = analysis.lark_cli_env() + + self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(legacy)) + + def test_root_config_with_analyzer_profile_wins_over_legacy_config(self): + with tempfile.TemporaryDirectory() as temp_dir: + home = Path(temp_dir) + config_root = home / ".lark-cli" + legacy = config_root / "hermes" + legacy.mkdir(parents=True) + (config_root / "config.json").write_text( + json.dumps({"apps": [{"name": "hermes-analyzer"}]}), + encoding="utf-8", + ) + (legacy / "config.json").write_text("{}", encoding="utf-8") + with patch.dict(os.environ, {}, clear=True), patch.object(Path, "home", return_value=home): + env = analysis.lark_cli_env() + + self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(config_root)) + + def test_empty_root_config_falls_back_to_legacy_config(self): + with tempfile.TemporaryDirectory() as temp_dir: + home = Path(temp_dir) + config_root = home / ".lark-cli" + legacy = config_root / "hermes" + legacy.mkdir(parents=True) + (config_root / "config.json").write_text("{}", encoding="utf-8") + (legacy / "config.json").write_text("{}", encoding="utf-8") + + env = analysis.lark_cli_env(base_env={}, home=home) + + self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(legacy)) + + def test_explicit_config_directory_is_preserved(self): + with tempfile.TemporaryDirectory() as temp_dir: + home = Path(temp_dir) + explicit = home / "explicit-lark-config" + config_root = home / ".lark-cli" + config_root.mkdir(parents=True) + (config_root / "config.json").write_text( + json.dumps({"profiles": {"hermes-analyzer": {}}}), + encoding="utf-8", + ) + + env = analysis.lark_cli_env( + base_env={"LARKSUITE_CLI_CONFIG_DIR": str(explicit)}, + home=home, + ) + + self.assertEqual(env["LARKSUITE_CLI_CONFIG_DIR"], str(explicit)) + + +class FeishuDocumentWriteTests(unittest.TestCase): + def test_preflight_refreshes_and_requires_both_identities(self): + response = { + "verified": True, + "identities": { + "bot": {"available": True, "verified": True}, + "user": {"available": True, "verified": True}, + }, + } + + with patch.object(analysis, "run_lark_cli", return_value=response) as run: + analysis.ensure_lark_cli_ready() + + run.assert_called_once_with(["auth", "status", "--json", "--verify"]) + + def test_preflight_rejects_missing_user_identity(self): + response = { + "verified": True, + "identities": { + "bot": {"available": True, "verified": True}, + "user": {"available": False, "verified": False}, + }, + } + + with patch.object(analysis, "run_lark_cli", return_value=response): + with self.assertRaisesRegex(RuntimeError, "user"): + analysis.ensure_lark_cli_ready() + + def test_bot_created_document_is_granted_to_current_user(self): + responses = [ + {"data": {"document": { + "url": "https://example.feishu.cn/docx/doc_token", + "document_id": "doc_token", + }}}, + {"identities": {"user": {"openId": "ou_current_user"}}}, + {"ok": True, "data": {"perm": "full_access"}}, + ] + + with patch.object(analysis, "run_lark_cli", side_effect=responses) as run: + url, doc_id = analysis.write_feishu_doc("测试分析", "# 内容") + + self.assertEqual(url, "https://example.feishu.cn/docx/doc_token") + self.assertEqual(doc_id, "doc_token") + create_args = run.call_args_list[0].args[0] + self.assertEqual(create_args[create_args.index("--as") + 1], "bot") + content_arg = create_args[create_args.index("--content") + 1] + self.assertFalse(Path(content_arg.removeprefix("@")).is_absolute()) + self.assertEqual( + run.call_args_list[0].kwargs["cwd"], + analysis.report_cache_path("测试分析").parent, + ) + grant_args = run.call_args_list[2].args[0] + self.assertIn("+member-add", grant_args) + self.assertIn("ou_current_user", grant_args) + self.assertIn("full_access", grant_args) + self.assertIn("--yes", grant_args) + + def test_confirmed_automatic_permission_grant_skips_duplicate_member_add(self): + response = { + "data": {"document": { + "url": "https://example.feishu.cn/docx/doc_token", + "document_id": "doc_token", + }}, + "permission_grant": {"status": "granted", "perm": "full_access"}, + } + + with patch.object(analysis, "run_lark_cli", return_value=response) as run: + url, doc_id = analysis.write_feishu_doc("测试分析", "# 内容") + + self.assertEqual(url, "https://example.feishu.cn/docx/doc_token") + self.assertEqual(doc_id, "doc_token") + run.assert_called_once() + + def test_missing_current_user_identity_stops_before_base_write(self): + responses = [ + {"data": {"document": { + "url": "https://example.feishu.cn/docx/doc_token", + "document_id": "doc_token", + }}}, + {"identities": {"user": {"ready": False}}}, + ] + + with patch.object(analysis, "run_lark_cli", side_effect=responses): + with self.assertRaisesRegex(RuntimeError, "open_id"): + analysis.write_feishu_doc("测试分析", "# 内容") + + +class LocalReportRecoveryTests(unittest.TestCase): + def test_only_complete_exact_window_report_is_reused(self): + with tempfile.TemporaryDirectory() as tmp: + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)): + title = "测试款 单款式多维度分析 2026-08-01~2026-08-03" + report_lines: list[str] = [] + for index, heading in enumerate(analysis.REPORT_HEADINGS): + report_lines.append(heading) + for subheading in analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3]: + report_lines.extend([subheading, "**重点**"]) + report = "\n".join(report_lines) + cache = analysis.report_cache_path(title) + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(report, encoding="utf-8") + + self.assertEqual( + analysis.load_validated_cached_report(title), + analysis.normalize_report_markdown(report), + ) + + cache.write_text("## 一、访客分析\n### 数据快照\n**重点**", encoding="utf-8") + self.assertIsNone(analysis.load_validated_cached_report(title)) + + +class ExternalEffectCheckpointTests(unittest.TestCase): + @staticmethod + def checkpoint_spec(report: str = "# 内容"): + return analysis.StyleAnalysisCheckpointSpec.for_report( + "测试款", + date(2026, 8, 1), + date(2026, 8, 3), + 3, + "测试款 单款式多维度分析 2026-08-01~2026-08-03", + report, + ) + + @staticmethod + def complete_report() -> str: + lines: list[str] = [] + for index, heading in enumerate(analysis.REPORT_HEADINGS): + lines.append(heading) + for subheading in analysis.REPORT_SUBHEADINGS[index * 3:(index + 1) * 3]: + lines.extend([subheading, "**重点**"]) + return analysis.normalize_report_markdown("\n".join(lines)) + + def test_permission_failure_resumes_same_created_document(self): + created = { + "data": {"document": { + "url": "https://example.feishu.cn/docx/doc_token", + "document_id": "doc_token", + }}, + "permission_grant": {"status": "failed"}, + } + auth = {"identities": {"user": {"openId": "ou_current_user"}}} + + with tempfile.TemporaryDirectory() as tmp: + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)): + spec = self.checkpoint_spec() + with patch.object( + analysis, + "run_lark_cli", + side_effect=[created, auth, RuntimeError("grant failed")], + ): + with self.assertRaisesRegex(RuntimeError, "grant failed"): + analysis.write_feishu_doc(spec.title, "# 内容", spec) + + state = analysis.load_style_analysis_checkpoint(spec) + self.assertEqual(state["phase"], "doc_created") + self.assertEqual(state["document"]["id"], "doc_token") + + with patch.object( + analysis, + "run_lark_cli", + side_effect=[auth, {"ok": True}], + ) as resumed: + url, doc_id = analysis.write_feishu_doc(spec.title, "# 内容", spec) + + self.assertEqual(url, "https://example.feishu.cn/docx/doc_token") + self.assertEqual(doc_id, "doc_token") + self.assertFalse(any( + call.args[0][:2] == ["docs", "+create"] + for call in resumed.call_args_list + )) + self.assertEqual( + analysis.load_style_analysis_checkpoint(spec)["phase"], + "permission_granted", + ) + + def test_crash_after_automatic_grant_resumes_without_member_add(self): + with tempfile.TemporaryDirectory() as tmp: + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)): + spec = self.checkpoint_spec() + analysis.claim_style_analysis_checkpoint(spec) + analysis.advance_style_analysis_checkpoint( + spec, + "doc_created", + document={ + "id": "doc_token", + "url": "https://example.feishu.cn/docx/doc_token", + }, + automatic_permission_status="granted", + ) + + with patch.object(analysis, "run_lark_cli") as run: + url, doc_id = analysis.write_feishu_doc(spec.title, "# 内容", spec) + + self.assertEqual(url, "https://example.feishu.cn/docx/doc_token") + self.assertEqual(doc_id, "doc_token") + run.assert_not_called() + self.assertEqual( + analysis.load_style_analysis_checkpoint(spec)["phase"], + "permission_granted", + ) + + def test_ambiguous_and_corrupt_checkpoints_fail_closed(self): + with tempfile.TemporaryDirectory() as tmp: + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)): + ambiguous = self.checkpoint_spec() + state, owns_creation = analysis.claim_style_analysis_checkpoint(ambiguous) + self.assertTrue(owns_creation) + self.assertEqual(state["phase"], "doc_creating") + with patch.object(analysis, "run_lark_cli") as run: + with self.assertRaisesRegex( + analysis.StyleAnalysisCheckpointError, + "doc_creating", + ): + analysis.write_feishu_doc(ambiguous.title, "# 内容", ambiguous) + run.assert_not_called() + + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp) / "corrupt"): + corrupt = self.checkpoint_spec() + corrupt.path().parent.mkdir(parents=True, exist_ok=True) + corrupt.path().write_text("{", encoding="utf-8") + with patch.object(analysis, "run_lark_cli") as run: + with self.assertRaisesRegex( + analysis.StyleAnalysisCheckpointError, + "损坏", + ): + analysis.write_feishu_doc(corrupt.title, "# 内容", corrupt) + run.assert_not_called() + + def test_report_hash_mismatch_fails_closed(self): + with tempfile.TemporaryDirectory() as tmp: + with patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)): + original = self.checkpoint_spec("# 原报告") + analysis.claim_style_analysis_checkpoint(original) + changed = self.checkpoint_spec("# 新报告") + + with patch.object(analysis, "run_lark_cli") as run: + with self.assertRaisesRegex( + analysis.StyleAnalysisCheckpointError, + "哈希不匹配", + ): + analysis.write_feishu_doc(changed.title, "# 新报告", changed) + run.assert_not_called() + + def test_base_and_postgres_phases_resume_without_recreating_document(self): + class Connection: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + report = self.complete_report() + daily = [ + {"date": "2026-08-01"}, + {"date": "2026-08-02"}, + {"date": "2026-08-03"}, + ] + created = { + "data": {"document": { + "url": "https://example.feishu.cn/docx/doc_token", + "document_id": "doc_token", + }}, + "permission_grant": {"status": "granted", "perm": "full_access"}, + } + target = {"base_token": "base", "table_id": "table"} + + with tempfile.TemporaryDirectory() as tmp: + with ( + patch.object(analysis, "CURATED_DATA_ROOT", Path(tmp)), + patch.object(analysis, "pull_daily", return_value=daily), + patch.object(analysis, "aggregate_period", return_value={"tm": {"sales": 1}}), + patch.object(analysis, "aggregate_style_total", return_value={"sales": 1}), + patch.object(analysis, "pull_persona", return_value=[]), + patch.object(analysis, "pull_reviews", return_value={}), + patch.object(analysis, "pull_main_image", return_value=[]), + patch.object(analysis, "pull_seeding", return_value={}), + patch.object(analysis, "build_user_prompt", return_value="prompt"), + patch.object(analysis, "run_lark_cli", return_value=created) as run, + patch.object( + analysis, + "write_analysis_link_to_base", + return_value="rec_1", + ) as write_base, + patch.object(analysis, "get_conn", return_value=Connection()), + patch.object( + analysis, + "upsert_style_analysis_report", + side_effect=[RuntimeError("pg unavailable"), 42], + ) as write_pg, + ): + title = "测试款 单款式多维度分析 2026-08-01~2026-08-03" + cache = analysis.report_cache_path(title) + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_text(report, encoding="utf-8") + spec = self.checkpoint_spec(report) + + with self.assertRaisesRegex(RuntimeError, "pg unavailable"): + analysis.analyze_one_style( + "测试款", + date(2026, 8, 1), + date(2026, 8, 3), + 3, + False, + target, + ) + base_state = analysis.load_style_analysis_checkpoint(spec) + self.assertEqual(base_state["phase"], "base_written") + self.assertEqual(base_state["base_record_id"], "rec_1") + + ok, _ = analysis.analyze_one_style( + "测试款", + date(2026, 8, 1), + date(2026, 8, 3), + 3, + False, + target, + ) + + self.assertTrue(ok) + self.assertEqual(run.call_count, 1) + write_base.assert_called_once() + self.assertEqual(write_pg.call_count, 2) + final_state = analysis.load_style_analysis_checkpoint(spec) + self.assertEqual(final_state["phase"], "pg_written") + self.assertEqual(final_state["pg_row_id"], 42) + + +class TargetBaseWriteTests(unittest.TestCase): + def test_existing_period_record_is_updated(self): + responses = [ + {"data": {"fields": [ + {"name": "时间", "type": "select"}, + {"name": "笔记分析汇总", "type": "text"}, + ]}}, + {"data": {"field": { + "id": "fld_time", "name": "时间", "type": "select", + "multiple": False, "options": [{"name": "2026-07-13~07-15"}], + }}}, + {"data": { + "data": [[['2026-07-13~07-15'], "old link"]], + "fields": ["时间", "笔记分析汇总"], + "record_id_list": ["rec_existing"], + }}, + {"ok": True, "data": {"record_id": "rec_existing"}}, + ] + with patch.object(analysis, "run_lark_cli", side_effect=responses) as run: + record_id = analysis.write_analysis_link_to_base( + {"base_token": "base", "table_id": "table"}, + "2026-07-13~07-15", + "https://example.feishu.cn/docx/doc", + "盖亚斜挎", + ) + + self.assertEqual(record_id, "rec_existing") + upsert_args = run.call_args_list[3].args[0] + self.assertIn("--record-id", upsert_args) + self.assertIn("rec_existing", upsert_args) + payload = json.loads(upsert_args[upsert_args.index("--json") + 1]) + self.assertIn("笔记分析汇总", payload) + + def test_missing_period_is_appended_without_dropping_existing_options(self): + responses = [ + {"data": {"field": { + "id": "fld_time", "name": "时间", "type": "select", + "multiple": False, "options": [{"name": "7月第二周", "hue": "Blue"}], + }}}, + {"ok": True}, + ] + with patch.object(analysis, "run_lark_cli", side_effect=responses) as run: + analysis.ensure_select_option( + {"base_token": "base", "table_id": "table"}, + "时间", "2026-07-13~07-15", "盖亚斜挎", + ) + + update_args = run.call_args_list[1].args[0] + definition = json.loads(update_args[update_args.index("--json") + 1]) + self.assertEqual( + [option["name"] for option in definition["options"]], + ["7月第二周", "2026-07-13~07-15"], + ) + self.assertIn("--yes", update_args) + + +class OrchestratorTests(unittest.TestCase): + def test_style_analysis_stage_runs_three_day_batch(self): + args = SimpleNamespace(dry_run=False) + with patch.object(orchestrator, "run_step_with_retry") as run: + orchestrator.run_style_analysis(args, "2026-07-15", failed=None) + + command = run.call_args.args[1] + self.assertIn("analyze_style_with_hermes.py", [Path(item).name for item in command]) + self.assertIn("--all-styles", command) + self.assertEqual(command[command.index("--days") + 1], "3") + self.assertEqual(command[command.index("--min-interval-days") + 1], "3") + self.assertFalse(run.call_args.kwargs["critical"]) + + +class LauncherScheduleTests(unittest.TestCase): + def test_daily_launcher_does_not_run_style_analysis(self): + launcher = ( + HISTORY_ROOT / "launchers_reference" / "run_daily_collect.bat" + ).read_text(encoding="utf-8") + + command_lines = [line for line in launcher.splitlines() if "orchestrate_daily_collection.py" in line] + self.assertEqual(len(command_lines), 1) + self.assertNotIn("style_analysis", command_lines[0]) + + def test_three_day_launcher_runs_only_style_analysis(self): + launcher = ( + HISTORY_ROOT / "launchers_reference" / "run_style_analysis_3d.bat" + ).read_text(encoding="utf-8") + + self.assertIn("analyze_style_with_hermes.py", launcher) + self.assertIn("--all-styles", launcher) + self.assertIn("--days 3", launcher) + self.assertIn("--skip-existing", launcher) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_style_config_loader_erp_source.py b/tests/modules/product_commerce/test_style_config_loader_erp_source.py similarity index 67% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_style_config_loader_erp_source.py rename to tests/modules/product_commerce/test_style_config_loader_erp_source.py index 53d915c..987e232 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_style_config_loader_erp_source.py +++ b/tests/modules/product_commerce/test_style_config_loader_erp_source.py @@ -1,13 +1,7 @@ from __future__ import annotations import io -import sys from contextlib import redirect_stdout -from pathlib import Path - - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(PROJECT_ROOT)) from config.style_config_loader import StyleConfigLoader @@ -43,3 +37,27 @@ def test_non_tm_erp_code_is_not_used_when_tm_code_is_missing() -> None: assert payload["styles"]["极星托特"]["erp_codes"] == [] assert "天猫分组未填写" in output.getvalue() + + +def test_exact_style_field_wins_over_style_content_prefix(monkeypatch) -> None: + loader = StyleConfigLoader() + monkeypatch.setattr( + loader, + "_run_lark", + lambda _args: { + "data": { + "fields": ["款式内容", "款式", "ERP款式编码", "平台"], + "data": [["营销文案", "盖世m1", "10416,10455", ["天猫"]]], + } + }, + ) + + records = loader._load_records_from_lark() + + assert records == [ + { + "style": "盖世m1", + "erp_codes": "10416,10455", + "platform": "天猫", + } + ] diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_sycm_market_rank.py b/tests/modules/product_commerce/test_sycm_market_rank.py similarity index 94% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_sycm_market_rank.py rename to tests/modules/product_commerce/test_sycm_market_rank.py index e26324a..9c43458 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_sycm_market_rank.py +++ b/tests/modules/product_commerce/test_sycm_market_rank.py @@ -4,26 +4,26 @@ from unittest.mock import MagicMock, patch from collect_sycm_market_rank import ( TARGET_MARKET_CATEGORIES, - _original_taobao_image_url, _feishu_append_xml_path, + _original_taobao_image_url, _product_item_id, _risk_control_reason, - build_feishu_doc_xml, build_feishu_category_xml, + build_feishu_doc_xml, close_market_rank_browser, collect_all_rank_pages, + create_feishu_doc, detect_risk_control_text, + ensure_market_rank_login, + group_products_by_category, inspect_feishu_batch_tables, inspect_feishu_category_tables, - group_products_by_category, - create_feishu_doc, parse_market_rank_html, parse_range_upper, prepare_market_rank_filters, prepare_market_rank_period, ) - SAMPLE_HTML = """ @@ -49,6 +49,33 @@ SAMPLE_HTML = """ class SycmMarketRankTests(unittest.TestCase): + def test_expired_login_without_credentials_fails_clearly(self): + with ( + patch("collect_sycm_market_rank.sycm.ACCOUNT", ""), + patch("collect_sycm_market_rank.sycm.PASSWORD", ""), + patch( + "collect_sycm_market_rank.sycm.ensure_logged_in", + return_value=None, + ), + self.assertRaisesRegex( + RuntimeError, + "登录态失效.*SYCM_ACCOUNT/SYCM_PASSWORD", + ), + ): + ensure_market_rank_login(MagicMock(), MagicMock()) + + def test_expired_login_does_not_swallow_login_exception(self): + with ( + patch("collect_sycm_market_rank.sycm.ACCOUNT", "configured"), + patch("collect_sycm_market_rank.sycm.PASSWORD", "configured"), + patch( + "collect_sycm_market_rank.sycm.ensure_logged_in", + side_effect=RuntimeError("login rejected"), + ), + self.assertRaisesRegex(RuntimeError, "自动登录失败: RuntimeError"), + ): + ensure_market_rank_login(MagicMock(), MagicMock()) + def test_parallel_documents_use_isolated_append_xml_files(self): first = _feishu_append_xml_path("https://example.feishu.cn/docx/first", 1) second = _feishu_append_xml_path("https://example.feishu.cn/docx/second", 1) diff --git a/tests/modules/product_commerce/test_sync_monthly_sales_sheet.py b/tests/modules/product_commerce/test_sync_monthly_sales_sheet.py new file mode 100644 index 0000000..19928c1 --- /dev/null +++ b/tests/modules/product_commerce/test_sync_monthly_sales_sheet.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +from datetime import date + +import pytest + +from gyxx_flow.modules.product_commerce import ( + collect_erp_monthly_metrics as erp_monthly, +) +from gyxx_flow.modules.product_commerce.collect_erp_monthly_metrics import ( + monthly_report_interval, + set_monthly_filters, +) +from gyxx_flow.modules.product_commerce.sync_monthly_sales_sheet import ( + ErpStylePlan, + MonthlyMetric, + MonthlySalesConfig, + ProductTarget, + SheetInfo, + _write_rows, + build_parser, + choose_template, + fetch_product_targets, + match_metrics, + merge_product_catalog, + month_bounds, + month_option, + month_sheet_title, + normalized_name, + parse_month, + plan_erp_styles, + product_text, + require_complete_erp_mappings, +) + + +def test_month_contract_uses_natural_month_and_existing_title_convention() -> None: + month = parse_month("2026-08-19") + + assert month == date(2026, 8, 1) + assert month_bounds(month) == (date(2026, 8, 1), date(2026, 8, 31)) + assert month_option(month) == "2026.08" + assert month_sheet_title(month) == "26年8月" + + +def test_erp_month_interval_uses_inclusive_natural_month_for_day_input() -> None: + assert monthly_report_interval(parse_month("2026-07-02")) == ( + "2026-07-01", + "2026-07-31", + ) + assert monthly_report_interval(parse_month("2026-12-31")) == ( + "2026-12-01", + "2026-12-31", + ) + assert monthly_report_interval(parse_month("2024-02-15")) == ( + "2024-02-01", + "2024-02-29", + ) + + +def test_selected_month_is_the_only_month_argument() -> None: + arguments = build_parser().parse_args(["--month", "2026-07"]) + + assert parse_month(arguments.month) == date(2026, 7, 1) + with pytest.raises(SystemExit): + build_parser().parse_args( + ["--month", "2026-07", "--budget-month", "2026-08"] + ) + + +def test_missing_erp_zero_override_is_explicit() -> None: + default = build_parser().parse_args(["--month", "2026-07"]) + override = build_parser().parse_args( + ["--month", "2026-07", "--allow-missing-erp-as-zero"] + ) + + assert default.allow_missing_erp_as_zero is False + assert override.allow_missing_erp_as_zero is True + + +def test_product_targets_are_read_without_a_month_filter() -> None: + config = MonthlySalesConfig( + spreadsheet_url="https://example.test/sheets/token", + base_url="https://example.test/base/token", + base_token="base", + table_id="table", + view_id="view", + product_field="品名", + target_field="目标销量", + month_field="月份", + product_aliases={}, + ) + + calls: list[list[str]] = [] + + def lark(args: list[str]) -> dict: + calls.append(args) + return { + "data": { + "fields": ["品名", "目标销量"], + "data": [["布谷", 500], ["白鹭", 350], ["蓝鹊", 400]], + "has_more": False, + } + } + + result = fetch_product_targets( + config, + lark=lark, + ) + + assert result == [ + ProductTarget("布谷", 500), + ProductTarget("白鹭", 350), + ProductTarget("蓝鹊", 400), + ] + assert "--filter-json" not in calls[0] + + +def test_full_catalog_unions_style_base_and_targets_and_sums_duplicates() -> None: + targets = [ + ProductTarget("布谷", 500), + ProductTarget("星迹&星迹2", 1000), + ProductTarget("星迹2", 500), + ProductTarget("预算孤儿", 200), + ] + styles = { + "布谷": {"erp_codes": ["10352", "10496"]}, + "星迹2": {"erp_codes": ["10504"]}, + "白鹭": {"erp_codes": ["10310"]}, + "蓝鹊": {"erp_codes": ["10279"]}, + } + + result = merge_product_catalog( + targets, + styles, + {"星迹&星迹2": "星迹2"}, + ) + + assert result == [ + ProductTarget("布谷", 500), + ProductTarget("星迹&星迹2", 1500), + ProductTarget("预算孤儿", 200), + ProductTarget("白鹭", 0), + ProductTarget("蓝鹊", 0), + ] + + +def test_product_text_turns_base_markdown_links_into_plain_product_names() -> None: + assert ( + product_text( + "[盖亚微单Pro生命进程（新）](https://example.feishu.cn/base/token)" + ) + == "盖亚微单Pro" + ) + assert product_text("阿波罗X1生命进程（新）") == "阿波罗X1" + assert product_text("拾影相机包（新） 副本") == "拾影相机包" + + +def test_metric_matching_uses_explicit_aliases_and_casefolded_exact_names() -> None: + targets = [ + ProductTarget("盖世M1", 300), + ProductTarget("星迹&星迹2", 1500), + ProductTarget("尚未入库", 200), + ] + metrics = { + "盖世m1": MonthlyMetric("盖世m1", 271, 36, 12), + "星迹2": MonthlyMetric("星迹2", 1858, 451, 12), + } + + rows, unmatched = match_metrics( + targets, + metrics, + {"星迹&星迹2": "星迹2"}, + ) + + assert normalized_name("星迹＆星迹2") == normalized_name("星迹+星迹2") + assert rows[0]["sales"] == 271 + assert rows[1]["returns"] == 451 + assert rows[2]["sales"] == 0 + assert unmatched == ["尚未入库"] + + +def test_template_is_latest_strictly_earlier_year_month_sheet() -> None: + sheets = [ + SheetInfo("a", "26年6月", 0, 198), + SheetInfo("b", "26年7月", 1, 198), + SheetInfo("c", "7月销量", 2, 200), + ] + + assert choose_template(sheets, date(2026, 8, 1)).sheet_id == "b" + + +def test_erp_style_plan_uses_style_base_codes_and_aliases() -> None: + targets = [ + ProductTarget("盖世M1", 300), + ProductTarget("星迹&星迹2", 1500), + ProductTarget("云栖相机双肩包", 200), + ] + styles = { + "盖世m1": {"erp_codes": ["10416", "10455", "10416"]}, + "星迹2": {"erp_codes": ["10504", "10394"]}, + "云栖": {"erp_codes": []}, + } + + plans, missing = plan_erp_styles( + targets, + styles, + { + "星迹&星迹2": "星迹2", + "云栖相机双肩包": "云栖", + }, + ) + + assert plans == [ + ErpStylePlan("盖世m1", ("10416", "10455")), + ErpStylePlan("星迹2", ("10504", "10394")), + ] + assert missing == ["云栖相机双肩包"] + + +def test_execute_guard_rejects_incomplete_erp_mapping_before_sheet_write() -> None: + with pytest.raises(RuntimeError, match="目标表未清空"): + require_complete_erp_mappings(["云栖相机双肩包", "轻风双肩包"]) + + require_complete_erp_mappings([]) + + +def test_monthly_filter_sets_natural_month_exact_shop_labels_and_erp_code() -> None: + class Frame: + def __init__(self) -> None: + self.script = "" + self.args = {} + + def evaluate(self, script, args): + self.script = script + self.args = args + return {"selected_shop_ids": ["shop_1"], "selected_shop_labels": ["甲店"]} + + frame = Frame() + + result = set_monthly_filters( + frame, + erp_code="10439", + start_date="2026-07-01", + end_date="2026-07-31", + shop_labels=["甲店"], + ) + + assert frame.args == { + "erpCode": "10439", + "startDate": "2026-07-01", + "endDate": "2026-07-31", + "shopLabels": ["甲店"], + } + assert 'input[name="shop_id"]' in frame.script + assert "#i_id" in frame.script + assert result["selected_shop_labels"] == ["甲店"] + + +def test_monthly_collector_sums_all_erp_codes_for_one_style(monkeypatch) -> None: + monkeypatch.setattr(erp_monthly.erp, "find_report_frames", lambda _page: (object(), object())) + observed_ranges: list[tuple[str, str, str]] = [] + + def run_one_code(_page, _frame, code, start_date, end_date, *_args): + observed_ranges.append((code, start_date, end_date)) + return { + "ok": True, + "yesterday_sales": {"10416": 1185, "10455": 904}[code], + "yesterday_returns": {"10416": 279, "10455": 159}[code], + } + + monkeypatch.setattr( + erp_monthly, + "_run_one_code", + run_one_code, + ) + + result = erp_monthly.collect_monthly_in_page( + object(), + [{"style_name": "盖世m1", "erp_style_codes": ["10416", "10455"]}], + date(2026, 7, 1), + shop_labels=["甲店"], + ) + + assert result["盖世m1"].sales == 2089 + assert result["盖世m1"].returns == 438 + assert observed_ranges == [ + ("10416", "2026-07-01", "2026-07-31"), + ("10455", "2026-07-01", "2026-07-31"), + ] + + +def test_monthly_collector_rejects_partial_erp_code_failure(monkeypatch) -> None: + monkeypatch.setattr(erp_monthly.erp, "find_report_frames", lambda _page: (object(), object())) + monkeypatch.setattr( + erp_monthly, + "_run_one_code", + lambda _page, _frame, code, *_args: { + "ok": code == "10416", + "yesterday_sales": 10 if code == "10416" else 0, + "yesterday_returns": 1 if code == "10416" else 0, + "error": "timeout" if code != "10416" else "", + }, + ) + + with pytest.raises(RuntimeError, match="10455"): + erp_monthly.collect_monthly_in_page( + object(), + [{"style_name": "盖世m1", "erp_style_codes": ["10416", "10455"]}], + date(2026, 7, 1), + shop_labels=["甲店"], + ) + + +def test_write_rows_repeats_first_data_row_format_without_overwriting_values() -> None: + calls: list[list[str]] = [] + config = MonthlySalesConfig( + spreadsheet_url="https://example.test/sheets/token", + base_url="https://example.test/base/token", + base_token="base", + table_id="table", + view_id="view", + product_field="品名", + target_field="目标销量", + month_field="月份", + product_aliases={}, + ) + rows = [ + {"product": "甲", "target_sales": 10, "sales": 8, "returns": 1}, + {"product": "乙", "target_sales": 20, "sales": 9, "returns": 2}, + ] + + _write_rows( + config, + SheetInfo("sheet", "26年7月", 0, 198), + rows, + lark=lambda args: calls.append(args) or {}, + ) + + format_call = next(args for args in calls if "+range-copy" in args) + assert format_call[format_call.index("--source-range") + 1] == "A2:G2" + assert format_call[format_call.index("--target-range") + 1] == "A3:G3" + assert format_call[format_call.index("--paste-type") + 1] == "formats" diff --git a/tests/modules/product_commerce/test_tm_daily_exit_contract.py b/tests/modules/product_commerce/test_tm_daily_exit_contract.py new file mode 100644 index 0000000..aa781c7 --- /dev/null +++ b/tests/modules/product_commerce/test_tm_daily_exit_contract.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from pathlib import Path + +import taobao_sycm_products as tm + + +class _Session: + def __init__(self) -> None: + self.context = object() + self.closed = False + + def close(self) -> None: + self.closed = True + + +def test_tm_main_fails_when_browser_connection_is_missing(monkeypatch) -> None: + monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (None, None)) + + assert tm.main(["--target-date", "2026-08-03"]) == 1 + + +def test_tm_main_fails_when_login_does_not_complete(monkeypatch) -> None: + session = _Session() + page = object() + monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page)) + monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: None) + monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0) + + assert tm.main(["--target-date", "2026-08-03"]) == 1 + assert session.closed is True + + +def test_tm_main_fails_when_target_date_cannot_be_selected(monkeypatch) -> None: + session = _Session() + page = object() + monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page)) + monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page) + monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None) + monkeypatch.setattr(tm, "select_yesterday", lambda _page: False) + monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0) + + assert tm.main(["--target-date", "2026-08-03"]) == 1 + assert session.closed is True + + +def test_tm_main_fails_when_download_is_missing(monkeypatch) -> None: + session = _Session() + page = object() + monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page)) + monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page) + monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None) + monkeypatch.setattr(tm, "select_yesterday", lambda _page: True) + monkeypatch.setattr(tm, "click_download", lambda _page: None) + monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0) + + assert tm.main(["--target-date", "2026-08-03"]) == 1 + assert session.closed is True + + +def test_tm_main_fails_when_download_has_no_valid_style( + monkeypatch, + tmp_path: Path, +) -> None: + session = _Session() + page = object() + report = tmp_path / "report.xlsx" + monkeypatch.setattr(tm, "connect_browser", lambda **_kwargs: (session, page)) + monkeypatch.setattr(tm, "ensure_logged_in", lambda *_args: page) + monkeypatch.setattr(tm, "navigate_to_products", lambda _page: None) + monkeypatch.setattr(tm, "select_yesterday", lambda _page: True) + monkeypatch.setattr(tm, "click_download", lambda _page: report) + monkeypatch.setattr(tm, "export_excel_to_json_and_md", lambda _path: (None, None)) + monkeypatch.setattr(tm, "summarize_styles_from_excel", lambda _path: []) + monkeypatch.setattr(tm, "_kill_port_listeners", lambda _port: 0) + + assert tm.main(["--target-date", "2026-08-03"]) == 1 + assert session.closed is True diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_tm_persona_recovery.py b/tests/modules/product_commerce/test_tm_persona_recovery.py similarity index 88% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_tm_persona_recovery.py rename to tests/modules/product_commerce/test_tm_persona_recovery.py index 2ac4580..a95ee2f 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_tm_persona_recovery.py +++ b/tests/modules/product_commerce/test_tm_persona_recovery.py @@ -1,15 +1,20 @@ -import sys import unittest from pathlib import Path from unittest.mock import Mock -PROJECT_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(PROJECT_ROOT)) - import taobao_dmp_item_crowd_insight_screenshots as dmp class TmPersonaRecoveryTests(unittest.TestCase): + def test_zero_chart_skip_is_not_platform_success(self): + self.assertEqual( + dmp.records_exit_code( + {"款A": ["123"]}, + [{"style_name": "款A", "status": "skipped"}], + ), + 2, + ) + def test_switch_drawer_failure_retries_page_then_rebuilds_session(self): failed = { "status": "failed", diff --git a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_wanxiang_report_template.py b/tests/modules/product_commerce/test_wanxiang_report_template.py similarity index 87% rename from src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_wanxiang_report_template.py rename to tests/modules/product_commerce/test_wanxiang_report_template.py index b1cfde2..6793d9c 100644 --- a/src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/test_wanxiang_report_template.py +++ b/tests/modules/product_commerce/test_wanxiang_report_template.py @@ -8,6 +8,22 @@ from taobao_wanxiang_ai_creative_report import select_report_template class WanxiangReportTemplateTests(unittest.TestCase): + def test_authenticated_account_chooser_on_login_route_is_valid_session(self): + page = Mock() + page.url = "https://one.alimama.com/index.html#!/login/index" + page.locator.return_value.inner_text.return_value = ( + "欢迎登录\nHi，测试店铺\n进入后台\n退出账户" + ) + + self.assertTrue(wanxiang.is_logged_in(page)) + + def test_credential_form_on_login_route_is_not_valid_session(self): + page = Mock() + page.url = "https://one.alimama.com/index.html#!/login/index" + page.locator.return_value.inner_text.return_value = "账号名\n请输入登录密码" + + self.assertFalse(wanxiang.is_logged_in(page)) + def test_supports_actual_wanxiang_template_label_spelling(self): self.assertIn("报表模板", wanxiang.REPORT_TEMPLATE_LABELS) self.assertIn("报表模版", wanxiang.REPORT_TEMPLATE_LABELS) diff --git a/tests/modules/shop_intelligence/test_browser_session_adapter.py b/tests/modules/shop_intelligence/test_browser_session_adapter.py new file mode 100644 index 0000000..3b91de2 --- /dev/null +++ b/tests/modules/shop_intelligence/test_browser_session_adapter.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from gyxx_flow.modules.shop_intelligence.browser_session import ( + install_jd_dom_compatibility, + reuse_or_run_jd_login, +) + + +class _AccountInput: + def __init__(self, *, visible: bool) -> None: + self._visible = visible + + @property + def first(self): + return self + + def count(self) -> int: + return int(self._visible) + + def is_visible(self, *, timeout: int) -> bool: + assert timeout == 1_000 + return self._visible + + +class _Context: + def __init__(self) -> None: + self.scripts: list[str] = [] + + def add_init_script(self, *, script: str) -> None: + self.scripts.append(script) + + +def test_jd_session_adapter_reuses_authenticated_profile() -> None: + inputs = {"shop": "account", "credential": "credential"} + page = SimpleNamespace( + url="https://shop.jd.com/jdm/home", + goto=lambda *_args, **_kwargs: None, + wait_for_timeout=lambda *_args: None, + locator=lambda _selector: _AccountInput(visible=False), + ) + + result = reuse_or_run_jd_login( + page, + lambda *_args: (_ for _ in ()).throw( + AssertionError("authenticated profile must not repeat source login") + ), + shop=inputs["shop"], + password=inputs["credential"], + ) + + assert result is None + + +def test_jd_session_adapter_invokes_source_login_when_form_is_visible() -> None: + calls: list[tuple[object, str, str]] = [] + inputs = {"shop": "account", "credential": "credential"} + page = SimpleNamespace( + url="https://shop.jd.com/jdm/home", + goto=lambda *_args, **_kwargs: None, + wait_for_timeout=lambda *_args: None, + locator=lambda _selector: _AccountInput(visible=True), + ) + + reuse_or_run_jd_login( + page, + lambda current_page, shop, password: calls.append( + (current_page, shop, password) + ), + shop=inputs["shop"], + password=inputs["credential"], + ) + + assert calls == [(page, "account", "credential")] + + +def test_jd_dom_compatibility_aliases_current_and_future_pages() -> None: + context = _Context() + evaluated: list[str] = [] + page = SimpleNamespace( + context=context, + evaluate=lambda script: evaluated.append(script), + ) + + install_jd_dom_compatibility(page) + + assert len(context.scripts) == 1 + assert evaluated == context.scripts + assert "jmtd-date-picker-combo" in context.scripts[0] + assert "jmt-combo-date-picker" in context.scripts[0] + assert "jmt-date-picker" in context.scripts[0] + assert "classList.remove('jmt-date-picker')" in context.scripts[0] + assert "element.dataset.eventContent === 'realtime'" in context.scripts[0] + assert "gyxx-disabled-week-number" in context.scripts[0] + assert "if (!document.documentElement)" in context.scripts[0] + assert "setTimeout(installObserver, 0)" in context.scripts[0] + assert "childList: true" in context.scripts[0] + assert "attributes: true" not in context.scripts[0] + assert 'data-event-content="week"' in context.scripts[0] + assert "currentWeekTag.click()" in context.scripts[0] + assert "__gyxxJdWeekActivationAt" in context.scripts[0] + assert "gyxxWeekActivationAttempted" in context.scripts[0] + assert ").find(visible)" in context.scripts[0] + assert "gyxxWeekAdapterInstalled" in context.scripts[0] + assert "addEventListener('mouseenter'" in context.scripts[0] diff --git a/tests/modules/shop_intelligence/test_collector_exit_semantics.py b/tests/modules/shop_intelligence/test_collector_exit_semantics.py new file mode 100644 index 0000000..82e6f12 --- /dev/null +++ b/tests/modules/shop_intelligence/test_collector_exit_semantics.py @@ -0,0 +1,386 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from gyxx_flow.modules.shop_intelligence.collectors import ( + dy_store_competitor_store_scraping as dy_collector, +) +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_data_collector, + jd_peer_store_data_collector, +) + + +class _PlaywrightContext: + def __init__(self, page: object) -> None: + self.page = page + + def __enter__(self): + context = SimpleNamespace( + new_page=lambda: self.page, + close=lambda: None, + ) + browser = SimpleNamespace( + new_context=lambda **kwargs: context, + close=lambda: None, + ) + chromium = SimpleNamespace(launch=lambda **kwargs: browser) + return SimpleNamespace(chromium=chromium) + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + +def test_jd_shop_collector_returns_nonzero_when_collection_fails( + monkeypatch, +) -> None: + monkeypatch.setattr( + jd_data_collector, + "sync_playwright", + lambda: _PlaywrightContext(object()), + ) + monkeypatch.setattr( + jd_data_collector, + "step_login", + lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")), + ) + + assert jd_data_collector.main([]) == 1 + + +def test_jd_shop_login_preserves_source_account_password_flow() -> None: + events: list[tuple[str, object]] = [] + + class Field: + def fill(self, value: str) -> None: + events.append(("fill", value)) + + def click(self) -> None: + events.append(("click", "login")) + + fields = { + 'input[placeholder="请输入账号名/邮箱"]': Field(), + 'input[placeholder="请输入登录密码"]': Field(), + 'button:has-text("立即登录")': Field(), + } + page = SimpleNamespace( + goto=lambda url, **kwargs: events.append(("goto", (url, kwargs))), + wait_for_timeout=lambda timeout: events.append(("wait", timeout)), + wait_for_load_state=lambda *_args, **_kwargs: None, + locator=lambda selector: fields[selector], + evaluate=lambda script: events.append(("evaluate", script)), + keyboard=SimpleNamespace( + press=lambda key: events.append(("key", key)), + ), + ) + + assert jd_data_collector.step_login(page, "account", "credential") is None + assert ("fill", "account") in events + assert ("fill", "credential") in events + assert ("click", "login") in events + + +def test_jd_peer_collector_returns_nonzero_when_collection_fails( + monkeypatch, +) -> None: + monkeypatch.setattr( + jd_peer_store_data_collector, + "sync_playwright", + lambda: _PlaywrightContext(object()), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_login", + lambda *args: (_ for _ in ()).throw(RuntimeError("page changed")), + ) + + assert jd_peer_store_data_collector.main([]) == 1 + + +def test_dy_callback_preserves_failure_for_main_process(monkeypatch) -> None: + failure = RuntimeError("date control changed") + persisted = [] + monkeypatch.setattr( + dy_collector, + "_page_automation_impl", + lambda page: (_ for _ in ()).throw(failure), + ) + monkeypatch.setattr( + dy_collector, + "_save_browser_state", + lambda *args: persisted.append(args), + ) + + with pytest.raises(RuntimeError, match="date control changed"): + dy_collector._page_automation(object()) + + assert dy_collector._automation_error is failure + assert persisted == [] + + +def test_dy_outer_callback_persists_final_cross_domain_state(monkeypatch) -> None: + events = [] + page = SimpleNamespace( + url="https://fxg.jinritemai.com/ffa/eco/experience-score" + ) + monkeypatch.setattr( + dy_collector, + "current_acceptance_policy", + lambda: SimpleNamespace(enabled=False), + ) + monkeypatch.setattr( + dy_collector, + "_page_automation_impl", + lambda actual_page: events.append(("source", actual_page)), + ) + monkeypatch.setattr( + dy_collector, + "_save_browser_state", + lambda actual_page, stage: events.append(("persist", actual_page, stage)), + ) + + dy_collector._page_automation(page) + + assert events == [ + ("source", page), + ("persist", page, "after cross-domain stages"), + ] + assert dy_collector._automation_error is None + + +def test_dy_final_state_snapshot_keeps_all_cross_domain_cookies( + monkeypatch, + tmp_path, +) -> None: + cookie_file = tmp_path / "cookies.json" + storage_file = tmp_path / "storage_state.json" + cookies = [ + {"name": "compass", "value": "one", "domain": ".jinritemai.com"}, + {"name": "qianchuan", "value": "two", "domain": ".oceanengine.com"}, + {"name": "fxg", "value": "three", "domain": "fxg.jinritemai.com"}, + ] + + class Context: + def cookies(self): + return cookies + + def storage_state(self, *, path: str) -> None: + with open(path, "w", encoding="utf-8") as stream: + json.dump({"cookies": cookies, "origins": []}, stream) + + monkeypatch.setattr(dy_collector, "COOKIES_FILE", str(cookie_file)) + monkeypatch.setattr(dy_collector, "STORAGE_STATE_FILE", str(storage_file)) + + dy_collector._save_browser_state( + SimpleNamespace(context=Context()), + "after cross-domain stages", + ) + + saved_cookies = json.loads(cookie_file.read_text(encoding="utf-8")) + saved_storage = json.loads(storage_file.read_text(encoding="utf-8")) + assert {item["domain"] for item in saved_cookies} == { + ".jinritemai.com", + ".oceanengine.com", + "fxg.jinritemai.com", + } + assert saved_storage["cookies"] == cookies + + +def test_dy_main_returns_nonzero_when_fetcher_swallows_callback_failure( + monkeypatch, + tmp_path, +) -> None: + monkeypatch.setattr( + dy_collector, + "current_acceptance_policy", + lambda: SimpleNamespace(enabled=True), + ) + monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module")) + monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data")) + monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store")) + monkeypatch.setattr( + dy_collector, + "COMPETITOR_DATA_DIR", + str(tmp_path / "competitor"), + ) + monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile")) + monkeypatch.setattr(dy_collector, "_load_cookies", lambda: []) + monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None) + monkeypatch.setattr( + dy_collector, + "_page_automation_impl", + lambda page: (_ for _ in ()).throw(RuntimeError("date control changed")), + ) + monkeypatch.setattr( + dy_collector, + "save_results", + lambda data: (_ for _ in ()).throw( + AssertionError("failed automation must not save fallback data") + ), + ) + + def fake_fetch(url, *, page_action, **kwargs): + try: + page_action(object()) + except RuntimeError: + pass + return SimpleNamespace(url=url, status=200) + + monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch) + + assert dy_collector.main() == 1 + + +def test_dy_main_treats_final_login_response_as_cookie_skip( + monkeypatch, + tmp_path, +) -> None: + monkeypatch.setattr( + dy_collector, + "current_acceptance_policy", + lambda: SimpleNamespace(enabled=True), + ) + monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module")) + monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data")) + monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store")) + monkeypatch.setattr( + dy_collector, + "COMPETITOR_DATA_DIR", + str(tmp_path / "competitor"), + ) + monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile")) + monkeypatch.setattr(dy_collector, "_load_cookies", lambda: []) + monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None) + monkeypatch.setattr(dy_collector, "_session_expired_exit_code", lambda: 75) + monkeypatch.setattr( + dy_collector, + "_page_automation_impl", + lambda page: (_ for _ in ()).throw(RuntimeError("navigation aborted")), + ) + monkeypatch.setattr( + dy_collector, + "save_results", + lambda data: (_ for _ in ()).throw( + AssertionError("expired session must not save fallback data") + ), + ) + + def fake_fetch(url, *, page_action, **kwargs): + try: + page_action(object()) + except RuntimeError: + pass + return SimpleNamespace( + url="https://compass.jinritemai.com/login?roleType=shop", + status=200, + ) + + monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch) + + assert dy_collector.main() == 75 + assert isinstance(dy_collector._automation_error, RuntimeError) + + +def test_dy_production_keeps_source_success_after_optional_page_redirect( + monkeypatch, + tmp_path, +) -> None: + monkeypatch.setattr( + dy_collector, + "current_acceptance_policy", + lambda: SimpleNamespace(enabled=False), + ) + monkeypatch.setattr(dy_collector, "OUTPUT_DIR", str(tmp_path / "module")) + monkeypatch.setattr(dy_collector, "DATA_DIR", str(tmp_path / "data")) + monkeypatch.setattr(dy_collector, "STORE_DATA_DIR", str(tmp_path / "store")) + monkeypatch.setattr( + dy_collector, + "COMPETITOR_DATA_DIR", + str(tmp_path / "competitor"), + ) + monkeypatch.setattr(dy_collector, "USER_DATA_DIR", str(tmp_path / "profile")) + monkeypatch.setattr(dy_collector, "_load_cookies", lambda: []) + monkeypatch.setattr(dy_collector, "enable_adaptive_fetchers", lambda: None) + monkeypatch.setattr(dy_collector, "_results_saved", True) + monkeypatch.setattr(dy_collector, "_automation_error", None) + monkeypatch.setattr(dy_collector, "_page_automation_impl", lambda page: None) + + def fake_fetch(url, *, page_action, **kwargs): + page_action(SimpleNamespace(url=url)) + return SimpleNamespace( + url="https://fxg.jinritemai.com/login/common", + status=200, + ) + + monkeypatch.setattr(dy_collector.DynamicFetcher, "fetch", fake_fetch) + + assert dy_collector.main() == 0 + + +def test_dy_login_redirect_is_not_treated_as_business_page() -> None: + assert dy_collector._is_login_page_url( + "https://fxg.jinritemai.com/login/common" + ) + assert not dy_collector._is_login_page_url( + "https://compass.jinritemai.com/shop/business-part" + ) + + +def test_dy_login_form_is_detected_when_spa_url_does_not_change() -> None: + counts = { + 'input[name="mobile"]': 1, + 'input[name="mobilecaptcha"]': 1, + ".account-center-action-button": 1, + } + page = SimpleNamespace( + locator=lambda selector: SimpleNamespace(count=lambda: counts[selector]) + ) + + assert dy_collector._has_interactive_login_form(page) + + +def test_dy_acceptance_login_redirect_returns_cookie_skip(monkeypatch) -> None: + recorded = [] + + class Policy: + enabled = True + + def record(self, event, **payload): + recorded.append((event, payload)) + + monkeypatch.setattr(dy_collector, "current_acceptance_policy", lambda: Policy()) + + assert dy_collector._session_expired_exit_code() == 75 + assert recorded[0][0] == "cookie_skipped" + assert recorded[0][1]["details"]["status"] == "SKIPPED_COOKIE" + + +def test_dy_session_expiry_is_classified_by_outer_callback(monkeypatch) -> None: + class Policy: + enabled = True + + monkeypatch.setattr( + dy_collector, + "current_acceptance_policy", + lambda: Policy(), + ) + page = SimpleNamespace( + url="https://compass.jinritemai.com/login?roleType=shop", + ) + monkeypatch.setattr(dy_collector, "_save_browser_debug", lambda *args: None) + monkeypatch.setattr( + dy_collector, + "_page_automation_impl", + lambda *_args: (_ for _ in ()).throw( + AssertionError("expired session must not enter source business logic") + ), + ) + + with pytest.raises( + dy_collector.BrowserSessionExpiredError, + match="redirected to an interactive login page", + ): + dy_collector._page_automation(page) diff --git a/tests/modules/shop_intelligence/test_db_batch_upsert.py b/tests/modules/shop_intelligence/test_db_batch_upsert.py new file mode 100644 index 0000000..931061a --- /dev/null +++ b/tests/modules/shop_intelligence/test_db_batch_upsert.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from gyxx_flow.modules.shop_intelligence.db import db as shop_db + + +class _Cursor: + rowcount = 1 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + +class _Connection: + def __init__(self) -> None: + self.cursor_instance = _Cursor() + + def cursor(self) -> _Cursor: + return self.cursor_instance + + +def test_dynamic_batch_upsert_reports_all_input_records(monkeypatch) -> None: + records = [ + {"week_start": "2026-07-27", "brand": "A", "rank": 1}, + {"week_start": "2026-07-27", "brand": "B", "rank": 2}, + {"week_start": "2026-07-27", "brand": "C", "rank": 3}, + ] + executed = {} + + monkeypatch.setattr( + shop_db, + "_get_table_columns", + lambda conn, table: {"week_start", "brand", "rank"}, + ) + + def fake_execute_batch(cursor, sql, data, *, page_size): + executed.update( + cursor=cursor, + sql=sql, + data=data, + page_size=page_size, + ) + + monkeypatch.setattr(shop_db.psycopg2.extras, "execute_batch", fake_execute_batch) + + connection = _Connection() + persisted = shop_db._dynamic_batch_upsert( + connection, + "jd_peer_store_ranking", + records, + ("week_start", "brand"), + ) + + assert connection.cursor_instance.rowcount == 1 + assert persisted == len(records) + assert executed["cursor"] is connection.cursor_instance + assert executed["data"] == [ + ["2026-07-27", "A", 1], + ["2026-07-27", "B", 2], + ["2026-07-27", "C", 3], + ] + assert executed["page_size"] == 200 diff --git a/tests/modules/shop_intelligence/test_douyin_price_appeal.py b/tests/modules/shop_intelligence/test_douyin_price_appeal.py new file mode 100644 index 0000000..2447352 --- /dev/null +++ b/tests/modules/shop_intelligence/test_douyin_price_appeal.py @@ -0,0 +1,1748 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from gyxx_flow.modules.shop_intelligence import douyin_price_appeal as appeal +from gyxx_flow.modules.shop_intelligence.collectors import ( + dy_store_competitor_store_scraping as collector, +) + + +class _EmptyLocator: + def count(self) -> int: + return 0 + + +class _VisibleControl: + def __init__(self) -> None: + self.clicked = False + + def count(self) -> int: + return 1 + + def nth(self, index: int) -> _VisibleControl: + assert index == 0 + return self + + def is_visible(self, timeout: int | None = None) -> bool: + del timeout + return True + + def scroll_into_view_if_needed(self, timeout: int | None = None) -> None: + del timeout + + def click(self, timeout: int | None = None) -> None: + del timeout + self.clicked = True + + +class _LocatorCollection: + def __init__(self, items: list[_VisibleControl]) -> None: + self.items = items + + def count(self) -> int: + return len(self.items) + + def nth(self, index: int) -> _VisibleControl: + return self.items[index] + + +class _CarouselDots: + def __init__(self, buttons: list[_VisibleControl]) -> None: + self.buttons = _LocatorCollection(buttons) + + def locator(self, selector: str) -> _LocatorCollection: + assert selector == "button" + return self.buttons + + +class _CarouselSlider: + def __init__(self, dots: list[_CarouselDots]) -> None: + self.dots = _LocatorCollection(dots) # type: ignore[arg-type] + + def locator(self, selector: str) -> _LocatorCollection: + assert selector == ".slick-dots" + return self.dots + + +class _PositionedControl(_VisibleControl): + def __init__( + self, + *, + box: dict[str, float], + depth: int, + metric_count: int | None = None, + metric_depth: int = 0, + parent: _PositionedControl | None = None, + fail_click: bool = False, + data_key: str = "", + active_states: tuple[tuple[bool, bool], ...] = ((False, True),), + click_failures: int = 0, + carousel_activation: dict[str, object] | None = None, + slider: _CarouselSlider | None = None, + ) -> None: + super().__init__() + self.box = box + self.depth = depth + self.metric_count = metric_count + self.metric_depth = metric_depth + self.parent = parent + self.fail_click = fail_click + self.data_key = data_key + self.active_states = iter(active_states) + self.latest_active_state = active_states[-1] + self.click_failures = click_failures + self.carousel_activation = carousel_activation + self.slider = slider + + def bounding_box(self) -> dict[str, float]: + return self.box + + def evaluate( + self, + script: str, + expected_count: int | None = None, + ) -> int | bool | dict[str, object] | None: + if "elementFromPoint" in script: + assert expected_count is not None + assert "closest('.slick-slide')" in script + slide_hidden, hit_safe = next( + self.active_states, + self.latest_active_state, + ) + if ( + slide_hidden + or not hit_safe + or self.metric_count != expected_count + ): + return None + return { + "key": self.data_key, + "text": f"待改价 {self.metric_count} 紧急", + "box": self.box, + } + if "dotsDomIndex" in script: + assert expected_count is not None + return self.carousel_activation + if "data-log_element_key" in script: + assert expected_count is not None + if self.metric_count != expected_count: + return None + return { + "key": self.data_key, + "text": f"待改价 {self.metric_count} 紧急", + "box": self.box, + } + if "expectedCount" in script: + assert expected_count is not None + return self.metric_depth if self.metric_count == expected_count else None + assert expected_count is None + if "window.innerWidth" in script: + left = self.box["x"] + top = self.box["y"] + right = left + self.box["width"] + bottom = top + self.box["height"] + return ( + self.box["width"] > 0 + and self.box["height"] > 0 + and min(right, 1280) - max(left, 0) > 0 + and min(bottom, 720) - max(top, 0) > 0 + ) + assert "parentElement" in script + return self.depth + + def locator(self, selector: str) -> _PositionedControl: + if selector == "xpath=..": + assert self.parent is not None + return self.parent + assert "slick-slider" in selector + assert self.slider is not None + return self.slider # type: ignore[return-value] + + def click(self, timeout: int | None = None) -> None: + if self.click_failures > 0: + self.click_failures -= 1 + raise TimeoutError("timeout: another element intercepts pointer events") + if self.fail_click: + raise TimeoutError("timeout: another element intercepts pointer events") + super().click(timeout=timeout) + + +class _PendingMetricPage: + def __init__( + self, + candidates: list[_PositionedControl], + *, + clock: _FakeClock | None = None, + ) -> None: + self.candidates = _LocatorCollection(candidates) + self.clock = clock + self.wait_calls = 0 + + def get_by_role( + self, + role: str, + *, + name: str, + exact: bool, + ) -> _EmptyLocator: + del role + assert name == "待改价" + assert exact is True + return _EmptyLocator() + + def get_by_text( + self, + name: str, + *, + exact: bool, + ) -> _LocatorCollection: + assert name == "待改价" + assert exact is True + return self.candidates + + def wait_for_timeout(self, milliseconds: int) -> None: + assert milliseconds == 250 + self.wait_calls += 1 + if self.clock is not None: + self.clock.advance(milliseconds / 1000) + + +class _AlternateEntryPage: + def __init__(self) -> None: + self.control = _VisibleControl() + + def get_by_role( + self, + role: str, + *, + name: str, + exact: bool, + ) -> _VisibleControl | _EmptyLocator: + assert exact is True + if role == "button" and name == "发起申诉": + return self.control + return _EmptyLocator() + + def get_by_text(self, name: str, *, exact: bool) -> _EmptyLocator: + del name, exact + return _EmptyLocator() + + def wait_for_timeout(self, milliseconds: int) -> None: + del milliseconds + + +class _LateShopIdentityPage: + url = appeal.CAMPAIGN_SQUARE_URL + + def __init__(self) -> None: + self.wait_calls = 0 + + def goto(self, *args: object, **kwargs: object) -> None: + del args, kwargs + + def get_by_text(self, name: str, *, exact: bool) -> _EmptyLocator: + del name, exact + return _EmptyLocator() + + def wait_for_timeout(self, milliseconds: int) -> None: + del milliseconds + self.wait_calls += 1 + + +class _FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class _LatePendingCountPage: + def __init__( + self, + clock: _FakeClock, + values: tuple[int | None, ...], + ) -> None: + assert values + self.clock = clock + self.values = iter(values) + self.latest = values[-1] + self.wait_calls = 0 + + def evaluate(self, script: str) -> int | None: + assert "待办事项" in script + assert "待改价" in script + return next(self.values, self.latest) + + def wait_for_timeout(self, milliseconds: int) -> None: + assert milliseconds == 250 + self.wait_calls += 1 + self.clock.advance(milliseconds / 1000) + + +class _DynamicPanel: + def __init__(self, values: tuple[str, ...], *, action_count: int = 1) -> None: + assert values + self.values = iter(values) + self.latest = values[-1] + self.read_calls = 0 + self.actions = _LocatorCollection( + [_VisibleControl() for _ in range(action_count)] + ) + + def inner_text(self, timeout: int | None = None) -> str: + del timeout + self.read_calls += 1 + return next(self.values, self.latest) + + def get_by_role( + self, + role: str, + *, + name: str, + exact: bool, + ) -> _LocatorCollection | _EmptyLocator: + assert name == "立即改价" + assert exact is True + return self.actions if role == "button" else _EmptyLocator() + + def get_by_text(self, name: str, *, exact: bool) -> _EmptyLocator: + del name, exact + return _EmptyLocator() + + +class _PanelNode(_VisibleControl): + def __init__(self, name: str, box: dict[str, float]) -> None: + super().__init__() + self.name = name + self.box = box + + def bounding_box(self) -> dict[str, float]: + return self.box + + +class _DrawerTitleMarker: + def __init__(self, content_wrapper: _PanelNode) -> None: + self.content_wrapper = content_wrapper + self.selectors: list[str] = [] + + def locator(self, selector: str) -> _LocatorCollection | _EmptyLocator: + self.selectors.append(selector) + if selector == appeal._PANEL_ROLE_DIALOG_XPATH: # noqa: SLF001 + return _EmptyLocator() + if selector == appeal._PANEL_CONTENT_WRAPPER_XPATH: # noqa: SLF001 + return _LocatorCollection([self.content_wrapper]) + pytest.fail("generic fallback must not beat the explicit content wrapper") + + +class _ScanPage: + def __init__(self, clock: _FakeClock | None = None) -> None: + self.clock = clock + self.wait_calls = 0 + + def wait_for_timeout(self, milliseconds: int) -> None: + self.wait_calls += 1 + if self.clock is not None: + self.clock.advance(milliseconds / 1000) + + +class _NextControl(_VisibleControl): + def __init__(self, *, disabled: bool) -> None: + super().__init__() + self.disabled = disabled + + def is_disabled(self) -> bool: + return self.disabled + + def is_enabled(self) -> bool: + return not self.disabled + + def get_attribute(self, name: str) -> str | None: + assert name == "aria-disabled" + return "true" if self.disabled else None + + +class _PaginationPanel: + def __init__( + self, + text: str, + *, + active_pages: list[int], + next_control: _NextControl | None = None, + ) -> None: + self.text = text + self.active_pages = active_pages + self.next_control = next_control + + def inner_text(self, timeout: int | None = None) -> str: + del timeout + return self.text + + def evaluate(self, script: str) -> list[int]: + assert "aria-current" in script + return self.active_pages + + def get_by_role( + self, + role: str, + *, + name: str, + exact: bool, + ) -> _LocatorCollection | _EmptyLocator: + assert role == "button" + assert name == "下一页" + assert exact is True + if self.next_control is None: + return _EmptyLocator() + return _LocatorCollection([self.next_control]) + + def get_by_title(self, name: str) -> _EmptyLocator: + assert name == "下一页" + return _EmptyLocator() + + +class _ActionFingerprintLocator: + def __init__( + self, + *, + log_extra: str | None, + row_key: str | None, + action_text: str, + previous_text: str, + ) -> None: + self.payload = { + "log_extra": log_extra, + "row_key": row_key, + "action_text": action_text, + "previous_text": previous_text, + } + + def evaluate(self, script: str) -> dict[str, str | None]: + assert "data-log-extra" in script + assert "data-row-key" in script + assert "previousElementSibling" in script + return dict(self.payload) + + +def test_pending_count_parser_ignores_unrelated_numbers() -> None: + assert appeal.parse_pending_count("待改价\n10\n紧急") == 10 + assert appeal.parse_pending_count("待改价 0 报名异常 56") == 0 + assert appeal.parse_pending_count("报名异常 56") is None + + +def test_shop_identity_can_render_after_campaign_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + page = _LateShopIdentityPage() + body_texts = iter( + ( + "活动广场 待办事项 待改价 9", + "活动广场 待办事项 待改价 9", + "活动广场 待办事项 待改价 9", + "活动广场 待办事项 待改价 9 光影行星箱包旗舰店", + ) + ) + latest = "活动广场 待办事项 待改价 9 光影行星箱包旗舰店" + monkeypatch.setattr( + appeal, + "_body_text", + lambda page: next(body_texts, latest), + ) + + appeal._navigate_to_campaign_square( # noqa: SLF001 + page, + expected_shop_name=appeal.EXPECTED_SHOP_NAME, + ) + + assert page.wait_calls == 1 + + +def test_pending_count_can_render_after_campaign_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _LatePendingCountPage(clock, (None, None, 8)) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + + count = appeal._read_pending_count(page, timeout_seconds=2) # noqa: SLF001 + + assert count == 8 + assert page.wait_calls == 4 + + +def test_placeholder_zero_is_not_accepted_before_later_nonzero( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _LatePendingCountPage(clock, (0,) * 12 + (8,)) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + + count = appeal._read_pending_count(page, timeout_seconds=10) # noqa: SLF001 + + assert count == 8 + assert clock.now < appeal._PENDING_ZERO_STABILITY_SECONDS # noqa: SLF001 + + +def test_genuinely_stable_zero_is_accepted_after_five_seconds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _LatePendingCountPage(clock, (0,)) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + + count = appeal._read_pending_count(page, timeout_seconds=10) # noqa: SLF001 + + assert count == 0 + assert clock.now == appeal._PENDING_ZERO_STABILITY_SECONDS # noqa: SLF001 + + +def test_nested_pending_metric_labels_deduplicate_to_same_active_card( + monkeypatch: pytest.MonkeyPatch, +) -> None: + card = _PositionedControl( + box={"x": 100, "y": 40, "width": 80, "height": 40}, + depth=5, + metric_count=8, + ) + outer = _PositionedControl( + box={"x": 100, "y": 40, "width": 80, "height": 40}, + depth=6, + metric_count=8, + metric_depth=1, + parent=card, + ) + inner = _PositionedControl( + box={"x": 110, "y": 50, "width": 45, "height": 20}, + depth=7, + metric_count=8, + metric_depth=1, + parent=card, + ) + page = _PendingMetricPage([outer, inner]) + panel = object() + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert card.clicked is True + assert inner.clicked is False + assert outer.clicked is False + + +def test_off_region_and_wrong_count_candidates_are_excluded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + off_region = _PositionedControl( + box={"x": 300, "y": 200, "width": 45, "height": 20}, + depth=7, + ) + wrong_count = _PositionedControl( + box={"x": 500, "y": 40, "width": 45, "height": 20}, + depth=7, + metric_count=7, + ) + correct = _PositionedControl( + box={"x": 100, "y": 40, "width": 45, "height": 20}, + depth=7, + metric_count=8, + ) + page = _PendingMetricPage([off_region, wrong_count, correct]) + panel = object() + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert correct.clicked is True + assert off_region.clicked is False + assert wrong_count.clicked is False + + +def test_off_viewport_clone_is_excluded_before_candidate_dedupe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + correct = _PositionedControl( + box={"x": 628, "y": 276, "width": 131, "height": 16}, + depth=7, + metric_count=8, + ) + cloned = _PositionedControl( + box={"x": 1892, "y": 276, "width": 131, "height": 16}, + depth=7, + metric_count=8, + ) + page = _PendingMetricPage([correct, cloned]) + panel = object() + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert correct.clicked is True + assert cloned.clicked is False + + +def test_pending_metric_clicks_matching_parent_card_not_inner_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + card = _PositionedControl( + box={"x": 628, "y": 260, "width": 131, "height": 50}, + depth=7, + metric_count=8, + ) + inner_label = _PositionedControl( + box={"x": 628, "y": 276, "width": 131, "height": 16}, + depth=8, + metric_count=8, + metric_depth=1, + parent=card, + fail_click=True, + ) + page = _PendingMetricPage([inner_label]) + panel = object() + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert card.clicked is True + assert inner_label.clicked is False + + +def test_pointer_interception_requeries_then_uses_normal_click( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + target = _PositionedControl( + box={"x": 628, "y": 260, "width": 131, "height": 50}, + depth=7, + metric_count=8, + click_failures=1, + data_key="button_square_remind_price_change", + ) + page = _PendingMetricPage([target], clock=clock) + panel = object() + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert target.clicked is True + assert page.wait_calls == 1 + + +def test_hidden_covered_slide_is_requeried_until_active_and_hit_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + dot_buttons = [_VisibleControl() for _ in range(4)] + slider = _CarouselSlider([_CarouselDots(dot_buttons)]) + target = _PositionedControl( + box={"x": 580, "y": 276, "width": 131, "height": 50}, + depth=7, + metric_count=8, + data_key="button_square_remind_price_change", + active_states=((True, False), (False, True)), + carousel_activation={ + "kind": "carousel", + "sliderIndex": 0, + "dotsDomIndex": 0, + "dotCount": 4, + "targetIndex": 1, + "activeIndex": 0, + }, + slider=slider, + ) + page = _PendingMetricPage([target], clock=clock) + panel = object() + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr( + appeal, + "_wait_for_pending_panel_ready", + lambda page, **kwargs: panel, + ) + + result = appeal._open_pending_list( # noqa: SLF001 + page, + expected_count=8, + timeout_seconds=2, + ) + + assert result is panel + assert target.clicked is True + assert dot_buttons[1].clicked is True + assert sum(button.clicked for button in dot_buttons) == 1 + assert page.wait_calls == 1 + + +def test_pending_panel_waits_for_delayed_rows_and_stable_signature( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + card = _PositionedControl( + box={"x": 628, "y": 260, "width": 131, "height": 50}, + depth=7, + metric_count=8, + ) + page = _PendingMetricPage([card], clock=clock) + panel = _DynamicPanel( + ( + "待改价", + "待改价 报名信息 报名状态 操作", + "待改价 报名信息 报名状态 操作 商品 ID: 123456789012 " + "立即改价 51:29:49 动态提示 A", + "待改价 报名信息 报名状态 操作 商品 ID: 123456789012 " + "立即改价 51:29:48 动态提示 B", + "待改价 报名信息 报名状态 操作 商品 ID: 123456789012 " + "立即改价 51:29:47 动态提示 C", + "待改价 报名信息 报名状态 操作 商品 ID: 123456789012 " + "立即改价 51:29:46 动态提示 D", + ) + ) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr( + appeal, + "_find_active_panel", + lambda page, markers: panel, + ) + + result = appeal._open_pending_list(page, expected_count=8) # noqa: SLF001 + + assert result is panel + assert card.clicked is True + assert panel.read_calls == 6 + assert clock.now == 1.25 + + +def test_panel_marker_prefers_content_wrapper_over_drawer_title_and_header() -> None: + content_wrapper = _PanelNode( + "auxo-drawer-content-wrapper", + {"x": 320, "y": 0, "width": 960, "height": 720}, + ) + marker = _DrawerTitleMarker(content_wrapper) + + panel = appeal._panel_ancestor_for_marker(marker) # noqa: SLF001 + + assert panel is content_wrapper + assert marker.selectors == [ + appeal._PANEL_ROLE_DIALOG_XPATH, # noqa: SLF001 + appeal._PANEL_CONTENT_WRAPPER_XPATH, # noqa: SLF001 + ] + + +def test_item_panel_waits_for_async_product_and_sku_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _PendingMetricPage([], clock=clock) + expected_item_id = "3774363265803616739" + panel = _DynamicPanel( + ( + "商品议价", + "商品议价 SKU维度议价 加载中", + "商品议价 商品 ID: 3774363265803616739 " + "SKU维度议价 SKU规格 ID: 3619462493048834 原价 ￥839", + ), + action_count=0, + ) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr( + appeal, + "_find_active_panel", + lambda page, markers: panel, + ) + + result = appeal._wait_for_item_panel_ready( # noqa: SLF001 + page, + ("手动改价", "商品议价"), + expected_item_id=expected_item_id, + timeout_seconds=2, + ) + + assert result is panel + assert panel.read_calls == 3 + assert page.wait_calls == 2 + + +def test_pending_panel_with_headers_only_never_becomes_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _PendingMetricPage([], clock=clock) + panel = _DynamicPanel( + ("待改价 报名信息 报名状态 操作",), + action_count=0, + ) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr( + appeal, + "_find_active_panel", + lambda page, markers: panel, + ) + + with pytest.raises(appeal.PriceAppealError, match="未形成稳定"): + appeal._wait_for_pending_panel_ready( # noqa: SLF001 + page, + timeout_seconds=1, + ) + + assert clock.now == 1.0 + + +def test_spatially_distinct_matching_pending_metric_candidates_are_rejected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + first = _PositionedControl( + box={"x": 100, "y": 40, "width": 45, "height": 20}, + depth=7, + metric_count=8, + ) + second = _PositionedControl( + box={"x": 300, "y": 40, "width": 45, "height": 20}, + depth=7, + metric_count=8, + ) + page = _PendingMetricPage([first, second], clock=clock) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + + with pytest.raises(appeal.PriceAppealError, match="多个活动且可命中"): + appeal._open_pending_list( # noqa: SLF001 + page, + expected_count=8, + timeout_seconds=1, + ) + + assert first.clicked is False + assert second.clicked is False + + +def test_action_fingerprint_ignores_live_countdown_changes() -> None: + first = appeal.normalize_action_fingerprint( + "商品 A 报名成功 待改价 23:14:15 立即改价" + ) + second = appeal.normalize_action_fingerprint( + "商品 A 报名成功\n待改价 23:13:59 立即改价" + ) + + assert first == second + assert "" in first + + +def test_action_fingerprint_normalizes_countdown_adjacent_to_chinese() -> None: + first = appeal.normalize_action_fingerprint("报名异常 待改价49:50:27 立即改价") + second = appeal.normalize_action_fingerprint("报名异常 待改价49:49:58 立即改价") + + assert first == second + assert first == "报名异常 待改价 立即改价" + + +def test_action_fingerprint_prefers_button_log_product_and_record_ids() -> None: + locator = _ActionFingerprintLocator( + log_extra=json.dumps( + { + "sub_activity_id": "7554013743270347034", + "product_id": "3774363265803616739", + "record_id": "7637822630288326966", + "status": "item_predict", + "report_from": "price_change", + } + ), + row_key="7637822630288326966", + action_text="报名成功 待改价 23:14:15 立即改价", + previous_text="商品头行 ID: 9999999999999999999", + ) + + fingerprint = appeal._action_fingerprint(locator) # noqa: SLF001 + + assert appeal._extract_item_id(fingerprint) == "3774363265803616739" # noqa: SLF001 + assert "sub_activity_id: 7554013743270347034" in fingerprint + assert "record_id: 7637822630288326966" in fingerprint + assert "9999999999999999999" not in fingerprint + assert "" in fingerprint + + +def test_action_fingerprint_falls_back_to_previous_product_row_and_row_key() -> None: + locator = _ActionFingerprintLocator( + log_extra=json.dumps( + {"sub_activity_id": "7610636843016552714"} + ), + row_key="7617013918167138569", + action_text="活动中 报名异常 待改价 20:22:31 立即改价", + previous_text="GYXX/光影行星 商品 ID: 3712970670381072447", + ) + + fingerprint = appeal._action_fingerprint(locator) # noqa: SLF001 + + assert appeal._extract_item_id(fingerprint) == "3712970670381072447" # noqa: SLF001 + assert "sub_activity_id: 7610636843016552714" in fingerprint + assert "data-row-key: 7617013918167138569" in fingerprint + assert "" in fingerprint + + +def test_same_product_in_multiple_activities_has_distinct_fingerprints() -> None: + product_id = "3774363265803616739" + first = _ActionFingerprintLocator( + log_extra=json.dumps( + { + "sub_activity_id": "7554013743270347034", + "product_id": product_id, + "record_id": "7637822630288326966", + } + ), + row_key="7637822630288326966", + action_text="报名成功 待改价 23:14:15 立即改价", + previous_text="", + ) + second = _ActionFingerprintLocator( + log_extra=json.dumps( + { + "sub_activity_id": "7610636843016552714", + "product_id": product_id, + "record_id": "7617013918167138569", + } + ), + row_key="7617013918167138569", + action_text="报名成功 待改价 23:13:59 立即改价", + previous_text="", + ) + + fingerprints = { + appeal._action_fingerprint(first), # noqa: SLF001 + appeal._action_fingerprint(second), # noqa: SLF001 + } + + assert len(fingerprints) == 2 + assert all(f"ID: {product_id}" in value for value in fingerprints) + + +@pytest.mark.parametrize( + ("log_extra", "row_key", "previous_text", "error"), + ( + ("{not-json", "7617013918167138569", "ID: 3712970670381072447", "JSON"), + ( + json.dumps( + { + "sub_activity_id": "7610636843016552714", + "record_id": "7617013918167138569", + } + ), + "7617013918167138569", + "没有商品编号", + "商品 ID", + ), + ( + json.dumps( + { + "product_id": "3712970670381072447", + "record_id": "7617013918167138569", + } + ), + "7617013918167138569", + "", + "sub_activity_id", + ), + ( + json.dumps( + { + "sub_activity_id": "7610636843016552714", + "product_id": "3712970670381072447", + } + ), + None, + "", + "data-row-key", + ), + ), +) +def test_action_fingerprint_fails_closed_on_incomplete_identity( + log_extra: str, + row_key: str | None, + previous_text: str, + error: str, +) -> None: + locator = _ActionFingerprintLocator( + log_extra=log_extra, + row_key=row_key, + action_text="待改价 20:22:31 立即改价", + previous_text=previous_text, + ) + + with pytest.raises(appeal.PriceAppealError, match=error): + appeal._action_fingerprint(locator) # noqa: SLF001 + + +def test_pending_scan_resets_to_top_and_finds_virtualized_bottom_action( + monkeypatch: pytest.MonkeyPatch, +) -> None: + page = _ScanPage() + position = 60.0 + targets: list[float] = [] + top_action = object() + bottom_action = object() + + def snapshot( + panel: object, + *, + target_top: float | None = None, + ) -> appeal._PendingScrollSnapshot: # noqa: SLF001 + nonlocal position + del panel + if target_top is not None: + targets.append(target_top) + position = max(0.0, min(100.0, target_top)) + return appeal._PendingScrollSnapshot( # noqa: SLF001 + "virtual-list", + position, + 200.0, + 100.0, + ) + + def candidates(panel: object) -> list[tuple[object, str]]: + del panel + if position < 80: + return [(top_action, "top")] + return [(bottom_action, "bottom")] + + monkeypatch.setattr(appeal, "_pending_scroll_snapshot", snapshot) + monkeypatch.setattr(appeal, "_pending_action_candidates", candidates) + monkeypatch.setattr( + appeal, + "_pending_content_signature", + lambda panel: f"position:{position}", + ) + monkeypatch.setattr( + appeal, + "_wait_for_pending_content_signature", + lambda page, panel: f"position:{position}", + ) + + result = appeal._next_unprocessed_action( # noqa: SLF001 + page, + object(), + {"top"}, + ) + + assert result == (bottom_action, "bottom") + assert targets[0] == 0.0 + assert targets[-1] == 80.0 + + +def test_pending_scan_requires_two_stable_bottom_reads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + page = _ScanPage() + reads = 0 + + def candidates(panel: object) -> list[tuple[object, str]]: + nonlocal reads + del panel + reads += 1 + return [] + + monkeypatch.setattr( + appeal, + "_pending_scroll_snapshot", + lambda panel, target_top=None: appeal._PendingScrollSnapshot( # noqa: SLF001 + None, + 0.0, + 100.0, + 100.0, + ), + ) + monkeypatch.setattr(appeal, "_pending_action_candidates", candidates) + monkeypatch.setattr( + appeal, + "_wait_for_pending_content_signature", + lambda page, panel: "only-view-details", + ) + + assert appeal._next_unprocessed_action( # noqa: SLF001 + page, + object(), + set(), + ) is None + assert reads == 2 + assert page.wait_calls == 1 + + +def test_pending_scan_fails_closed_when_non_bottom_scroll_makes_no_progress( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _ScanPage(clock) + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr( + appeal, + "_pending_scroll_snapshot", + lambda panel, target_top=None: appeal._PendingScrollSnapshot( # noqa: SLF001 + "stuck-list", + 0.0, + 200.0, + 100.0, + ), + ) + monkeypatch.setattr(appeal, "_pending_action_candidates", lambda panel: []) + monkeypatch.setattr( + appeal, + "_pending_content_signature", + lambda panel: "same-rows", + ) + monkeypatch.setattr( + appeal, + "_wait_for_pending_content_signature", + lambda page, panel: "same-rows", + ) + + with pytest.raises(appeal.PriceAppealError, match="没有任何进展"): + appeal._next_unprocessed_action( # noqa: SLF001 + page, + object(), + set(), + ) + + +def test_pending_action_candidates_union_role_and_text_and_skip_view_details( + monkeypatch: pytest.MonkeyPatch, +) -> None: + role_action = object() + text_action = object() + query_names: list[str] = [] + + class Panel: + def get_by_role(self, role: str, *, name: str, exact: bool) -> str: + query_names.append(name) + assert exact is True + return f"role:{role}" + + def get_by_text(self, name: str, *, exact: bool) -> str: + query_names.append(name) + assert exact is True + return "text" + + def visible(locator: str, *, label: str) -> list[object]: + del label + if locator == "role:button": + return [role_action] + if locator == "text": + return [text_action] + return [] + + monkeypatch.setattr(appeal, "_visible_items_strict", visible) + monkeypatch.setattr(appeal, "_clickable_action_target", lambda value: value) + monkeypatch.setattr( + appeal, + "_action_dom_key", + lambda value: "role" if value is role_action else "text", + ) + monkeypatch.setattr( + appeal, + "_action_fingerprint", + lambda value: ( + "商品 ID: 123456789012 立即改价" + if value is role_action + else "商品 ID: 987654321098 立即改价" + ), + ) + + result = appeal._pending_action_candidates(Panel()) # noqa: SLF001 + + assert [item[0] for item in result] == [role_action, text_action] + assert query_names == ["立即改价", "立即改价", "立即改价"] + assert "查看详情" not in query_names + + +def test_same_fingerprint_for_distinct_actions_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = object() + second = object() + + class Panel: + def get_by_role(self, role: str, *, name: str, exact: bool) -> str: + del name, exact + return role + + def get_by_text(self, name: str, *, exact: bool) -> str: + del name, exact + return "text" + + def visible(locator: str, *, label: str) -> list[object]: + del label + if locator == "button": + return [first, second] + return [] + + monkeypatch.setattr(appeal, "_visible_items_strict", visible) + monkeypatch.setattr(appeal, "_action_dom_key", lambda value: str(id(value))) + monkeypatch.setattr( + appeal, + "_action_fingerprint", + lambda value: "商品 ID: 123456789012 立即改价", + ) + + with pytest.raises(appeal.PriceAppealError, match="多个无法区分"): + appeal._pending_action_candidates(Panel()) # noqa: SLF001 + + +def test_pagination_parses_single_page_eight_of_ten_without_advancing() -> None: + panel = _PaginationPanel( + "待改价 报名信息 报名状态 操作 共8条 1 10条/页", + active_pages=[1], + ) + + pagination = appeal._read_pending_pagination(panel) # noqa: SLF001 + + assert pagination.total_items == 8 + assert pagination.page_size == 10 + assert pagination.active_page == 1 + assert pagination.has_next is False + assert appeal._advance_pending_page(panel, pagination) is False # noqa: SLF001 + + +def test_pagination_requires_next_control_when_more_rows_exist() -> None: + panel = _PaginationPanel( + "待改价 报名信息 报名状态 操作 共18条 1 10条/页", + active_pages=[1], + ) + pagination = appeal._read_pending_pagination(panel) # noqa: SLF001 + + with pytest.raises(appeal.PriceAppealError, match="未找到.*下一页"): + appeal._advance_pending_page(panel, pagination) # noqa: SLF001 + + +def test_page_change_waits_for_next_page_and_changed_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _FakeClock() + page = _ScanPage(clock) + panel = object() + states = iter(((1, "old"), (1, "old"), (2, "new"))) + current_signature = "old" + + def pagination(current_panel: object) -> appeal._PendingPagination: # noqa: SLF001 + nonlocal current_signature + assert current_panel is panel + active_page, current_signature = next(states) + return appeal._PendingPagination(18, 10, active_page) # noqa: SLF001 + + monkeypatch.setattr(appeal.time, "monotonic", clock.monotonic) + monkeypatch.setattr(appeal, "_page_requires_login", lambda page: False) + monkeypatch.setattr(appeal, "_find_active_panel", lambda page, markers: panel) + monkeypatch.setattr(appeal, "_read_pending_pagination", pagination) + monkeypatch.setattr( + appeal, + "_pending_content_signature", + lambda current_panel: current_signature, + ) + + result = appeal._wait_for_pending_page_change( # noqa: SLF001 + page, + before=appeal._PendingPagination(18, 10, 1), # noqa: SLF001 + before_signature="old", + timeout_seconds=1, + ) + + assert result is panel + assert page.wait_calls == 2 + + +def test_current_page_actions_are_returned_before_next_page( + monkeypatch: pytest.MonkeyPatch, +) -> None: + page = _ScanPage() + page_one = object() + page_two = object() + first = object() + second = object() + third = object() + advances: list[int] = [] + + def next_action( + current_page: _ScanPage, + panel: object, + processed: set[str], + ) -> tuple[object, str] | None: + assert current_page is page + if panel is page_one: + if "first" not in processed: + return first, "first" + if "second" not in processed: + return second, "second" + return None + return third, "third" + + monkeypatch.setattr(appeal, "_next_unprocessed_action", next_action) + monkeypatch.setattr( + appeal, + "_read_pending_pagination", + lambda panel: appeal._PendingPagination(18, 10, 1), # noqa: SLF001 + ) + monkeypatch.setattr(appeal, "_pending_content_signature", lambda panel: "page1") + monkeypatch.setattr( + appeal, + "_advance_pending_page", + lambda panel, pagination: advances.append(pagination.active_page) or True, + ) + monkeypatch.setattr( + appeal, + "_wait_for_pending_page_change", + lambda page, **kwargs: page_two, + ) + + assert appeal._next_action_across_pages( # noqa: SLF001 + page, + page_one, + set(), + ) == (first, "first") + assert advances == [] + assert appeal._next_action_across_pages( # noqa: SLF001 + page, + page_one, + {"first"}, + ) == (second, "second") + assert advances == [] + assert appeal._next_action_across_pages( # noqa: SLF001 + page, + page_one, + {"first", "second"}, + ) == (third, "third") + assert advances == [1] + + +def test_alternate_appeal_entry_is_supported() -> None: + page = _AlternateEntryPage() + + selected = appeal._wait_and_click_named_control( # noqa: SLF001 + page, + appeal.APPEAL_ENTRY_LABELS, + timeout_seconds=1, + ) + + assert selected == "发起申诉" + assert page.control.clicked is True + + +def test_zero_pending_items_finishes_without_clicking( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + screenshots: list[Path] = [] + monkeypatch.setattr(appeal, "_evidence_dir", lambda: tmp_path) + monkeypatch.setattr( + appeal, + "_navigate_to_campaign_square", + lambda page, **kwargs: None, + ) + monkeypatch.setattr(appeal, "_read_pending_count", lambda page: 0) + monkeypatch.setattr( + appeal, + "_open_pending_list", + lambda page, **kwargs: pytest.fail( + "zero pending items must not open the list" + ), + ) + monkeypatch.setattr( + appeal, + "_capture_screenshot", + lambda page, path: screenshots.append(path), + ) + + summary = appeal.run_price_appeals(object()) + + assert summary.status == "no_pending_items" + assert summary.initial_pending_count == 0 + assert summary.submitted_items == 0 + assert screenshots == [tmp_path / "no-pending.png"] + payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) + assert payload["status"] == "no_pending_items" + + +def test_all_items_support_both_entry_variants_and_write_redacted_evidence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + counts = iter((2, 1, 0)) + fingerprints = ( + "商品 A 商品 ID: 123456789012 立即改价", + "商品 B 商品 ID: 987654321098 立即改价", + ) + actions = iter(((object(), fingerprints[0]), (object(), fingerprints[1]))) + entries = iter(("申诉", "发起申诉")) + clicks: list[str] = [] + + monkeypatch.setattr(appeal, "_evidence_dir", lambda: tmp_path) + monkeypatch.setattr( + appeal, + "_navigate_to_campaign_square", + lambda page, **kwargs: None, + ) + monkeypatch.setattr(appeal, "_read_pending_count", lambda page: next(counts)) + monkeypatch.setattr(appeal, "_open_pending_list", lambda page, **kwargs: None) + monkeypatch.setattr( + appeal, + "_next_action_across_pages", + lambda page, panel, processed: next(actions), + ) + monkeypatch.setattr( + appeal, + "_wait_for_item_panel_ready", + lambda page, markers, **kwargs: object(), + ) + monkeypatch.setattr(appeal, "_is_registration_edit_panel", lambda panel: False) + monkeypatch.setattr(appeal, "_verify_item_identity", lambda panel, item_id: None) + monkeypatch.setattr( + appeal, + "_click_locator", + lambda locator, *, label: clicks.append(label), + ) + monkeypatch.setattr( + appeal, + "_wait_and_click_named_control", + lambda page, names, **kwargs: next(entries), + ) + monkeypatch.setattr( + appeal, + "_complete_appeal_form", + lambda page, **kwargs: None, + ) + monkeypatch.setattr(appeal, "_capture_screenshot", lambda page, path: None) + + summary = appeal.run_price_appeals(object()) + + assert summary.status == "completed" + assert summary.submitted_items == 2 + assert summary.entry_variants == ["申诉", "发起申诉"] + assert summary.submitted_item_refs == [ + hashlib.sha256(value.encode("utf-8")).hexdigest()[:12] + for value in fingerprints + ] + assert clicks == ["立即改价", "立即改价"] + evidence = (tmp_path / "summary.json").read_text(encoding="utf-8") + assert "123456789012" not in evidence + assert "987654321098" not in evidence + + +@pytest.mark.parametrize("actionable_count", (2, 11)) +def test_dashboard_count_does_not_limit_actual_actionable_rows( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + actionable_count: int, +) -> None: + actions = iter( + [ + (object(), f"商品 {index} 商品 ID: {123456789000 + index} 立即改价") + for index in range(actionable_count) + ] + + [None] + ) + clicks: list[str] = [] + list_open_count = 0 + + def open_pending_list(page: object, *, expected_count: int) -> None: + nonlocal list_open_count + del page + assert expected_count == 9 + list_open_count += 1 + + monkeypatch.setattr(appeal, "_evidence_dir", lambda: tmp_path) + monkeypatch.setattr( + appeal, + "_navigate_to_campaign_square", + lambda page, **kwargs: None, + ) + monkeypatch.setattr(appeal, "_read_pending_count", lambda page: 9) + monkeypatch.setattr(appeal, "_open_pending_list", open_pending_list) + monkeypatch.setattr( + appeal, + "_next_action_across_pages", + lambda page, panel, processed: next(actions), + ) + monkeypatch.setattr( + appeal, + "_wait_for_item_panel_ready", + lambda page, markers, **kwargs: object(), + ) + monkeypatch.setattr(appeal, "_is_registration_edit_panel", lambda panel: False) + monkeypatch.setattr(appeal, "_verify_item_identity", lambda panel, item_id: None) + monkeypatch.setattr( + appeal, + "_click_locator", + lambda locator, *, label: clicks.append(label), + ) + monkeypatch.setattr( + appeal, + "_wait_and_click_named_control", + lambda page, names, **kwargs: "申诉", + ) + monkeypatch.setattr(appeal, "_complete_appeal_form", lambda page, **kwargs: None) + monkeypatch.setattr(appeal, "_capture_screenshot", lambda page, path: None) + + summary = appeal.run_price_appeals(object()) + + assert summary.initial_pending_count == 9 + assert summary.actionable_items == actionable_count + assert summary.submitted_items == actionable_count + assert summary.status == "completed_with_unactionable_items" + assert clicks == ["立即改价"] * actionable_count + assert list_open_count == actionable_count + 1 + + +def test_registration_edit_without_appeal_is_recorded_and_skipped( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + fingerprint = "商品 ID: 3712970670381072447 立即改价" + actions = iter(((object(), fingerprint), None)) + screenshots: list[Path] = [] + navigations = 0 + + def navigate(page: object, **kwargs: object) -> None: + nonlocal navigations + del page, kwargs + navigations += 1 + + monkeypatch.setattr(appeal, "_evidence_dir", lambda: tmp_path) + monkeypatch.setattr(appeal, "_navigate_to_campaign_square", navigate) + monkeypatch.setattr(appeal, "_read_pending_count", lambda page: 7) + monkeypatch.setattr(appeal, "_open_pending_list", lambda page, **kwargs: None) + monkeypatch.setattr( + appeal, + "_next_action_across_pages", + lambda page, panel, processed: next(actions), + ) + monkeypatch.setattr( + appeal, + "_wait_for_item_panel_ready", + lambda page, markers, **kwargs: object(), + ) + monkeypatch.setattr(appeal, "_verify_item_identity", lambda panel, item_id: None) + monkeypatch.setattr(appeal, "_is_registration_edit_panel", lambda panel: True) + monkeypatch.setattr(appeal, "_find_named_control", lambda *args, **kwargs: None) + monkeypatch.setattr(appeal, "_click_locator", lambda locator, **kwargs: None) + monkeypatch.setattr( + appeal, + "_wait_and_click_named_control", + lambda *args, **kwargs: pytest.fail("registration edit must not appeal"), + ) + monkeypatch.setattr( + appeal, + "_complete_appeal_form", + lambda *args, **kwargs: pytest.fail("registration edit must not submit"), + ) + monkeypatch.setattr( + appeal, + "_capture_screenshot", + lambda page, path: screenshots.append(path), + ) + + summary = appeal.run_price_appeals(object()) + + item_ref = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()[:12] + assert summary.status == "completed_with_unactionable_items" + assert summary.actionable_items == 1 + assert summary.submitted_items == 0 + assert summary.skipped_non_appealable_items == 1 + assert summary.skipped_non_appealable_item_refs == [item_ref] + assert navigations == 2 + assert screenshots == [tmp_path / "skipped-no-appeal-001.png"] + + +def test_price_appeal_mode_requires_internal_execute_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert collector._parse_price_appeal_mode(["--price-appeal"]) == (100, False) + assert collector._parse_price_appeal_mode( # noqa: SLF001 + ["--price-appeal", "--execute", "--max-items", "8"] + ) == (8, True) + monkeypatch.setattr( + collector.DynamicFetcher, + "fetch", + lambda *args, **kwargs: pytest.fail("safe preview must not open a browser"), + ) + + assert collector._run_price_appeal_mode(max_items=8, execute=False) == 0 + + +def test_callback_failure_is_never_retried_after_possible_external_effect( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls = 0 + + def fail_after_callback(*args: object, **kwargs: object) -> None: + nonlocal calls + del args, kwargs + calls += 1 + collector._automation_error = None + collector._price_appeal_summary = appeal.PriceAppealSummary( + status="completed" + ) + raise RuntimeError("browser wrapper failure") + + monkeypatch.setenv("GYXX_COMMAND_ID", collector.PRICE_APPEAL_COMMAND_ID) + monkeypatch.setenv("GYXX_RUN_ID", "test-run") + monkeypatch.setattr(collector, "USER_DATA_DIR", str(tmp_path / "profile")) + monkeypatch.setattr(collector, "enable_adaptive_fetchers", lambda: None) + monkeypatch.setattr(collector, "_load_cookies", lambda: [{"name": "session"}]) + monkeypatch.setattr(collector.DynamicFetcher, "fetch", fail_after_callback) + + assert collector._run_price_appeal_mode(max_items=1, execute=True) == 1 + assert calls == 1 + + +def test_unmanaged_execute_is_rejected_before_browser_launch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_COMMAND_ID", raising=False) + monkeypatch.delenv("GYXX_RUN_ID", raising=False) + monkeypatch.setattr( + collector.DynamicFetcher, + "fetch", + lambda *args, **kwargs: pytest.fail("unmanaged run must not launch Chrome"), + ) + + assert collector._run_price_appeal_mode(max_items=1, execute=True) == 1 + + +class _SubmissionStatusPanel: + def __init__(self, rows: dict[str, str]) -> None: + self.rows = rows + + def evaluate(self, script: str, expected_item_id: str) -> dict[str, int]: + assert "议价申诉中" in script + assert "待审核" in script + matching = [ + status + for item_id, status in self.rows.items() + if item_id == expected_item_id + ] + return { + "matched_rows": len(matching), + "success_rows": sum( + "议价申诉中" in status and "待审核" in status + for status in matching + ), + } + + +def test_exact_item_pending_appeal_status_counts_as_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + panel = _SubmissionStatusPanel( + {"3774363265803616739": "议价申诉中，待审核"} + ) + monkeypatch.setattr(appeal, "_find_active_panel", lambda page, markers: panel) + monkeypatch.setattr(appeal, "_visible_feedback_texts", lambda page: ()) + + appeal._wait_for_submission_result( # noqa: SLF001 + object(), + expected_item_id="3774363265803616739", + timeout_seconds=1, + ) + + +def test_other_item_pending_appeal_status_is_not_target_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + panel = _SubmissionStatusPanel( + {"3753423549986111710": "议价申诉中，待审核"} + ) + monkeypatch.setattr(appeal, "_find_active_panel", lambda page, markers: panel) + + assert appeal._item_row_has_pending_appeal_status( # noqa: SLF001 + object(), + expected_item_id="3774363265803616739", + ) is False + + +def test_target_item_without_pending_appeal_status_is_not_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + panel = _SubmissionStatusPanel( + {"3774363265803616739": "报名异常 立即改价"} + ) + monkeypatch.setattr(appeal, "_find_active_panel", lambda page, markers: panel) + + assert appeal._item_row_has_pending_appeal_status( # noqa: SLF001 + object(), + expected_item_id="3774363265803616739", + ) is False + + +def test_unknown_submission_result_never_counts_as_success() -> None: + with pytest.raises(appeal.PriceAppealError, match="结果不确定"): + appeal._wait_for_submission_result( # noqa: SLF001 + object(), + expected_item_id="3774363265803616739", + timeout_seconds=0, + ) + + +def test_item_identity_is_required_for_destructive_action() -> None: + assert appeal._extract_item_id("商品 ID: 3730219261105406099") == ( # noqa: SLF001 + "3730219261105406099" + ) + with pytest.raises(appeal.PriceAppealError, match="商品 ID"): + appeal._extract_item_id("只有商品名称，没有稳定标识") # noqa: SLF001 + + +def test_weekly_and_manual_paths_share_the_same_browser_state_lock( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(collector, "DATA_ROOT", tmp_path) + monkeypatch.setenv( + "GYXX_BROWSER_STATE_KEY", + "shop_intelligence:collectors/dy_store_competitor_store_scraping.py", + ) + monkeypatch.setenv("GYXX_BROWSER_LOCK_TIMEOUT_SECONDS", "0") + monkeypatch.setenv("GYXX_RUN_ID", "weekly-run") + weekly_lock = collector._browser_state_lock() # noqa: SLF001 + monkeypatch.setenv("GYXX_RUN_ID", "manual-run") + manual_lock = collector._browser_state_lock() # noqa: SLF001 + + assert weekly_lock.path == manual_lock.path + with weekly_lock: + with pytest.raises(collector.ResourceBusyError, match="resource is busy"): + with manual_lock: + pytest.fail("the same browser state must never be leased twice") diff --git a/tests/modules/shop_intelligence/test_dy_market_category_selection.py b/tests/modules/shop_intelligence/test_dy_market_category_selection.py new file mode 100644 index 0000000..78b29f5 --- /dev/null +++ b/tests/modules/shop_intelligence/test_dy_market_category_selection.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +from datetime import date +from types import SimpleNamespace + +import pytest + +from gyxx_flow.modules.shop_intelligence.collectors import ( + dy_store_competitor_store_scraping as collector, +) + + +class _Locator: + def __init__(self, *, count: int = 1) -> None: + self._count = count + self.clicked = False + self.waited = False + + @property + def first(self): + return self + + def count(self) -> int: + return self._count + + def click(self, **_kwargs) -> None: + self.clicked = True + + def wait_for(self, **_kwargs) -> None: + self.waited = True + + +def test_market_category_controls_are_bound_to_visible_cascader() -> None: + selectors: list[str] = [] + closed_menu = _Locator(count=0) + trigger = _Locator() + opened_menu = _Locator() + + class Page: + def locator(self, selector: str): + selectors.append(selector) + if selector == ".aurora-cascader-menu:visible": + return closed_menu if selectors.count(selector) == 1 else opened_menu + if selector == ".aurora-form-item-control .aurora-cascader:visible": + return trigger + raise AssertionError(selector) + + assert collector._click_market_category_dropdown(Page()) + assert trigger.clicked + assert opened_menu.waited + + +def test_market_category_option_uses_visible_titled_menu_item() -> None: + located: list[str] = [] + option = _Locator() + page = SimpleNamespace( + locator=lambda selector: located.append(selector) or option, + ) + + assert collector._click_market_category_option(page, "双肩包") + assert located == [ + '.aurora-cascader-menu ' + 'li.aurora-cascader-menu-item[role="menuitemcheckbox"]' + '[title="双肩包"]:visible' + ] + assert option.waited + assert option.clicked + + +def test_select_market_category_requires_path_and_fresh_table(monkeypatch) -> None: + expected = collector._expected_market_category_path("双肩包") + paths = iter([collector._expected_market_category_path("腰包/胸包"), expected]) + signatures = iter(["old rows", "new rows"]) + clicked: list[str] = [] + refresh_checks: list[tuple[str, str]] = [] + page = SimpleNamespace( + evaluate=lambda *_args, **_kwargs: None, + wait_for_timeout=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setattr( + collector, + "_selected_market_category_path", + lambda _page: next(paths), + ) + monkeypatch.setattr( + collector, + "_market_table_signature", + lambda _page: next(signatures), + ) + monkeypatch.setattr( + collector, + "_click_market_category_dropdown", + lambda _page: True, + ) + monkeypatch.setattr( + collector, + "_click_market_category_option", + lambda _page, label: clicked.append(label) or True, + ) + monkeypatch.setattr( + collector, + "_wait_for_market_category_refresh", + lambda _page, *, expected_path, previous_signature: refresh_checks.append( + (expected_path, previous_signature) + ), + ) + monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None) + + collector._select_market_category(page, "双肩包") + + assert clicked == ["鞋靴箱包", "箱包", "时尚箱包", "双肩包"] + assert refresh_checks == [(expected, "old rows")] + + +def test_select_market_category_rejects_stale_table(monkeypatch) -> None: + expected = collector._expected_market_category_path("双肩包") + paths = iter([collector._expected_market_category_path("腰包/胸包"), expected]) + page = SimpleNamespace( + evaluate=lambda *_args, **_kwargs: None, + wait_for_timeout=lambda *_args, **_kwargs: None, + ) + + monkeypatch.setattr( + collector, + "_selected_market_category_path", + lambda _page: next(paths), + ) + monkeypatch.setattr( + collector, + "_market_table_signature", + lambda _page: "same rows", + ) + monkeypatch.setattr( + collector, + "_click_market_category_dropdown", + lambda _page: True, + ) + monkeypatch.setattr( + collector, + "_click_market_category_option", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + collector, + "_wait_for_market_category_refresh", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr(collector, "_wait_after_action", lambda *_args, **_kwargs: None) + + with pytest.raises( + collector.MarketCategorySelectionError, + match="still contains stale rows", + ): + collector._select_market_category(page, "双肩包") + + +def test_market_ranking_collection_never_extracts_after_selection_failure( + monkeypatch, +) -> None: + extracted: list[str] = [] + monkeypatch.setattr( + collector, + "_previous_week_range", + lambda: (date(2026, 7, 27), date(2026, 8, 2)), + ) + monkeypatch.setattr( + collector, + "_open_market_rank_shop_page", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + collector, + "_select_market_previous_week", + lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + collector, + "_select_market_category", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + collector.MarketCategorySelectionError("category did not change") + ), + ) + monkeypatch.setattr( + collector, + "_extract_market_ranking", + lambda _page, category: extracted.append(category), + ) + + with pytest.raises( + collector.MarketCategorySelectionError, + match="category did not change", + ): + collector._collect_market_rankings(object()) + + assert extracted == [] diff --git a/tests/modules/shop_intelligence/test_feishu_writer_idempotency.py b/tests/modules/shop_intelligence/test_feishu_writer_idempotency.py new file mode 100644 index 0000000..abb509f --- /dev/null +++ b/tests/modules/shop_intelligence/test_feishu_writer_idempotency.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import json + +import pytest + +from gyxx_flow.modules.shop_intelligence import lark_cli +from gyxx_flow.modules.shop_intelligence.config import ( + DY_PEER_BASE_FIELDS, + DY_PEER_CAT_MAP, + JD_PEER_BASE_FIELDS, + JD_PEER_BRAND_MAP, + JD_PEER_CATEGORY_MAP, +) +from gyxx_flow.modules.shop_intelligence.writers import peer_store_writer + + +def test_shop_idempotency_reads_all_pages_and_normalizes_month( + monkeypatch: pytest.MonkeyPatch, +) -> None: + offsets: list[int] = [] + + def fake_run(_node, _script, args, **_kwargs): + payload = json.loads(args[-1]) + offsets.append(payload["offset"]) + if payload["offset"] == 0: + return { + "data": { + "fields": ["time", "platform", "month"], + "data": [["7月第5周（7.27-8.02）", ["JD"], ["2026.7月"]]], + "record_id_list": ["other-platform"], + "has_more": True, + } + } + return { + "data": { + "fields": ["time", "platform", "month"], + "data": [["7月第5周（7.27-8.02）", ["DY"], ["2026.7月"]]], + "record_id_list": ["target"], + "has_more": False, + } + } + + monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run) + + record_id = lark_cli.find_existing_record( + "node", + "script", + "base", + "table", + {"时间": "time", "平台": "platform", "月份": "month"}, + "7月第5周（7.27-8.2）", + "DY", + month_str="2026.07月", + ) + + assert record_id == "target" + assert offsets == [0, 1] + + +def test_shop_idempotency_fails_closed_when_identity_index_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + lark_cli, + "run_lark_cli", + lambda *_args, **_kwargs: { + "data": { + "fields": ["time", "month"], + "data": [], + "record_id_list": [], + "has_more": False, + } + }, + ) + + with pytest.raises(RuntimeError, match="平台"): + lark_cli.find_existing_record( + "node", + "script", + "base", + "table", + {"时间": "time", "平台": "platform", "月份": "month"}, + "7月第5周（7.27-8.2）", + "DY", + month_str="2026.07月", + ) + + +def test_record_search_pagination_has_a_hard_page_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(lark_cli, "_MAX_RECORD_SEARCH_PAGES", 1) + monkeypatch.setattr( + lark_cli, + "run_lark_cli", + lambda *_args, **_kwargs: { + "data": { + "fields": ["time"], + "data": [["7月第5周（7.27-8.2）"]], + "record_id_list": ["record-1"], + "has_more": True, + } + }, + ) + + with pytest.raises(RuntimeError, match="安全上限"): + lark_cli.search_record_pages( + "node", + "script", + "base", + "table", + { + "keyword": "7月第5周", + "search_fields": ["time"], + "select_fields": ["time"], + "limit": 1, + }, + ) + + +def test_peer_idempotency_reads_all_pages_and_requires_identity_indexes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + offsets: list[int] = [] + + def fake_run(_node, _script, args, **_kwargs): + payload = json.loads(args[-1]) + offsets.append(payload["offset"]) + if payload["offset"] == 0: + return { + "data": { + "fields": ["time", "brand", "month"], + "data": [["7月第5周（7.27-8.02）", ["other"], ["2026.7月"]]], + "record_id_list": ["other-brand"], + "has_more": True, + } + } + return { + "data": { + "fields": ["time", "brand", "month"], + "data": [["7月第5周（7.27-8.02）", ["NIID"], ["2026.7月"]]], + "record_id_list": ["target"], + "has_more": False, + } + } + + monkeypatch.setattr(lark_cli, "run_lark_cli", fake_run) + + record_id = peer_store_writer._find_existing_peer_record( + "node", + "script", + "table", + {"时间": "time", "竞店名称": "brand", "月份": "month"}, + "7月第5周（7.27-8.2）", + "NIID", + "2026.07月", + ) + + assert record_id == "target" + assert offsets == [0, 1] + + monkeypatch.setattr( + lark_cli, + "run_lark_cli", + lambda *_args, **_kwargs: { + "data": { + "fields": ["time", "month"], + "data": [], + "record_id_list": [], + "has_more": False, + } + }, + ) + with pytest.raises(RuntimeError, match="竞店名称"): + peer_store_writer._find_existing_peer_record( + "node", + "script", + "table", + {"时间": "time", "竞店名称": "brand", "月份": "month"}, + "7月第5周（7.27-8.2）", + "NIID", + "2026.07月", + ) + + +def test_jd_rerun_explicitly_clears_only_na_category( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + brand_json_name = next(iter(JD_PEER_BRAND_MAP)) + category_names = list(JD_PEER_CATEGORY_MAP) + artifact = tmp_path / "jd-peer.json" + artifact.write_text( + json.dumps( + { + "数据周期": "2026-07-27 ~ 2026-08-02", + "数据": { + brand_json_name: { + category_names[0]: {"rank": "N/A"}, + category_names[1]: {"rank": "7"}, + category_names[2]: {"rank": "3"}, + } + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + writes: list[dict] = [] + monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, "")) + monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script")) + monkeypatch.setattr( + peer_store_writer, + "_build_month_options_cache", + lambda *_args: {"2026.07月": "2026.07月"}, + ) + monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing") + monkeypatch.setattr( + peer_store_writer, + "_upsert_peer_record", + lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}}, + ) + + assert peer_store_writer._write_jd_peer_store(str(artifact)) is True + assert len(writes) == 1 + assert writes[0][JD_PEER_CATEGORY_MAP[category_names[0]]] is None + assert writes[0][JD_PEER_CATEGORY_MAP[category_names[1]]] == 7 + assert writes[0][JD_PEER_CATEGORY_MAP[category_names[2]]] == 3 + + +def test_dy_brand_iteration_uses_both_category_brand_sets( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + artifact = tmp_path / "dy-peer.json" + artifact.write_text( + json.dumps( + { + "date_range": "2026/07/27 - 2026/08/02", + "results": { + DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}}, + DY_PEER_CAT_MAP["双肩包"]: {"beta": {"rank": "22"}}, + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + writes: list[dict] = [] + monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, "")) + monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script")) + monkeypatch.setattr( + peer_store_writer, + "_build_brand_options_cache", + lambda *_args: {"alpha": "Alpha", "beta": "Beta"}, + ) + monkeypatch.setattr( + peer_store_writer, + "_build_month_options_cache", + lambda *_args: {"2026.07月": "2026.07月"}, + ) + monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing") + monkeypatch.setattr( + peer_store_writer, + "_upsert_peer_record", + lambda *_args: writes.append(_args[-1]) or {"data": {"record_id_list": ["existing"]}}, + ) + + assert peer_store_writer._write_dy_peer_store(str(artifact)) is True + assert [row[DY_PEER_BASE_FIELDS["竞店名称"]] for row in writes] == [ + "Alpha", + "Beta", + ] + + +def test_dy_local_shop_validation_failure_propagates_false( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + peer_artifact = tmp_path / "dy-peer.json" + peer_artifact.write_text( + json.dumps( + { + "date_range": "2026/07/27 - 2026/08/02", + "results": { + DY_PEER_CAT_MAP["斜挎包"]: {"alpha": {"rank": "11"}}, + DY_PEER_CAT_MAP["双肩包"]: {"alpha": {"rank": "22"}}, + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + shop_artifact = tmp_path / "dy-shop.json" + shop_artifact.write_text("{}", encoding="utf-8") + monkeypatch.setattr(peer_store_writer, "validate_peer_payload", lambda *_args: (True, "")) + monkeypatch.setattr(peer_store_writer, "resolve_lark_cli", lambda: ("node", "script")) + monkeypatch.setattr( + peer_store_writer, + "_build_brand_options_cache", + lambda *_args: {"alpha": "Alpha"}, + ) + monkeypatch.setattr( + peer_store_writer, + "_build_month_options_cache", + lambda *_args: {"2026.07月": "2026.07月"}, + ) + monkeypatch.setattr(peer_store_writer, "_find_existing_peer_record", lambda *_args: "existing") + monkeypatch.setattr( + peer_store_writer, + "_upsert_peer_record", + lambda *_args: {"data": {"record_id_list": ["existing"]}}, + ) + monkeypatch.setattr(peer_store_writer, "_write_dy_local_shop_row", lambda *_args: False) + + assert ( + peer_store_writer._write_dy_peer_store( + str(peer_artifact), + str(shop_artifact), + ) + is False + ) + + +def test_invalid_dy_local_shop_rankings_return_false_without_upsert( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + artifact = tmp_path / "shop.json" + artifact.write_text(json.dumps({"market_rankings": {}}), encoding="utf-8") + monkeypatch.setattr( + peer_store_writer, + "validate_market_rankings", + lambda *_args: (False, "missing rankings"), + ) + monkeypatch.setattr( + peer_store_writer, + "_upsert_peer_record", + lambda *_args: pytest.fail("invalid local rankings must not be written"), + ) + + assert ( + peer_store_writer._write_dy_local_shop_row( + "node", + "script", + "7月第5周（7.27-8.2）", + "2026.07月", + str(artifact), + ) + is False + ) + + +def test_configured_peer_identity_fields_are_non_empty() -> None: + for fields in (JD_PEER_BASE_FIELDS, DY_PEER_BASE_FIELDS): + assert fields["时间"] + assert fields["月份"] + assert fields["竞店名称"] diff --git a/tests/modules/shop_intelligence/test_jd_peer_category_switch.py b/tests/modules/shop_intelligence/test_jd_peer_category_switch.py new file mode 100644 index 0000000..cddc20a --- /dev/null +++ b/tests/modules/shop_intelligence/test_jd_peer_category_switch.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_peer_store_data_collector as collector, +) + + +class _HiddenMenuLocator: + def count(self) -> int: + return 0 + + +class _Page: + def __init__(self) -> None: + self.keyboard = SimpleNamespace(press=lambda _key: None) + + def locator(self, _selector: str) -> _HiddenMenuLocator: + return _HiddenMenuLocator() + + def wait_for_timeout(self, _timeout_ms: int) -> None: + return None + + +def test_category_path_normalization_is_exact() -> None: + assert collector._normalize_category_path(" 男包 → 男士双肩包 ") == ( + "男包 > 男士双肩包" + ) + assert collector._expected_category_path("男士双肩包", "男包") == ( + "男包 > 男士双肩包" + ) + assert collector._expected_category_path("男包", None) == "男包" + + +def test_category_menu_closes_stale_portals_before_opening_selector() -> None: + class Menus: + def __init__(self, page) -> None: + self.page = page + self.first = self + + def count(self) -> int: + return self.page.menu_count + + def wait_for(self, **_kwargs) -> None: + return None + + class Trigger: + def __init__(self, page) -> None: + self.page = page + + def click(self, **_kwargs) -> None: + self.page.menu_count = 1 + + class Selector: + def __init__(self, page) -> None: + self.page = page + + def locator(self, _selector: str) -> Trigger: + return Trigger(self.page) + + class Page: + def __init__(self) -> None: + self.menu_count = 2 + self.escapes = 0 + self.keyboard = SimpleNamespace(press=self.press) + + def press(self, key: str) -> None: + assert key == "Escape" + self.escapes += 1 + self.menu_count = 0 + + def locator(self, _selector: str) -> Menus: + return Menus(self) + + def wait_for_timeout(self, _timeout_ms: int) -> None: + return None + + page = Page() + + menu = collector._visible_category_menu(page, Selector(page)) + + assert menu.count() == 1 + assert page.escapes == 1 + + +def test_category_menu_targets_only_outer_dropdown_panel() -> None: + source = __import__("inspect").getsource(collector._visible_category_menu) + + assert ".jmtd-dropdown-panel.jmtd-dropdown-lists:visible" in source + + +def test_successful_category_switch_closes_its_dropdown_portal( + monkeypatch, +) -> None: + class Menus: + def __init__(self, page) -> None: + self.page = page + + def count(self) -> int: + return self.page.menu_count + + class Trigger: + def __init__(self, page) -> None: + self.page = page + + def click(self, **_kwargs) -> None: + self.page.menu_count = 0 + + class Selector: + def __init__(self, page) -> None: + self.page = page + + def locator(self, _selector: str) -> Trigger: + return Trigger(self.page) + + class Page: + def __init__(self) -> None: + self.menu_count = 1 + self.keyboard = SimpleNamespace(press=lambda _key: None) + + def locator(self, _selector: str) -> Menus: + return Menus(self) + + def wait_for_timeout(self, _timeout_ms: int) -> None: + return None + + page = Page() + selector = Selector(page) + monkeypatch.setattr(collector, "_visible_category_selector", lambda _page: selector) + monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before") + monkeypatch.setattr( + collector, "_selected_category_path", lambda _selector: "男包 > 男士双肩包" + ) + + assert collector.switch_category(page, "男士双肩包", "男包") is True + assert page.menu_count == 0 + + +def test_ranking_refresh_accepts_only_explicit_empty_state(monkeypatch) -> None: + monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "") + monkeypatch.setattr( + collector, "_ranking_table_is_explicitly_empty", lambda _page: True + ) + + assert collector._wait_for_ranking_table_refresh( + object(), "previous", timeout_ms=1 + ) is True + + +def test_ranking_refresh_rejects_unverified_empty_state(monkeypatch) -> None: + monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "") + monkeypatch.setattr( + collector, "_ranking_table_is_explicitly_empty", lambda _page: False + ) + monkeypatch.setattr(collector.time, "sleep", lambda _seconds: None) + + assert collector._wait_for_ranking_table_refresh( + object(), "previous", timeout_ms=0 + ) is False + + +def test_switch_category_requires_selected_state_and_table_refresh( + monkeypatch, +) -> None: + page = _Page() + selector = object() + menu = object() + observed: list[object] = [] + + monkeypatch.setattr( + collector, + "_visible_category_selector", + lambda actual_page: selector if actual_page is page else None, + ) + monkeypatch.setattr(collector, "_ranking_table_fingerprint", lambda _page: "before") + monkeypatch.setattr(collector, "_selected_category_path", lambda _selector: "男包") + monkeypatch.setattr( + collector, + "_visible_category_menu", + lambda actual_page, actual_selector: ( + menu if (actual_page, actual_selector) == (page, selector) else None + ), + ) + monkeypatch.setattr( + collector, + "_select_category_option", + lambda actual_page, actual_menu, category, parent: observed.append( + (actual_page, actual_menu, category, parent) + ), + ) + monkeypatch.setattr( + collector, + "_wait_for_selected_category", + lambda actual_selector, expected: ( + observed.append((actual_selector, expected, "selected")) or True + ), + ) + monkeypatch.setattr( + collector, + "_wait_for_ranking_table_refresh", + lambda actual_page, previous: ( + observed.append((actual_page, previous, "refreshed")) or True + ), + ) + + assert collector.switch_category(page, "男士双肩包", "男包") is True + assert observed == [ + (page, menu, "男士双肩包", "男包"), + (selector, "男包 > 男士双肩包", "selected"), + (page, "before", "refreshed"), + ] + + +def test_failed_category_switch_never_extracts_previous_table( + monkeypatch, +) -> None: + page = _Page() + extracted: list[str] = [] + + monkeypatch.setattr(collector, "SHOPS", ["peer-shop"]) + monkeypatch.setattr( + collector, + "CATEGORIES", + [("男士双肩包", "男包"), ("男士单肩/斜挎包", "男包")], + ) + monkeypatch.setattr(collector, "select_last_week_any_day", lambda _page: None) + monkeypatch.setattr(collector, "search_shop", lambda *_args: None) + monkeypatch.setattr( + collector, + "switch_category", + lambda _page, category, _parent: category == "男士单肩/斜挎包", + ) + + def extract_rows(_page): + extracted.append("called") + return [ + { + "rank": "7", + "shop": "peer-shop", + "amount": "￥1万~￥2万", + "orders": "10~50", + "followers": "3", + "visitors": "1,000~2,000", + "searchClicks": "600~800", + } + ] + + monkeypatch.setattr(collector, "extract_ranking_table", extract_rows) + + result = collector.step_collect_competitor_data(page)["peer-shop"] + + assert result["男士双肩包"]["rank"] == "N/A" + assert "禁止复用" not in result["男士双肩包"]["error"] + assert "类目切换或榜单刷新失败" in result["男士双肩包"]["error"] + assert result["男士单肩/斜挎包"]["rank"] == "7" + assert extracted == ["called"] diff --git a/tests/modules/shop_intelligence/test_jd_self_operated_brand_daily.py b/tests/modules/shop_intelligence/test_jd_self_operated_brand_daily.py new file mode 100644 index 0000000..6d3930d --- /dev/null +++ b/tests/modules/shop_intelligence/test_jd_self_operated_brand_daily.py @@ -0,0 +1,225 @@ +import unittest +from datetime import date +from decimal import Decimal +from types import SimpleNamespace + +from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_self_operated_brand_daily as brand_daily, +) +from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily import ( + browser_runtime_options, + build_db_records, + is_authenticated_shop_url, + iter_dates, + parse_decimal, + parse_percent, + resolve_account_identifier, +) + + +def test_browser_runtime_defaults_to_bundled_chromium(monkeypatch): + monkeypatch.delenv("GYXX_BROWSER_CHANNEL", raising=False) + + assert browser_runtime_options() == {} + + +def test_browser_runtime_can_use_installed_chrome(monkeypatch): + monkeypatch.setenv("GYXX_BROWSER_CHANNEL", "chrome") + + assert browser_runtime_options() == {"channel": "chrome"} + + +class DateRangeTests(unittest.TestCase): + def test_iter_dates_is_inclusive(self): + self.assertEqual( + list(iter_dates(date(2026, 6, 29), date(2026, 7, 2))), + [ + date(2026, 6, 29), + date(2026, 6, 30), + date(2026, 7, 1), + date(2026, 7, 2), + ], + ) + + def test_iter_dates_rejects_reverse_range(self): + with self.assertRaisesRegex(ValueError, "start_date"): + list(iter_dates(date(2026, 7, 2), date(2026, 7, 1))) + + +class LoginStateTests(unittest.TestCase): + def test_shop_page_is_authenticated(self): + self.assertTrue( + is_authenticated_shop_url( + "https://shop.jd.com/jdm/vc/data-management/performance/brandPerformance" + ) + ) + + def test_passport_and_security_pages_are_not_authenticated(self): + self.assertFalse( + is_authenticated_shop_url( + "https://passport.shop.jd.com/login/index.action/jdm" + ) + ) + self.assertFalse( + is_authenticated_shop_url("https://aq.jd.com/certified/index?p=abc") + ) + + +class _FakeLocator: + def __init__(self, value=None): + self.value = value + self.first = self + + def count(self): + return int(self.value is not None) + + def get_attribute(self, name): + assert name == "title" + return self.value + + def inner_text(self): + return self.value or "" + + +class _FakePage: + def __init__(self, values, cookies=None): + self.values = values + self.context = SimpleNamespace(cookies=lambda: cookies or []) + + def locator(self, selector): + return _FakeLocator(self.values.get(selector)) + + +class AccountIdentifierTests(unittest.TestCase): + def test_configured_account_takes_priority(self): + self.assertEqual( + resolve_account_identifier(_FakePage({}), " gyxx2022 "), + "gyxx2022", + ) + + def test_cookie_session_uses_signed_in_pin(self): + page = _FakePage( + { + ".shop-menu-accountV1__account-dropdown .content-pin[title]": ( + "gyxx2022" + ), + ".shop-menu-accountV1__right-account-top-name[title]": ( + "惠州鑫华达科技有限公司" + ), + } + ) + self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022") + + def test_cookie_session_falls_back_to_company_name(self): + page = _FakePage( + { + ".shop-menu-accountV1__right-account-top-name[title]": ( + "惠州鑫华达科技有限公司" + ) + } + ) + self.assertEqual( + resolve_account_identifier(page, ""), + "惠州鑫华达科技有限公司", + ) + + def test_cookie_session_uses_jd_pin_cookie_without_account_menu(self): + page = _FakePage( + {}, + cookies=[ + { + "name": "pin", + "value": "gyxx%32%30%32%32", + "domain": ".jd.com", + } + ], + ) + self.assertEqual(resolve_account_identifier(page, ""), "gyxx2022") + + def test_cookie_session_requires_stable_identifier(self): + with self.assertRaisesRegex(RuntimeError, "无法识别当前京麦账号"): + resolve_account_identifier(_FakePage({}), "") + + +class ValueParsingTests(unittest.TestCase): + def test_parse_decimal_handles_commas_and_currency(self): + self.assertEqual(parse_decimal("¥34,605.80"), Decimal("34605.80")) + self.assertEqual(parse_decimal("122.00"), Decimal("122.00")) + + def test_parse_decimal_preserves_negative_values(self): + self.assertEqual(parse_decimal("-2,173.20"), Decimal("-2173.20")) + + def test_parse_decimal_returns_none_for_placeholder(self): + for value in (None, "", "-", "--", "—", "N/A"): + self.assertIsNone(parse_decimal(value)) + + def test_parse_percent_converts_display_percent_to_ratio(self): + self.assertEqual(parse_percent("6.25%"), Decimal("0.0625")) + self.assertEqual(parse_percent("0.06"), Decimal("0.06")) + + +class RecordBuildingTests(unittest.TestCase): + def test_build_db_records_maps_all_columns(self): + rows = [ + { + "品牌": "光影行星（GYXX）", + "财务销量": "122.00", + "收入": "34,605.80", + "毛利": "2,173.20", + "毛利率": "6.00%", + "库存金额": "586,149.83", + } + ] + + records = build_db_records( + account="gyxx2022", + data_date=date(2026, 7, 29), + rows=rows, + source_url="https://shop.jd.com/example", + ) + + self.assertEqual(len(records), 1) + record = records[0] + self.assertEqual(record["account"], "gyxx2022") + self.assertEqual(record["data_date"], date(2026, 7, 29)) + self.assertEqual(record["brand"], "光影行星（GYXX）") + self.assertEqual(record["financial_sales"], Decimal("122.00")) + self.assertEqual(record["revenue"], Decimal("34605.80")) + self.assertEqual(record["gross_profit"], Decimal("2173.20")) + self.assertEqual(record["gross_margin"], Decimal("0.06")) + self.assertEqual(record["inventory_amount"], Decimal("586149.83")) + self.assertEqual(record["raw_row"]["收入"], "34,605.80") + + def test_build_db_records_skips_empty_placeholder_row(self): + records = build_db_records( + account="gyxx2022", + data_date=date(2026, 7, 29), + rows=[{"品牌": "-", "收入": "-"}], + source_url="https://shop.jd.com/example", + ) + self.assertEqual(records, []) + + +def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch): + class FakePolicy: + enabled = True + + def preflight_cookie(self, binding, *, environment): + return SimpleNamespace(should_skip=True, reason="browser state is missing") + + monkeypatch.setattr(brand_daily, "current_acceptance_policy", lambda: FakePolicy()) + monkeypatch.setattr(brand_daily, "binding_from_environment", lambda: "binding") + monkeypatch.setattr( + brand_daily, + "parse_args", + lambda argv: (_ for _ in ()).throw( + AssertionError("arguments must not be parsed after a cookie skip") + ), + ) + + assert brand_daily.main([]) == COOKIE_SKIP_EXIT_CODE + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/modules/shop_intelligence/test_jd_self_operated_product_daily.py b/tests/modules/shop_intelligence/test_jd_self_operated_product_daily.py new file mode 100644 index 0000000..9d7212b --- /dev/null +++ b/tests/modules/shop_intelligence/test_jd_self_operated_product_daily.py @@ -0,0 +1,108 @@ +import unittest +from datetime import date +from decimal import Decimal + +from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_self_operated_product_daily as product_daily, +) +from gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily import ( + build_product_db_records, + parse_total_count, + validate_collected_rows, +) + + +class TotalCountTests(unittest.TestCase): + def test_parse_total_count(self): + self.assertEqual(parse_total_count("共50条"), 50) + self.assertEqual(parse_total_count("共 1,234 条"), 1234) + + def test_parse_total_count_rejects_unknown_text(self): + with self.assertRaisesRegex(ValueError, "总条数"): + parse_total_count("暂无数据") + + +class ProductRecordTests(unittest.TestCase): + def test_maps_all_product_columns(self): + rows = [ + { + "商品编码": "100232520894", + "商品名称": "光影行星测试商品", + "财务销量": "1.00", + "收入": "145.14", + "成本": "196.60", + "毛利": "-51.46", + "优惠券抵减数": "50.68", + "积分抵减数": "0", + "满返满减额": "0", + "库存周转天数": "70.55", + } + ] + + records = build_product_db_records( + account="gyxx2022", + data_date=date(2026, 7, 30), + rows=rows, + source_url="https://shop.jd.com/example", + ) + + self.assertEqual(len(records), 1) + record = records[0] + self.assertEqual(record["product_code"], "100232520894") + self.assertEqual(record["product_name"], "光影行星测试商品") + self.assertEqual(record["financial_sales"], Decimal("1.00")) + self.assertEqual(record["revenue"], Decimal("145.14")) + self.assertEqual(record["cost"], Decimal("196.60")) + self.assertEqual(record["gross_profit"], Decimal("-51.46")) + self.assertEqual(record["coupon_deduction"], Decimal("50.68")) + self.assertEqual(record["points_deduction"], Decimal("0")) + self.assertEqual(record["promotion_deduction"], Decimal("0")) + self.assertEqual(record["inventory_turnover_days"], Decimal("70.55")) + + def test_skips_row_without_product_code(self): + self.assertEqual( + build_product_db_records( + "gyxx2022", + date(2026, 7, 30), + [{"商品编码": "-", "商品名称": "占位行"}], + "https://shop.jd.com/example", + ), + [], + ) + + +class PaginationValidationTests(unittest.TestCase): + def test_accepts_complete_unique_rows(self): + rows = [{"商品编码": "1"}, {"商品编码": "2"}] + self.assertEqual(validate_collected_rows(rows, 2), rows) + + def test_rejects_duplicate_product_codes(self): + rows = [{"商品编码": "1"}, {"商品编码": "1"}] + with self.assertRaisesRegex(RuntimeError, "重复"): + validate_collected_rows(rows, 2) + + def test_rejects_missing_rows(self): + with self.assertRaisesRegex(RuntimeError, "总条数"): + validate_collected_rows([{"商品编码": "1"}], 2) + + +def test_acceptance_preflight_skips_before_argument_parsing(monkeypatch): + monkeypatch.setattr( + product_daily, + "preflight_acceptance_login", + lambda: COOKIE_SKIP_EXIT_CODE, + ) + monkeypatch.setattr( + product_daily, + "parse_args", + lambda argv: (_ for _ in ()).throw( + AssertionError("arguments must not be parsed after a cookie skip") + ), + ) + + assert product_daily.main([]) == COOKIE_SKIP_EXIT_CODE + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/modules/shop_intelligence/test_peer_store_hardening.py b/tests/modules/shop_intelligence/test_peer_store_hardening.py new file mode 100644 index 0000000..513ab1f --- /dev/null +++ b/tests/modules/shop_intelligence/test_peer_store_hardening.py @@ -0,0 +1,517 @@ +from __future__ import annotations + +import json +import sys +from copy import deepcopy +from importlib import import_module +from pathlib import Path + +import pytest + +from gyxx_flow.catalog import WorkflowCatalog +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule +from gyxx_flow.modules.shop_intelligence.config import DY_PLATFORM, JD_PLATFORM +from gyxx_flow.modules.shop_intelligence.db import db as shop_db +from gyxx_flow.modules.shop_intelligence.quality import validate_peer_payload + +run_peer_store = import_module( + "gyxx_flow.modules.shop_intelligence.runners.run_peer_store" +) +PROJECT_ROOT = Path(__file__).resolve().parents[3] + +JD_BRANDS = ( + "Bellroy官方旗舰店", + "tomtoc官方旗舰店", + "探迹者TAJEZZO官方旗舰店", + "BAGSMART旗舰店", + "GROTTO个乐官方旗舰店", + "NIID箱包旗舰店", + "光影行星 GYXX官方旗舰店", +) +JD_CATEGORIES = ( + "男士包袋", + "男士双肩包", + "男士单肩/斜挎包", + "电脑包", + "男士腰包/胸包", + "旅行包", + "休闲运动包", +) +DY_BRANDS = ( + "Bellroy", + "tomtoc", + "BAGSMART", + "GROTTO", + "NIID", + "reeyee", + "VAOPER", + "探迹者", +) +DY_CATEGORIES = ("单肩包/斜挎包", "双肩包", "腰包/胸包") + + +def _jd_ranked_entry(brand: str, seed: int) -> dict: + return { + "rank": str(seed + 1), + "shop": brand, + "amount": f"￥{seed + 10}万~￥{seed + 20}万", + "orders": f"{seed + 20}~{seed + 30}", + "followers": str(seed + 40), + "visitors": f"{seed + 50}~{seed + 60}", + "searchClicks": str(seed + 70), + } + + +def _jd_unranked_entry(brand: str) -> dict: + return { + "rank": "N/A", + "amount": "N/A", + "orders": "N/A", + "followers": "N/A", + "visitors": "N/A", + "searchClicks": "N/A", + "error": f"当前类目榜单未找到店铺: {brand}", + } + + +def _jd_payload() -> dict: + return { + "数据": { + brand: { + category: _jd_ranked_entry(brand, brand_index * 20 + category_index) + for category_index, category in enumerate(JD_CATEGORIES) + } + for brand_index, brand in enumerate(JD_BRANDS) + } + } + + +def _dy_range(seed: int) -> dict: + return { + "min": float(seed), + "max": float(seed + 1), + "open_ended": False, + "raw_values": [str(seed)], + } + + +def _dy_entry( + brand: str, + category: str, + seed: int, + *, + ranked: bool = True, + matched: bool = True, +) -> dict: + rank = str(seed + 1) if ranked else "" + stores = ( + [ + { + "shop_name": f"{brand}旗舰店", + "rank": rank, + "row_text": f"{brand}-{seed}", + } + ] + if matched + else [] + ) + return { + "brand": brand, + "category": category, + "matched_shop_count": len(stores), + "rank": rank, + "rank_display": rank or "未进入前200名", + "payment_amount": _dy_range(seed + 10), + "order_count": _dy_range(seed + 20), + "product_click_users": _dy_range(seed + 30), + "stores": stores, + "captured_at": "2026-08-04T10:00:00", + } + + +def _dy_payload() -> dict: + return { + "categories": list(DY_CATEGORIES), + "brands": list(DY_BRANDS), + "results": { + category: { + brand: _dy_entry(brand, category, category_index * 100 + brand_index) + for brand_index, brand in enumerate(DY_BRANDS) + } + for category_index, category in enumerate(DY_CATEGORIES) + }, + } + + +def test_runner_rejects_success_without_fresh_peer_artifact(monkeypatch) -> None: + marker = ("D:/old-peer.json", 1, 10) + monkeypatch.setattr(run_peer_store, "_artifact_marker", lambda _platform: marker) + + result = run_peer_store._run_with_fresh_artifact( + JD_PLATFORM, + lambda: (JD_PLATFORM, 0, "collector output"), + ) + + assert result[1] == 1 + assert result[3] == "" + assert "no fresh peer-store file" in result[2] + + +def test_runner_passes_only_the_fresh_peer_artifact( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "new-peer.json" + artifact.write_text( + json.dumps(_jd_payload(), ensure_ascii=False), + encoding="utf-8", + ) + marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size) + markers = iter((None, marker)) + monkeypatch.setattr( + run_peer_store, + "_artifact_marker", + lambda _platform: next(markers), + ) + + result = run_peer_store._run_with_fresh_artifact( + JD_PLATFORM, + lambda: (JD_PLATFORM, 0, "ok"), + ) + + assert result == (JD_PLATFORM, 0, "ok", str(artifact)) + + +def test_runner_rejects_identical_jd_payload_copied_across_categories( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "bad-peer.json" + payload = _jd_payload() + repeated = _jd_ranked_entry(JD_BRANDS[0], 22) + for category in JD_CATEGORIES: + payload["数据"][JD_BRANDS[0]][category] = repeated + artifact.write_text( + json.dumps(payload, ensure_ascii=False), + encoding="utf-8", + ) + marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size) + markers = iter((None, marker)) + monkeypatch.setattr( + run_peer_store, + "_artifact_marker", + lambda _platform: next(markers), + ) + + result = run_peer_store._run_with_fresh_artifact( + JD_PLATFORM, + lambda: (JD_PLATFORM, 0, "ok"), + ) + + assert result[1] == 1 + assert result[3] == "" + assert "identical payload copied across categories" in result[2] + + +def test_quality_gate_allows_one_unranked_brand_when_other_brands_differ() -> None: + payload = _jd_payload() + brand = JD_BRANDS[0] + payload["数据"][brand] = { + category: _jd_unranked_entry(brand) for category in JD_CATEGORIES + } + + assert validate_peer_payload(JD_PLATFORM, payload) == (True, "") + + +def test_quality_gate_rejects_explicit_category_switch_error() -> None: + payload = _jd_payload() + payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[1]] = { + **_jd_unranked_entry(JD_BRANDS[0]), + "error": "类目切换或榜单刷新失败", + } + + valid, reason = validate_peer_payload(JD_PLATFORM, payload) + + assert valid is False + assert "类目切换或榜单刷新失败" in reason + + +def test_direct_jd_peer_persistence_rejects_repeated_payload_before_connecting( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "bad-peer-direct.json" + payload = _jd_payload() + repeated = _jd_ranked_entry(JD_BRANDS[-2], 22) + for category in JD_CATEGORIES: + payload["数据"][JD_BRANDS[-2]][category] = repeated + artifact.write_text( + json.dumps(payload, ensure_ascii=False), + encoding="utf-8", + ) + monkeypatch.setattr(shop_db, "DB_ENABLED", True) + monkeypatch.setattr( + shop_db, + "get_conn", + lambda: pytest.fail("invalid payload must not connect to PostgreSQL"), + ) + + with pytest.raises(ValueError, match="JD peer-store data quality check failed"): + shop_db.persist_jd_peer_store(str(artifact)) + + +def test_quality_gate_requires_exact_jd_brand_and_category_sets() -> None: + missing_brand = _jd_payload() + missing_brand["数据"].pop(JD_BRANDS[-1]) + valid, reason = validate_peer_payload(JD_PLATFORM, missing_brand) + assert valid is False + assert "brand set mismatch" in reason + + missing_category = _jd_payload() + missing_category["数据"][JD_BRANDS[0]].pop(JD_CATEGORIES[-1]) + valid, reason = validate_peer_payload(JD_PLATFORM, missing_category) + assert valid is False + assert "category set mismatch" in reason + + +def test_quality_gate_rejects_malformed_jd_ranking_entry() -> None: + payload = _jd_payload() + payload["数据"][JD_BRANDS[0]][JD_CATEGORIES[0]] = "23" + + valid, reason = validate_peer_payload(JD_PLATFORM, payload) + + assert valid is False + assert "ranking entry is not an object" in reason + + +def test_quality_gate_allows_verified_dy_unranked_results() -> None: + payload = _dy_payload() + for category in DY_CATEGORIES: + for brand_index, brand in enumerate(DY_BRANDS): + payload["results"][category][brand] = _dy_entry( + brand, + category, + brand_index, + ranked=False, + matched=False, + ) + + assert validate_peer_payload(DY_PLATFORM, payload) == (True, "") + + +def test_quality_gate_rejects_identical_dy_data_in_two_categories() -> None: + payload = _dy_payload() + first_category, second_category = DY_CATEGORIES[:2] + for brand in DY_BRANDS: + repeated = deepcopy(payload["results"][first_category][brand]) + repeated["category"] = second_category + payload["results"][second_category][brand] = repeated + + valid, reason = validate_peer_payload(DY_PLATFORM, payload) + + assert valid is False + assert "identical payload copied across categories" in reason + + +def test_quality_gate_requires_exact_dy_sets_and_entry_structure() -> None: + payload = _dy_payload() + payload["results"][DY_CATEGORIES[0]].pop(DY_BRANDS[-1]) + valid, reason = validate_peer_payload(DY_PLATFORM, payload) + assert valid is False + assert "brand set mismatch" in reason + + payload = _dy_payload() + payload["results"][DY_CATEGORIES[0]][DY_BRANDS[0]].pop("stores") + valid, reason = validate_peer_payload(DY_PLATFORM, payload) + assert valid is False + assert "ranking fields missing stores" in reason + + +def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None: + monkeypatch.setattr( + sys, + "argv", + ["run_peer_store", "--platform", "dy", "--skip-feishu"], + ) + monkeypatch.setattr( + run_peer_store, + "setup_runner_logging", + lambda _name: "test.log", + ) + monkeypatch.setattr( + run_peer_store, + "skip_feishu_for_acceptance", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + run_peer_store, + "_run_with_fresh_artifact", + lambda platform, _collector: ( + platform, + COOKIE_SKIP_EXIT_CODE, + "expired", + "", + ), + ) + monkeypatch.setattr( + run_peer_store, + "_persist_and_write", + lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"), + ) + + assert run_peer_store.main() == COOKIE_SKIP_EXIT_CODE + + +def test_persist_zero_is_reported_as_failure_and_feishu_can_be_skipped( + monkeypatch, +) -> None: + monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0) + monkeypatch.setattr( + run_peer_store, + "write_peer_store", + lambda *_args, **_kwargs: pytest.fail("Feishu must remain skipped"), + ) + + errors = run_peer_store._persist_and_write( + JD_PLATFORM, + True, + "D:/fresh-peer.json", + skip_feishu=True, + ) + + assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"] + + +def test_persist_zero_stops_peer_feishu_write(monkeypatch) -> None: + monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 0) + monkeypatch.setattr( + run_peer_store, + "write_peer_store", + lambda *_args, **_kwargs: pytest.fail("Feishu must not run after DB failure"), + ) + + errors = run_peer_store._persist_and_write( + JD_PLATFORM, + True, + "D:/fresh-peer.json", + ) + + assert errors == [f"{JD_PLATFORM} DB 持久化失败: 未写入任何记录"] + + +def test_dy_peer_rejects_shop_json_from_another_week_before_writes( + monkeypatch, + tmp_path: Path, +) -> None: + peer_json = tmp_path / "peer.json" + peer_json.write_text( + json.dumps({"date_range": "2026/07/27 - 2026/08/02"}), + encoding="utf-8", + ) + shop_json = tmp_path / "shop.json" + shop_json.write_text( + json.dumps({"date_range": "2026/07/20 - 2026/07/26"}), + encoding="utf-8", + ) + monkeypatch.setattr( + run_peer_store, + "_latest_shop_json_for_dy", + lambda: str(shop_json), + ) + monkeypatch.setattr( + run_peer_store, + "persist_dy_peer_store", + lambda _path: pytest.fail("mismatched snapshots must fail before PG"), + ) + monkeypatch.setattr( + run_peer_store, + "write_peer_store", + lambda *_args, **_kwargs: pytest.fail("mismatched snapshots must not write Feishu"), + ) + + errors = run_peer_store._persist_and_write( + DY_PLATFORM, + True, + str(peer_json), + ) + + assert len(errors) == 1 + assert "周区间不一致" in errors[0] + + +def test_dy_peer_passes_matching_shop_json_to_writer( + monkeypatch, + tmp_path: Path, +) -> None: + peer_json = tmp_path / "peer.json" + peer_json.write_text( + json.dumps({"date_range": "2026/07/27 - 2026/08/02"}), + encoding="utf-8", + ) + shop_json = tmp_path / "shop.json" + shop_json.write_text( + json.dumps({"date_range": "2026-07-27 ~ 2026-08-02"}), + encoding="utf-8", + ) + observed = {} + monkeypatch.setattr( + run_peer_store, + "_latest_shop_json_for_dy", + lambda: str(shop_json), + ) + monkeypatch.setattr(run_peer_store, "persist_dy_peer_store", lambda _path: 3) + + def fake_write(platform, actual_peer_json, actual_shop_json): + observed.update( + platform=platform, + peer_json=actual_peer_json, + shop_json=actual_shop_json, + ) + return True + + monkeypatch.setattr(run_peer_store, "write_peer_store", fake_write) + + errors = run_peer_store._persist_and_write( + DY_PLATFORM, + True, + str(peer_json), + ) + + assert errors == [] + assert observed == { + "platform": DY_PLATFORM, + "peer_json": str(peer_json), + "shop_json": str(shop_json), + } + + +def test_false_feishu_writer_result_is_reported(monkeypatch) -> None: + monkeypatch.setattr(run_peer_store, "persist_jd_peer_store", lambda _path: 3) + monkeypatch.setattr(run_peer_store, "write_peer_store", lambda *_args: False) + + errors = run_peer_store._persist_and_write( + JD_PLATFORM, + True, + "D:/fresh-peer.json", + ) + + assert errors == [f"{JD_PLATFORM} 竞店飞书写入失败: writer 返回 False"] + + +def test_competitor_platforms_are_independent_graph_steps() -> None: + module = ShopIntelligenceModule.from_catalog( + WorkflowCatalog.load(PROJECT_ROOT / "config") + ) + competitor = next( + definition + for definition in module.workflow_definitions() + if definition.workflow_id == "shop.competitor.weekly" + ) + + assert tuple(step.step_id for step in competitor.steps) == ("jd", "dy") + assert all(not step.depends_on for step in competitor.steps) + assert tuple(step.action.command_args for step in competitor.steps) == ( + ("--platform", "jd"), + ("--platform", "dy"), + ) diff --git a/tests/modules/shop_intelligence/test_run_jd_self_operated_daily.py b/tests/modules/shop_intelligence/test_run_jd_self_operated_daily.py new file mode 100644 index 0000000..69879fe --- /dev/null +++ b/tests/modules/shop_intelligence/test_run_jd_self_operated_daily.py @@ -0,0 +1,39 @@ +import unittest +from datetime import date + +from gyxx_flow.modules.shop_intelligence.runners.run_jd_self_operated_daily import ( + build_collector_args, + previous_day, +) + + +class TargetDateTests(unittest.TestCase): + def test_previous_day_handles_month_boundary(self): + self.assertEqual(previous_day(date(2026, 8, 1)), date(2026, 7, 31)) + + def test_previous_day_handles_year_boundary(self): + self.assertEqual(previous_day(date(2027, 1, 1)), date(2026, 12, 31)) + + +class CollectorArgsTests(unittest.TestCase): + def test_builds_single_day_headless_args(self): + self.assertEqual( + build_collector_args(date(2026, 7, 30), headless=True), + [ + "--start-date", + "2026-07-30", + "--end-date", + "2026-07-30", + "--headless", + ], + ) + + def test_headful_args_omit_headless_flag(self): + self.assertNotIn( + "--headless", + build_collector_args(date(2026, 7, 30), headless=False), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/modules/shop_intelligence/test_shop_acceptance_feishu_boundaries.py b/tests/modules/shop_intelligence/test_shop_acceptance_feishu_boundaries.py new file mode 100644 index 0000000..19ece2b --- /dev/null +++ b/tests/modules/shop_intelligence/test_shop_acceptance_feishu_boundaries.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +from gyxx_flow.modules.shop_intelligence import lark_cli + + +def _enable_acceptance(monkeypatch, tmp_path: Path) -> Path: + evidence = tmp_path / "shop-evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.delenv("GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", raising=False) + return evidence + + +def test_shop_shared_lark_cli_physically_skips_bitable_write( + monkeypatch, + tmp_path: Path, +) -> None: + evidence = _enable_acceptance(monkeypatch, tmp_path) + + def fail_if_spawned(*args, **kwargs): + raise AssertionError("lark-cli subprocess must not start") + + monkeypatch.setattr(lark_cli.subprocess, "run", fail_if_spawned) + result = lark_cli.run_lark_cli( + "node", + "run.js", + ["base", "+record-batch-update", "--table-id", "tbl_test"], + ) + + assert result["acceptance_skipped"] is True + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["event"] == "feishu_write_skipped" + assert payload["operation"].endswith("record-batch-update") + + +def test_shop_shared_lark_cli_keeps_production_write_behavior(monkeypatch) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="") + + monkeypatch.setattr(lark_cli.subprocess, "run", fake_run) + result = lark_cli.run_lark_cli( + "node", + "run.js", + ["base", "+record-batch-create", "--table-id", "tbl_test"], + ) + + assert result == {"ok": True} + assert len(calls) == 1 diff --git a/tests/modules/shop_intelligence/test_shop_metrics_hardening.py b/tests/modules/shop_intelligence/test_shop_metrics_hardening.py new file mode 100644 index 0000000..b21268b --- /dev/null +++ b/tests/modules/shop_intelligence/test_shop_metrics_hardening.py @@ -0,0 +1,916 @@ +from __future__ import annotations + +import ast +import inspect +import json +import sys +from datetime import date +from importlib import import_module +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from gyxx_flow.core.exit_codes import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.shop_intelligence import lark_cli as shop_lark_cli +from gyxx_flow.modules.shop_intelligence.collectors import ( + jd_data_collector, + jd_peer_store_data_collector, + jd_self_operated_brand_daily, + taobao_sycm, +) +from gyxx_flow.modules.shop_intelligence.db import db as shop_db +from gyxx_flow.modules.shop_intelligence.lark_cli import ( + format_week, + normalize_week_label, + week_search_prefix, +) +from gyxx_flow.modules.shop_intelligence.quality import validate_shop_payload +from gyxx_flow.modules.shop_intelligence.writers import shop_base_writer + +run_shop = import_module("gyxx_flow.modules.shop_intelligence.runners.run_shop") + + +def test_jd_playwright_wait_for_function_passes_argument_by_keyword() -> None: + tree = ast.parse( + Path(jd_data_collector.__file__).read_text(encoding="utf-8-sig") + ) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "wait_for_function" + ] + + assert calls + assert all(len(call.args) == 1 for call in calls) + + +def test_jd_cross_month_week_uses_visible_iso_week_row() -> None: + source = inspect.getsource(jd_data_collector.select_last_week_any_day) + + assert "target_year = last_sunday.year" in source + assert "jmtd-date-picker-header-btn-prev-month" in source + assert "jmtd-date-picker-week-number-cell" in source + assert "clicked_iso_week_row_" in source + assert "jmtd-date-picker-combo-tag-echo" in source + + +def test_jd_navigation_retries_transient_failures() -> None: + class Page: + def __init__(self) -> None: + self.attempts = 0 + self.waits: list[int] = [] + + def goto(self, *_args, **_kwargs) -> None: + self.attempts += 1 + if self.attempts < 3: + raise RuntimeError("net::ERR_PROXY_CONNECTION_FAILED") + + def wait_for_timeout(self, timeout_ms: int) -> None: + self.waits.append(timeout_ms) + + page = Page() + jd_data_collector.goto_with_retry(page, "https://example.invalid") + + assert page.attempts == 3 + assert page.waits == [3000, 6000] + + +def test_jd_waits_for_summary_card_hydration_after_week_switch() -> None: + source = inspect.getsource(jd_data_collector.step_shop_star_and_trade) + + assert "page.wait_for_timeout(8000)" in source + assert ".sz-summary-item" in source + assert ".sz-summary-item-index [data-value]" in source + assert "real-time-module-right-shop-level-content-my-level-data-value" in source + + +def test_jd_star_accepts_current_bare_score_value() -> None: + assert jd_data_collector._extract_jd_star("5.0") == "5.0星" + assert jd_data_collector._extract_jd_star("5.0详情") == "5.0星" + assert jd_data_collector._extract_jd_star("未获取到") == "未获取到" + + +def test_week_label_is_canonical_and_historical_padding_is_equivalent() -> None: + week_label, month_label, *_ = format_week("2026-07-27", "2026-08-02") + + assert week_label == "7月第5周（7.27-8.2）" + assert month_label == "2026.07月" + assert normalize_week_label("7月第5周（7.27-8.02）") == normalize_week_label( + week_label + ) + assert normalize_week_label(" 7月第5周 ( 7.27 - 8.02 ) ") == normalize_week_label( + week_label + ) + assert week_search_prefix("7月第5周（7.27-8.02）") == "7月第5周" + + +def test_find_existing_record_matches_padding_but_not_another_year( + monkeypatch, +) -> None: + observed = {} + + def fake_run(_node, _script, args, **_kwargs): + observed["payload"] = json.loads(args[-1]) + return { + "data": { + "fields": ["time", "platform", "month"], + "data": [ + ["7月第5周（7.27-8.02）", ["DY"], ["2025.07月"]], + ["7月第5周（7.27-8.02）", ["DY"], ["2026.07月"]], + ], + "record_id_list": ["old-year", "same-week"], + } + } + + monkeypatch.setattr(shop_lark_cli, "run_lark_cli", fake_run) + fields = { + "时间": "time", + "平台": "platform", + "月份": "month", + } + + record_id = shop_lark_cli.find_existing_record( + "node", + "script", + "base", + "table", + fields, + "7月第5周（7.27-8.2）", + "DY", + month_str="2026.07月", + ) + + assert record_id == "same-week" + assert observed["payload"]["keyword"] == "7月第5周" + + +def test_dy_shop_feishu_sales_uses_confirmed_net_sales_amount() -> None: + extracted = shop_base_writer._extract_dy_metrics( + { + "date_range": "2026/07/27 - 2026/08/02", + "metrics": { + "成交金额": "¥413,988.46", + "成交退款金额": "¥119,095.48", + }, + "derived_metrics": {"raw": {"销售额": 294892.98}}, + "experience_score": {"score": "95"}, + } + ) + + assert extracted["sales_amount"] == 294892.98 + assert extracted["star_rating"] == 95.0 + + +@pytest.mark.parametrize( + "invalid_value", + ["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, "5星"], +) +def test_jd_shop_quality_rejects_invalid_or_negative_sales(invalid_value) -> None: + valid, reason = validate_shop_payload( + run_shop.JD_PLATFORM, + { + "transaction": {"成交金额": {"value": invalid_value}}, + "star_rating_raw": "5.0星", + }, + ) + + assert valid is False + assert "成交金额" in reason + + +@pytest.mark.parametrize( + "invalid_value", + ["加载失败", "未采集到", "¥", "abc", float("nan"), -0.01, 5.01, "¥4.9"], +) +def test_jd_shop_quality_rejects_invalid_or_out_of_range_score( + invalid_value, +) -> None: + valid, reason = validate_shop_payload( + run_shop.JD_PLATFORM, + { + "transaction": {"成交金额": {"value": "￥311,671.16"}}, + "star_rating_raw": invalid_value, + }, + ) + + assert valid is False + assert "店铺评分" in reason + + +@pytest.mark.parametrize( + ("field", "invalid_value", "reason_text"), + [ + ("gross", "加载失败", "成交金额"), + ("gross", "¥", "成交金额"), + ("gross", float("nan"), "成交金额"), + ("gross", -1, "成交金额"), + ("net", "abc", "净销售额"), + ("net", float("nan"), "净销售额"), + ("net", -1, "净销售额"), + ("score", "未采集到", "店铺评分"), + ("score", float("nan"), "店铺评分"), + ("score", -1, "店铺评分"), + ("score", 100.01, "店铺评分"), + ], +) +def test_dy_shop_quality_rejects_invalid_metrics( + field, + invalid_value, + reason_text, +) -> None: + gross = "¥413,988.46" + net = 294892.98 + score = "95分" + if field == "gross": + gross = invalid_value + elif field == "net": + net = invalid_value + else: + score = invalid_value + + valid, reason = validate_shop_payload( + run_shop.DY_PLATFORM, + { + "metrics": {"成交金额": gross}, + "derived_metrics": {"raw": {"销售额": net}}, + "experience_score": {"score": score}, + }, + ) + + assert valid is False + assert reason_text in reason + + +def test_shop_quality_accepts_strict_real_metric_formats() -> None: + assert validate_shop_payload( + run_shop.JD_PLATFORM, + { + "transaction": {"成交金额": {"value": "￥311,671.16"}}, + "star_rating_raw": "5.0星", + }, + ) == (True, "") + assert validate_shop_payload( + run_shop.JD_PLATFORM, + { + "transaction": {"成交金额": {"value": "￥264,650.81"}}, + "star_rating_raw": ( + "店铺星级星级运营店铺星级我的店铺星级4.7详情较昨日持平" + ), + }, + ) == (True, "") + assert validate_shop_payload( + run_shop.DY_PLATFORM, + { + "metrics": {"成交金额": "¥413,988.46"}, + "derived_metrics": {"raw": {"销售额": 294892.98}}, + "experience_score": {"score": "95分"}, + }, + ) == (True, "") + + +def test_jd_shop_feishu_rejects_missing_metrics_instead_of_writing_zero() -> None: + with pytest.raises(ValueError, match="缺少有效成交金额、店铺评分"): + shop_base_writer._extract_jd_metrics( + { + "week_start": "2026-07-27", + "week_end": "2026-08-02", + "star_rating_raw": "未获取到", + "transaction": {"成交金额": {"value": "N/A"}}, + } + ) + + +def test_jd_current_dom_metric_text_and_star_are_normalized() -> None: + assert jd_data_collector._normalize_jd_metric_text( + "￥\n3\n1\n1\n,\n6\n7\n1\n.\n1\n6" + ) == "￥311,671.16" + assert jd_data_collector._extract_jd_star( + "店铺星级\n星级运营\n我的店铺星级\n5.0详情\n较昨日持平" + ) == "5.0星" + + +def test_jd_week_event_suffix_uses_iso_week() -> None: + assert jd_data_collector._week_event_suffix( + jd_data_collector.datetime(2026, 7, 27) + ) == "2026-31" + + +def test_runner_rejects_success_without_fresh_artifact(monkeypatch) -> None: + marker = ("D:/old.json", 1, 10) + monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: marker) + + result = run_shop._run_with_fresh_artifact( + "JD", + lambda: ("JD", 0, "collector output"), + ) + + assert result[1] == 1 + assert result[3] == "" + assert "no fresh metrics file" in result[2] + + +def test_runner_passes_only_the_new_artifact(monkeypatch) -> None: + markers = iter((None, ("D:/new.json", 2, 20))) + monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers)) + monkeypatch.setattr( + run_shop, + "_validate_metrics_artifact", + lambda _platform, _path: (True, ""), + ) + + result = run_shop._run_with_fresh_artifact( + "JD", + lambda: ("JD", 0, "ok"), + ) + + assert result == ("JD", 0, "ok", "D:/new.json") + + +def test_runner_rejects_fresh_but_empty_metrics_artifact( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "dy.json" + artifact.write_text('{"metrics": {}}', encoding="utf-8") + markers = iter((None, (str(artifact), 2, artifact.stat().st_size))) + monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers)) + + result = run_shop._run_with_fresh_artifact( + run_shop.DY_PLATFORM, + lambda: (run_shop.DY_PLATFORM, 0, "collector output"), + ) + + assert result[1] == 1 + assert result[3] == "" + assert "shop metrics are empty" in result[2] + + +def test_runner_rejects_jd_placeholder_metrics_before_external_writes( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "jd.json" + artifact.write_text( + json.dumps( + { + "star_rating_raw": "未获取到", + "transaction": {"成交金额": {"value": "N/A"}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + marker = (str(artifact), artifact.stat().st_mtime_ns, artifact.stat().st_size) + markers = iter((None, marker)) + monkeypatch.setattr(run_shop, "_artifact_marker", lambda _platform: next(markers)) + + result = run_shop._run_with_fresh_artifact( + run_shop.JD_PLATFORM, + lambda: (run_shop.JD_PLATFORM, 0, "collector output"), + ) + + assert result[1] == 1 + assert result[3] == "" + assert "missing valid 成交金额/店铺评分" in result[2] + + +def test_direct_jd_persistence_rejects_placeholder_metrics_before_connecting( + monkeypatch, + tmp_path: Path, +) -> None: + artifact = tmp_path / "jd-invalid.json" + artifact.write_text( + json.dumps( + { + "star_rating_raw": "未获取到", + "transaction": {"成交金额": {"value": "N/A"}}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + monkeypatch.setattr(shop_db, "DB_ENABLED", True) + monkeypatch.setattr( + shop_db, + "get_conn", + lambda: pytest.fail("invalid payload must not connect to PostgreSQL"), + ) + + with pytest.raises(ValueError, match="JD shop data quality check failed"): + shop_db.persist_jd_shop(str(artifact)) + + +def test_jd_pg_transaction_value_does_not_depend_on_item_key_order() -> None: + metrics = { + "shop": "test-shop", + "week_start": "2026-07-27", + "week_end": "2026-08-02", + "star_rating_raw": "5.0", + "transaction": { + "成交金额": { + "环比": "-19.8%", + "同行同级均值": "¥999.00", + "value": "¥311,671.16", + } + }, + } + + row = shop_db._build_jd_shop_row(metrics) + + assert row is not None + assert row["deal_amount"] == 311671.16 + assert row["deal_amount_qoq"] == "-19.8%" + assert row["deal_amount_peer_avg"] == "¥999.00" + + +@pytest.mark.parametrize( + ("platform", "persist_name", "platform_code"), + [ + (run_shop.JD_PLATFORM, "persist_jd_shop", "JD"), + (run_shop.DY_PLATFORM, "persist_dy_shop", "DY"), + ], +) +@pytest.mark.parametrize("persisted_count", [0, -1]) +def test_shop_nonpositive_persistence_stops_feishu_write( + monkeypatch, + platform: str, + persist_name: str, + platform_code: str, + persisted_count: int, +) -> None: + monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "") + monkeypatch.setattr(run_shop, persist_name, lambda _path: persisted_count) + monkeypatch.setattr( + run_shop, + "write_shop", + lambda *_args: pytest.fail("Feishu must not run after DB failure"), + ) + + errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json") + + assert errors == [f"{platform_code} DB 持久化失败: 未写入任何记录"] + + +@pytest.mark.parametrize( + ("platform", "persist_name", "platform_code"), + [ + (run_shop.JD_PLATFORM, "persist_jd_shop", "JD"), + (run_shop.DY_PLATFORM, "persist_dy_shop", "DY"), + ], +) +def test_shop_false_writer_result_is_reported( + monkeypatch, + platform: str, + persist_name: str, + platform_code: str, +) -> None: + monkeypatch.setattr(run_shop, "ensure_ai_analysis", lambda *_args: "") + monkeypatch.setattr(run_shop, persist_name, lambda _path: 1) + monkeypatch.setattr(run_shop, "write_shop", lambda *_args: False) + + errors = run_shop._persist_and_write(platform, True, "D:/fresh-shop.json") + + assert errors == [f"{platform_code} 飞书写入失败: writer 返回 False"] + + +def test_runner_propagates_cookie_skip_without_writes(monkeypatch) -> None: + monkeypatch.setattr( + sys, + "argv", + ["run_shop", "--platform", "dy", "--skip-feishu"], + ) + monkeypatch.setattr(run_shop, "setup_runner_logging", lambda _name: "test.log") + monkeypatch.setattr( + run_shop, + "skip_feishu_for_acceptance", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + run_shop, + "_run_with_fresh_artifact", + lambda platform, _collector: ( + platform, + COOKIE_SKIP_EXIT_CODE, + "expired", + "", + ), + ) + monkeypatch.setattr( + run_shop, + "_persist_and_write", + lambda *_args, **_kwargs: pytest.fail("skipped platform must not write"), + ) + + assert run_shop.main() == COOKIE_SKIP_EXIT_CODE + + +def test_shop_analysis_uses_local_analyzer_hermes(monkeypatch) -> None: + observed = {} + + class Response: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"choices": [{"message": {"content": "analysis"}}]} + + class Session: + trust_env = True + + def post(self, url, **kwargs): + observed["session"] = self + observed["url"] = url + observed.update(kwargs) + return Response() + + monkeypatch.setenv( + "HERMES_ANALYZER_URL", + "http://127.0.0.1:8642/v1/chat/completions", + ) + monkeypatch.setenv("HERMES_ANALYZER_TOKEN", "test-token") + monkeypatch.setattr(shop_base_writer.requests, "Session", Session) + + assert shop_base_writer._call_hermes("test prompt") == "analysis" + assert observed["url"].startswith("http://127.0.0.1:8642/") + assert observed["session"].trust_env is False + assert observed["headers"]["Authorization"] == "Bearer test-token" + + +def test_shop_analysis_falls_back_to_local_cli_when_gateway_requires_auth( + monkeypatch, +) -> None: + class Response: + status_code = 401 + + def raise_for_status(self) -> None: + error = shop_base_writer.requests.HTTPError("unauthorized") + error.response = self + raise error + + class Session: + trust_env = True + + def post(self, *_args, **_kwargs): + return Response() + + monkeypatch.setattr(shop_base_writer.requests, "Session", Session) + monkeypatch.setattr( + shop_base_writer, + "_call_hermes_cli", + lambda prompt, *, timeout: f"cli:{prompt}:{timeout}", + ) + + assert shop_base_writer._call_hermes("prompt", timeout=12) == "cli:prompt:12" + + +def test_shop_analysis_reads_analyzer_profile_gateway_key( + monkeypatch, + tmp_path: Path, +) -> None: + profile = tmp_path / "profiles" / "data-analyzer" + profile.mkdir(parents=True) + (profile / ".env").write_text( + "API_SERVER_KEY=profile-key\n", + encoding="utf-8", + ) + observed = {} + + class Response: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return {"choices": [{"message": {"content": "analysis"}}]} + + class Session: + trust_env = True + + def post(self, _url, **kwargs): + observed.update(kwargs) + return Response() + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("HERMES_ANALYZER_TOKEN", raising=False) + monkeypatch.delenv("GYXX_HERMES_API_KEY", raising=False) + monkeypatch.delenv("HERMES_API_KEY", raising=False) + monkeypatch.setattr(shop_base_writer.requests, "Session", Session) + + assert shop_base_writer._call_hermes("prompt") == "analysis" + assert observed["headers"]["Authorization"] == "Bearer profile-key" + + +def test_jd_shop_uses_bound_profile_port_and_persists_state( + monkeypatch, + tmp_path: Path, +) -> None: + page = object() + observed = {} + + class Context: + pages = [page] + closed = False + + def cookies(self): + return [{"name": "session", "value": "saved"}] + + def storage_state(self, *, path: str) -> None: + Path(path).write_text('{"cookies": []}', encoding="utf-8") + + def close(self) -> None: + self.closed = True + + context = Context() + + class Chromium: + def launch_persistent_context(self, **kwargs): + observed.update(kwargs) + return context + + class Playwright: + chromium = Chromium() + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + cookie_file = tmp_path / "state" / "cookies.json" + storage_file = tmp_path / "state" / "storage.json" + profile = tmp_path / "profile" + monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile)) + monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22105") + monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file)) + monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file)) + monkeypatch.setattr(jd_data_collector, "sync_playwright", Playwright) + monkeypatch.setattr(jd_data_collector, "step_login", lambda *_args: None) + monkeypatch.setattr( + jd_data_collector, + "step_enter_shangzhi", + lambda *_args: page, + ) + monkeypatch.setattr( + jd_data_collector, + "step_shop_star_and_trade", + lambda *_args: ({}, {}), + ) + monkeypatch.setattr(jd_data_collector, "step_deal_summary", lambda *_args: {}) + monkeypatch.setattr(jd_data_collector, "step_merchant_ranking", lambda *_args: {}) + monkeypatch.setattr(jd_data_collector, "step_jzt_non_full_site", lambda *_args: {}) + monkeypatch.setattr(jd_data_collector, "step_jzt_full_site", lambda *_args: {}) + monkeypatch.setattr( + jd_data_collector, + "generate_report", + lambda *_args: str(tmp_path / "report.json"), + ) + + assert jd_data_collector.main(["--password", "test-only"]) == 0 + assert observed["user_data_dir"] == str(profile) + assert "--remote-debugging-port=22105" in observed["args"] + assert "--no-proxy-server" in observed["args"] + assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "session" + assert storage_file.is_file() + assert context.closed is True + + +def test_jd_peer_uses_bound_profile_port_and_persists_state( + monkeypatch, + tmp_path: Path, +) -> None: + page = object() + observed = {} + + class Context: + pages = [page] + closed = False + + def cookies(self): + return [{"name": "peer-session", "value": "saved"}] + + def storage_state(self, *, path: str) -> None: + Path(path).write_text('{"cookies": []}', encoding="utf-8") + + def close(self) -> None: + self.closed = True + + context = Context() + + class Chromium: + def launch_persistent_context(self, **kwargs): + observed.update(kwargs) + return context + + class Playwright: + chromium = Chromium() + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + cookie_file = tmp_path / "state" / "cookies.json" + storage_file = tmp_path / "state" / "storage.json" + profile = tmp_path / "profile" + monkeypatch.setenv("GYXX_BROWSER_PROFILE_DIR", str(profile)) + monkeypatch.setenv("GYXX_BROWSER_CDP_PORT", "22106") + monkeypatch.setenv("GYXX_BROWSER_COOKIE_FILE", str(cookie_file)) + monkeypatch.setenv("GYXX_BROWSER_STORAGE_STATE_FILE", str(storage_file)) + monkeypatch.setattr(jd_peer_store_data_collector, "sync_playwright", Playwright) + monkeypatch.setattr(jd_peer_store_data_collector, "step_login", lambda *_args: None) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_enter_shangzhi", + lambda *_args: page, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_return_old_version", + lambda *_args: None, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_goto_vender_ranks", + lambda *_args: None, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_collect_competitor_data", + lambda *_args: {}, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "save_results", + lambda *_args: str(tmp_path / "peer.json"), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "step_goto_product_ranks", + lambda *_args: None, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "collect_new_products", + lambda *_args: {}, + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "save_new_product_results", + lambda *_args: None, + ) + + assert jd_peer_store_data_collector.main(["--password", "test-only"]) == 0 + assert observed["user_data_dir"] == str(profile) + assert "--remote-debugging-port=22106" in observed["args"] + assert json.loads(cookie_file.read_text(encoding="utf-8"))[0]["name"] == "peer-session" + assert storage_file.is_file() + assert context.closed is True + + +def test_jd_peer_selects_week_once_before_category_loop(monkeypatch) -> None: + calls = [] + monkeypatch.setattr( + jd_peer_store_data_collector, + "SHOPS", + ["peer-shop"], + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "CATEGORIES", + [("男士双肩包", "男包")], + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "select_last_week_any_day", + lambda _page: calls.append("initial-week"), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "search_shop", + lambda *_args: calls.append("search"), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "switch_category", + lambda *_args: calls.append("category"), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "reselect_last_week", + lambda _page: calls.append("reselect-week"), + ) + monkeypatch.setattr( + jd_peer_store_data_collector, + "extract_ranking_table", + lambda _page: [], + ) + + page = SimpleNamespace(wait_for_timeout=lambda *_args: None) + jd_peer_store_data_collector.step_collect_competitor_data(page) + + assert calls == ["initial-week", "search", "category"] + + +def test_jd_peer_product_rank_navigation_does_not_wait_for_network_idle() -> None: + observed: dict[str, object] = {} + + class Page: + def goto(self, url: str, **kwargs) -> None: + observed["url"] = url + observed.update(kwargs) + + def wait_for_timeout(self, timeout_ms: int) -> None: + observed["settle_ms"] = timeout_ms + + jd_peer_store_data_collector.step_goto_product_ranks(Page()) + + assert observed == { + "url": "https://sz.jd.com/szweb/sz/view/industryMarket/productRanks.html", + "wait_until": "domcontentloaded", + "timeout": 60000, + "settle_ms": 8000, + } + + +def test_jd_self_operated_reports_date_not_yet_available() -> None: + page = MagicMock() + mode = MagicMock() + mode.count.return_value = 0 + date_picker = MagicMock() + dropdown = MagicMock() + enabled_target = MagicMock() + disabled_target = MagicMock() + selectable = MagicMock() + enabled_target.count.return_value = 0 + disabled_target.count.return_value = 1 + selectable.evaluate_all.return_value = [ + "2026-08-01", + "2026-08-02", + None, + ] + + def locate(selector: str): + if selector == ".jd-select-selection-item": + return SimpleNamespace(first=mode) + if selector == ".jd-picker-range": + return SimpleNamespace(first=date_picker) + if selector == ".jd-picker-dropdown:visible": + return SimpleNamespace(first=dropdown) + raise AssertionError(selector) + + def locate_dropdown(selector: str): + if selector == ( + "td[title].jd-picker-cell-in-view:not(.jd-picker-cell-disabled)" + ): + return selectable + if selector.endswith(":not(.jd-picker-cell-disabled)"): + return enabled_target + if selector.endswith(".jd-picker-cell-disabled"): + return disabled_target + raise AssertionError(selector) + + page.locator.side_effect = locate + dropdown.locator.side_effect = locate_dropdown + + with pytest.raises(RuntimeError, match="当前最晚可选日期为 2026-08-02"): + jd_self_operated_brand_daily.set_single_day( + page, + date(2026, 8, 3), + ) + + +def test_tmall_returns_failure_when_browser_cannot_start(monkeypatch) -> None: + monkeypatch.setattr(taobao_sycm, "sync_cookies_from_daily_profile", lambda *_args: 0) + monkeypatch.setattr(taobao_sycm, "connect_browser", lambda: (None, None)) + + assert taobao_sycm.main() == 1 + + +def test_tmall_profile_seed_preserves_cookie_encryption_state(tmp_path: Path) -> None: + source = tmp_path / "source" + target = tmp_path / "target" + (source / "Default" / "Network").mkdir(parents=True) + (source / "Local State").write_text("encryption-state", encoding="utf-8") + (source / "Default" / "Network" / "Cookies").write_bytes(b"cookie-db") + + copied = taobao_sycm.sync_cookies_from_daily_profile(source, target) + + assert copied >= 2 + assert (target / "Local State").read_text(encoding="utf-8") == "encryption-state" + assert (target / "Default" / "Network" / "Cookies").read_bytes() == b"cookie-db" + + +def test_tmall_close_always_releases_bound_cdp(monkeypatch) -> None: + closed = [] + killed = [] + monkeypatch.setattr(taobao_sycm, "DEBUG_PORT", 22107) + monkeypatch.setattr(taobao_sycm, "_kill_port_listeners", killed.append) + + taobao_sycm._close_session(SimpleNamespace(close=lambda: closed.append(True))) + + assert closed == [True] + assert killed == [22107] diff --git a/tests/modules/shop_intelligence/test_shop_nested_child_bindings.py b/tests/modules/shop_intelligence/test_shop_nested_child_bindings.py new file mode 100644 index 0000000..9a0ba33 --- /dev/null +++ b/tests/modules/shop_intelligence/test_shop_nested_child_bindings.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path +from types import SimpleNamespace + +import psutil +import pytest + +from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.shop_intelligence.runners import utils + + +def test_runner_rebinds_browser_environment_after_caller_overrides(monkeypatch) -> None: + observed: dict[str, object] = {} + + def fake_rebind(target, base_environment): + observed["target"] = Path(target) + observed["base"] = dict(base_environment) + return { + **base_environment, + "GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py", + "GYXX_BROWSER_CDP_PORT": "child-port", + "GYXX_BROWSER_PROFILE_DIR": "child-profile", + "GYXX_BROWSER_COOKIE_FILE": "child-cookie", + "GYXX_BROWSER_STORAGE_STATE_FILE": "child-storage", + } + + def fake_run(command, *, environment, timeout): + observed["command"] = command + observed["environment"] = environment + observed["timeout"] = timeout + return subprocess.CompletedProcess(command, 0, stdout=b"ok", stderr=b"") + + monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind) + monkeypatch.setattr(utils, "_run_managed_command", fake_run) + target = str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py") + + code, output = utils.run_collector( + "example", + target, + [], + env={"KEEP": "yes", "GYXX_BROWSER_CDP_PORT": "caller-port"}, + ) + + environment = observed["environment"] + assert code == 0 + assert output == "ok" + assert observed["target"] == Path(target) + assert observed["base"]["GYXX_BROWSER_CDP_PORT"] == "caller-port" + assert environment["KEEP"] == "yes" + assert environment["GYXX_BROWSER_CDP_PORT"] == "child-port" + assert environment["GYXX_BROWSER_PROFILE_DIR"] == "child-profile" + assert environment["GYXX_BROWSER_COOKIE_FILE"] == "child-cookie" + assert environment["GYXX_BROWSER_STORAGE_STATE_FILE"] == "child-storage" + assert observed["timeout"] == 3600 + + +def test_runner_skips_invalid_cookie_before_starting_child(monkeypatch) -> None: + observed: dict[str, object] = {} + + def fake_rebind(target, base_environment): + return { + **base_environment, + "GYXX_WORKFLOW_ACCEPTANCE": "1", + "GYXX_SCRIPT_ID": "shop_intelligence:collectors/example.py", + } + + class FakePolicy: + enabled = True + + def preflight_cookie(self, binding, *, environment): + observed["binding"] = binding + observed["environment"] = environment + return SimpleNamespace(should_skip=True, reason="browser state is missing") + + monkeypatch.setattr(utils, "environment_for_child_script", fake_rebind) + monkeypatch.setattr(utils, "binding_from_environment", lambda env: "binding") + monkeypatch.setattr(utils, "current_acceptance_policy", lambda env: FakePolicy()) + monkeypatch.setattr( + utils, + "_run_managed_command", + lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("child process must not start") + ), + ) + + code, output = utils.run_collector( + "example", + str(Path(utils.PROJECT_ROOT) / "collectors" / "example.py"), + [], + ) + + assert code == COOKIE_SKIP_EXIT_CODE + assert output == "browser state is missing" + assert observed["binding"] == "binding" + assert observed["environment"]["GYXX_WORKFLOW_ACCEPTANCE"] == "1" + + +def test_runner_timeout_terminates_collector_grandchild_before_return( + monkeypatch, + tmp_path: Path, +) -> None: + identity_file = tmp_path / "grandchild.json" + marker_file = tmp_path / "grandchild-finished.txt" + grandchild_code = ( + "import pathlib,time; " + "time.sleep(2.8); " + f"pathlib.Path({str(marker_file)!r}).write_text('alive', encoding='utf-8')" + ) + collector = tmp_path / "collector.py" + collector.write_text( + "\n".join( + ( + "import json, pathlib, psutil, subprocess, sys, time", + f"child = subprocess.Popen([sys.executable, '-c', {grandchild_code!r}], close_fds=False)", + f"pathlib.Path({str(identity_file)!r}).write_text(json.dumps({{'pid': child.pid, 'created': psutil.Process(child.pid).create_time()}}), encoding='utf-8')", + "time.sleep(10)", + ) + ), + encoding="utf-8", + ) + + monkeypatch.setattr( + utils, + "environment_for_child_script", + lambda _target, environment: environment, + ) + monkeypatch.setattr( + utils, + "current_acceptance_policy", + lambda _environment: SimpleNamespace(enabled=False), + ) + + code, output = utils.run_collector( + "timeout-probe", + str(collector), + [], + env=dict(os.environ), + timeout=2, + ) + + assert code == 1 + assert "timeout-probe" in output + assert identity_file.is_file() + identity = json.loads(identity_file.read_text(encoding="utf-8")) + try: + descendant = psutil.Process(identity["pid"]) + same_process_is_live = ( + descendant.create_time() + == pytest.approx(identity["created"], abs=0.01) + and descendant.is_running() + and descendant.status() != psutil.STATUS_ZOMBIE + ) + except psutil.Error: + same_process_is_live = False + assert same_process_is_live is False + time.sleep(1.0) + assert marker_file.exists() is False + + +def test_runner_redacts_sensitive_command_arguments() -> None: + command = [ + "python", + "collector.py", + "--password", + "placeholder", + "--api-key=placeholder", + "--output", + "safe-path", + ] + + assert utils._redact_command(command) == [ + "python", + "collector.py", + "--password", + "[REDACTED]", + "--api-key=[REDACTED]", + "--output", + "safe-path", + ] + + +def test_acceptance_skips_entire_feishu_writer_path(monkeypatch) -> None: + observed: list[str] = [] + + class FakePolicy: + def skip_feishu_write(self, operation): + observed.append(operation) + return True + + monkeypatch.setattr(utils, "current_acceptance_policy", lambda: FakePolicy()) + + assert utils.skip_feishu_for_acceptance("shop.writer", requested=False) is True + assert observed == ["shop.writer"] + + +def test_explicit_feishu_skip_does_not_need_policy(monkeypatch) -> None: + monkeypatch.setattr( + utils, + "current_acceptance_policy", + lambda: (_ for _ in ()).throw(AssertionError("policy must not be loaded")), + ) + + assert utils.skip_feishu_for_acceptance("shop.writer", requested=True) is True diff --git a/tests/modules/shop_intelligence/test_source_business_logic_parity.py b/tests/modules/shop_intelligence/test_source_business_logic_parity.py new file mode 100644 index 0000000..9570157 --- /dev/null +++ b/tests/modules/shop_intelligence/test_source_business_logic_parity.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import ast +import hashlib +import os +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +TARGET_COLLECTOR_ROOT = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "shop_intelligence" + / "collectors" +) +SOURCE_COLLECTOR_ROOT = Path( + os.getenv("GYXX_SHOP_SOURCE_ROOT", r"D:\shop-data-flow") +) / "collectors" + +# These functions implement page navigation, date/category selection, extraction, +# and iteration order. Portability adaptations belong in imports, configuration, +# persistence, browser-session setup, and main/runner boundaries instead. +CORE_BUSINESS_FUNCTIONS = { + "jd_data_collector.py": ( + "select_last_week_any_day", + "select_last_week_in_old_datepicker", + "step_enter_shangzhi", + "step_jzt_full_site", + "step_jzt_non_full_site", + "step_merchant_ranking", + "step_shop_star_and_trade", + "switch_category", + ), + "jd_peer_store_data_collector.py": ( + "clear_and_search_brand", + "collect_new_products", + "reselect_last_week", + "search_product_by_name", + "select_last_week_any_day", + "select_product_category", + "step_enter_shangzhi", + ), + "dy_store_competitor_store_scraping.py": ( + "_check_fxg_login", + "_collect_competitor_rankings", + "_collect_experience_score", + "_ensure_fxg_login", + "_extract_experience_score", + "_open_market_rank_shop_page", + "_select_menu_and_date", + ), +} + +# SHA-256 of ast.dump(function, include_attributes=False) from the reviewed +# D:\shop-data-flow source. A source change must be reviewed before updating this +# snapshot; copying a new hash merely to make the test pass defeats the guard. +SOURCE_AST_SHA256 = { + ("jd_data_collector.py", "select_last_week_any_day"): ( + "0c85d64e6d47b08a5971caec3afe391271fd6bc94d86bab65f6ebae9655b8a6f" + ), + ("jd_data_collector.py", "select_last_week_in_old_datepicker"): ( + "ff542b968c43b92304ccb15a2282acf3d64ab7258dee020bae51f1c7f5f5f973" + ), + ("jd_data_collector.py", "step_enter_shangzhi"): ( + "6ba9e2df11ecbf0aae88fb2ed3cab8abe5157121418a0426384f8c43368b9362" + ), + ("jd_data_collector.py", "step_jzt_full_site"): ( + "0cf4276039e96650879ba6fff8822322245dc581ad3020785ed6cb8a84e11e8f" + ), + ("jd_data_collector.py", "step_jzt_non_full_site"): ( + "2a911597995c4d9a2cc47b106d6a8c5a811932eea0d83e928b0dcafcdb6864a5" + ), + ("jd_data_collector.py", "step_merchant_ranking"): ( + "027626587724aa224fca6146dae61ea505374e78ca36603c96769b36725dfac7" + ), + ("jd_data_collector.py", "step_shop_star_and_trade"): ( + "2e54324b6091e961eb6fbc704aca2867a91ce829709c2824e8a6ee8fa2260790" + ), + ("jd_data_collector.py", "switch_category"): ( + "f6dfdc7025958cbb85476c31b4930f2b57a8f67316c3000a8e38f5d49acb6aef" + ), + ("jd_peer_store_data_collector.py", "clear_and_search_brand"): ( + "32327c849483a44aa8c2b8cab8210f8e80767d3e2b86fd6789d855669062dc00" + ), + ("jd_peer_store_data_collector.py", "collect_new_products"): ( + "c2cbf6528a756d4fb26f5b8794177cbcb37790584f85660eff0a851ecffc8f58" + ), + ("jd_peer_store_data_collector.py", "reselect_last_week"): ( + "5897bd8afb8bf0604daace2a13628ea002f753afd8de7956246a4ee18e89cbd9" + ), + ("jd_peer_store_data_collector.py", "search_product_by_name"): ( + "2fd758156e716d2c666b8c1c1d9edb0c72cad83c56c4c44b09900bd7eb0bd940" + ), + ("jd_peer_store_data_collector.py", "select_last_week_any_day"): ( + "0a9b5fad4ea111e7b2afe39241508b5d036b84006091188b95ac85b652e7e3f9" + ), + ("jd_peer_store_data_collector.py", "select_product_category"): ( + "ff947df3e6e455176d723c93fb2af06ab92a72ac9174f4a5ae2db306201d5182" + ), + ("jd_peer_store_data_collector.py", "step_enter_shangzhi"): ( + "d6bfb283c00c842a129d8f81f0da61cae752e421e2dafaffdb87fd20603f0f2d" + ), + ("dy_store_competitor_store_scraping.py", "_collect_competitor_rankings"): ( + "de55dded8a9341b955e000e4fafb8b4914533b25e06dc2aca930535836cfc0b5" + ), + ("dy_store_competitor_store_scraping.py", "_check_fxg_login"): ( + "1928b38f6c167971139b147565b51e727e65a74e49ba21fe40d47711a7641e6a" + ), + ("dy_store_competitor_store_scraping.py", "_collect_experience_score"): ( + "4609852591005c5b3ba8f9704b4db49918173f846176d9e50da6d4c8ee07914a" + ), + ("dy_store_competitor_store_scraping.py", "_ensure_fxg_login"): ( + "ae69c5c58563c6cdb97fd6af380a24e2d275ec6623038a58b5e15a7bda2777f4" + ), + ("dy_store_competitor_store_scraping.py", "_extract_experience_score"): ( + "5a28dbc7fa627fea384dae3d5f9b85fb0542b585c7312918ef355810209089df" + ), + ("dy_store_competitor_store_scraping.py", "_open_market_rank_shop_page"): ( + "b60acaeaf3b1073144f3f570eb539ad87945fd92f2577cf338b3b496ec542d35" + ), + ("dy_store_competitor_store_scraping.py", "_select_menu_and_date"): ( + "4ffb5e89e7a7af3d0af126adf420399a0017fba8e49b9185266baf7307e3e549" + ), +} + +# Deliberately reviewed target-only compatibility fixes. The source project stays +# read-only; these hashes ensure the migrated collector cannot drift beyond the +# current JD DOM adaptation without another explicit review. +TARGET_AST_SHA256_OVERRIDES = { + ("jd_data_collector.py", "select_last_week_any_day"): ( + "8f6aed11b39d8e19ec4146e9b32eee96f5977fdbef694311534346bacec5d847" + ), + ("jd_data_collector.py", "step_shop_star_and_trade"): ( + "77aa873543861f1fad63d0bd1ace60528ab20592b0bffef3d46ad192f2103161" + ), + ("jd_data_collector.py", "step_merchant_ranking"): ( + "187cd3cfffcb97a5bf9f1ed4dde0d117ee65f82c1cf431df4c7c9c9ee2a7b282" + ), + ("jd_data_collector.py", "step_jzt_non_full_site"): ( + "1bd53eba5d0123e63d2e628d9772c0efa57fc8d2f87d79a4f3c41a311132952d" + ), +} + +# These two source functions were reviewed after a production run proved that +# a failed category click silently copied the previous category's table. They +# intentionally diverge from D:\shop-data-flow and retain their own reviewed +# target snapshots so the defect cannot be reintroduced by a future sync. +REVIEWED_DEFECT_FIX_AST_SHA256 = { + ("jd_peer_store_data_collector.py", "step_collect_competitor_data"): ( + "bf6836516dc3e70d00c145acbdfa00b6ddeee480f9927d7236e82ba8b1430ff2" + ), + ("jd_peer_store_data_collector.py", "switch_category"): ( + "bb4dc034447666682e19dc8cce05903c92d53852669ac93cc6810b18ad310d13" + ), +} + +FUNCTION_CASES = tuple( + (filename, function_name) + for filename, function_names in CORE_BUSINESS_FUNCTIONS.items() + for function_name in function_names +) + + +def _call_name(node: ast.AST) -> str | None: + if not isinstance(node, ast.Call): + return None + if isinstance(node.func, ast.Name): + return node.func.id + if isinstance(node.func, ast.Attribute): + return node.func.attr + return None + + +def _is_session_expiry_guard(node: ast.stmt) -> bool: + if not isinstance(node, ast.If) or node.orelse: + return False + detector_names = {"_is_login_page_url", "_has_interactive_login_form"} + detectors = { + name + for child in ast.walk(node.test) + if (name := _call_name(child)) is not None + } + raises_session_expired = any( + isinstance(child, ast.Raise) + and child.exc is not None + and _call_name(child.exc) == "BrowserSessionExpiredError" + for statement in node.body + for child in ast.walk(statement) + ) + return bool(detectors & detector_names) and raises_session_expired + + +def _is_navigation_session_wrapper(node: ast.Try) -> bool: + if len(node.body) != 1 or node.orelse or node.finalbody or len(node.handlers) != 1: + return False + navigation = node.body[0] + if not ( + isinstance(navigation, ast.Expr) + and isinstance(navigation.value, ast.Call) + and _call_name(navigation.value) == "goto" + ): + return False + handler_body = node.handlers[0].body + return ( + len(handler_body) == 2 + and _is_session_expiry_guard(handler_body[0]) + and isinstance(handler_body[1], ast.Raise) + and handler_body[1].exc is None + ) + + +class _ApprovedBoundaryNormalizer(ast.NodeTransformer): + """Remove only the approved cookie/session-expiry adapter statements.""" + + def visit_If(self, node: ast.If) -> ast.AST | None: # noqa: N802 + if _is_session_expiry_guard(node): + return None + return self.generic_visit(node) + + def visit_Try(self, node: ast.Try) -> ast.AST | list[ast.AST]: # noqa: N802 + if _is_navigation_session_wrapper(node): + return [self.visit(statement) for statement in node.body] + return self.generic_visit(node) + + +def _function_ast(path: Path, function_name: str) -> str: + tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path)) + matches = [ + node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == function_name + ] + assert len(matches) == 1, f"expected exactly one top-level {function_name} in {path}" + normalized = _ApprovedBoundaryNormalizer().visit(matches[0]) + assert isinstance(normalized, (ast.FunctionDef, ast.AsyncFunctionDef)) + return ast.dump(normalized, annotate_fields=True, include_attributes=False) + + +def _ast_sha256(function_ast: str) -> str: + return hashlib.sha256(function_ast.encode("utf-8")).hexdigest() + + +def test_ast_snapshot_covers_exactly_the_reviewed_business_functions() -> None: + assert set(SOURCE_AST_SHA256) == set(FUNCTION_CASES) + assert set(SOURCE_AST_SHA256).isdisjoint(REVIEWED_DEFECT_FIX_AST_SHA256) + + +@pytest.mark.parametrize( + ("filename", "function_name"), + REVIEWED_DEFECT_FIX_AST_SHA256, + ids=lambda value: value, +) +def test_reviewed_local_defect_fixes_match_target_snapshot( + filename: str, + function_name: str, +) -> None: + target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name) + + assert _ast_sha256(target_ast) == REVIEWED_DEFECT_FIX_AST_SHA256[ + (filename, function_name) + ] + + +@pytest.mark.parametrize( + ("filename", "function_name"), + FUNCTION_CASES, + ids=lambda value: value, +) +def test_migrated_core_business_logic_matches_reviewed_ast_snapshot( + filename: str, + function_name: str, +) -> None: + target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name) + + expected = TARGET_AST_SHA256_OVERRIDES.get( + (filename, function_name), + SOURCE_AST_SHA256[(filename, function_name)], + ) + assert _ast_sha256(target_ast) == expected, ( + f"{filename}:{function_name} changed the reviewed source business logic; " + "only explicitly reviewed target compatibility adapters may differ" + ) + + +@pytest.mark.skipif( + not SOURCE_COLLECTOR_ROOT.is_dir(), + reason="local D:\\shop-data-flow source project is unavailable", +) +@pytest.mark.parametrize( + ("filename", "function_name"), + FUNCTION_CASES, + ids=lambda value: value, +) +def test_reviewed_snapshot_and_migration_match_current_local_source( + filename: str, + function_name: str, +) -> None: + source_ast = _function_ast(SOURCE_COLLECTOR_ROOT / filename, function_name) + target_ast = _function_ast(TARGET_COLLECTOR_ROOT / filename, function_name) + expected = SOURCE_AST_SHA256[(filename, function_name)] + + assert _ast_sha256(source_ast) == expected, ( + f"{filename}:{function_name} changed in D:\\shop-data-flow; review the " + "source change before deliberately refreshing this snapshot" + ) + override = TARGET_AST_SHA256_OVERRIDES.get((filename, function_name)) + if override: + assert _ast_sha256(target_ast) == override + else: + assert target_ast == source_ast, ( + f"{filename}:{function_name} no longer matches the current source project" + ) diff --git a/tests/test_acceptance_cookie_fail_closed.py b/tests/test_acceptance_cookie_fail_closed.py new file mode 100644 index 0000000..ae43118 --- /dev/null +++ b/tests/test_acceptance_cookie_fail_closed.py @@ -0,0 +1,493 @@ +from __future__ import annotations + +import importlib +import io +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.content_marketing.data.tools import ( + friday_relogin_parallel as weekly_relogin, +) + + +@pytest.fixture +def acceptance_environment( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Path: + evidence = tmp_path / "cookie-evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + return evidence + + +def _forbidden(message: str): + def fail(*_args, **_kwargs): + pytest.fail(message) + + return fail + + +def _config_file(tmp_path: Path) -> Path: + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "login": { + "url": "https://login.example.test/", + "username": "acceptance-user", + "headless": False, + } + } + ), + encoding="utf-8", + ) + return config + + +def _import_supply_script(module_name: str): + """Import legacy scripts without letting their stream wrapper close pytest IO.""" + + stdout = sys.stdout + stderr = sys.stderr + text_wrapper = io.TextIOWrapper + + def preserve_pytest_stream(buffer, *args, **kwargs): + if buffer is getattr(stdout, "buffer", None): + return stdout + if buffer is getattr(stderr, "buffer", None): + return stderr + return text_wrapper(buffer, *args, **kwargs) + + io.TextIOWrapper = preserve_pytest_stream # type: ignore[assignment,misc] + try: + return importlib.import_module( + f"gyxx_flow.modules.supply_chain.orchestrator.scripts.{module_name}" + ) + finally: + io.TextIOWrapper = text_wrapper + sys.stdout = stdout + sys.stderr = stderr + + +def test_weekly_relogin_skips_the_entire_qr_workflow( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + weekly_relogin.subprocess, + "Popen", + _forbidden("acceptance mode must not launch relogin subprocesses"), + ) + monkeypatch.setattr( + weekly_relogin, + "_send_lark", + _forbidden("acceptance mode must not send QR screenshots"), + ) + + assert weekly_relogin.main() == COOKIE_SKIP_EXIT_CODE + assert "SKIPPED_COOKIE" in capsys.readouterr().out + assert "content.relogin.weekly" in acceptance_environment.read_text( + encoding="utf-8" + ) + + +@pytest.mark.parametrize( + "module_name", + [ + "batch_rescrape_xiaohongshu", + "batch_rescrape_douyin", + "batch_rescrape_bilibili", + ], +) +def test_batch_rescrape_relogin_never_starts_a_qr_process( + module_name: str, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + module = importlib.import_module( + f"gyxx_flow.modules.content_marketing.data.tools.{module_name}" + ) + monkeypatch.setattr( + module.subprocess, + "call", + _forbidden("acceptance mode must not invoke a relogin script"), + ) + + assert module.do_relogin() is False + assert "SKIPPED_COOKIE" in capsys.readouterr().out + evidence = acceptance_environment.read_text(encoding="utf-8") + assert f"content.{module_name}.relogin" in evidence + + +@pytest.mark.parametrize( + "module_name", + [ + "batch_rescrape_xiaohongshu", + "batch_rescrape_douyin", + "batch_rescrape_bilibili", + ], +) +def test_batch_rescrape_propagates_cookie_skip_without_cooldown( + module_name: str, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = importlib.import_module( + f"gyxx_flow.modules.content_marketing.data.tools.{module_name}" + ) + monkeypatch.setattr( + module, + "collect_urls", + lambda: [ + {"id": index, "url": f"https://example.test/{index}", "title": ""} + for index in range(1, 4) + ], + ) + monkeypatch.setattr( + module.scraper, + "scrape_comments", + _forbidden_scrape, + ) + monkeypatch.setattr( + module.time, + "sleep", + _forbidden("acceptance cookie skips must not enter retry cooldown"), + ) + + assert module.main(dry_run=True) == COOKIE_SKIP_EXIT_CODE + + +def _forbidden_scrape(*_args, **_kwargs): + raise RuntimeError("cookie session is invalid") + + +@pytest.mark.parametrize( + ("module_name", "manager_name", "login_name"), + [ + ("ProductReplenishment", "JstManager", "login"), + ("PurchaseConfirmation", "ERPLoginManager", "login_with_playwright"), + ("PurchaseOrderUpdate", "ERPLoginManager", "login_with_playwright"), + ], +) +def test_supply_acceptance_without_credentials_never_starts_browser_login( + module_name: str, + manager_name: str, + login_name: str, + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_SUPPLY_ERP_PASSWORD", raising=False) + module = _import_supply_script(module_name) + manager_type = getattr(module, manager_name) + if module_name == "ProductReplenishment": + manager = manager_type(str(_config_file(tmp_path))) + else: + manager = manager_type(str(_config_file(tmp_path)), cdp_port=None) + monkeypatch.setattr( + module, + "sync_playwright", + _forbidden("acceptance mode must not start an embedded browser"), + ) + + login = getattr(manager, login_name) + assert login() is False + assert manager._acceptance_cookie_skipped is True + assert "SKIPPED_COOKIE" in acceptance_environment.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "module_name", + ["PurchaseConfirmation"], +) +def test_supply_missing_credentials_do_not_navigate_or_fill_login( + module_name: str, + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_SUPPLY_ERP_PASSWORD", raising=False) + module = _import_supply_script(module_name) + manager = module.ERPLoginManager(str(_config_file(tmp_path)), cdp_port=22555) + page = SimpleNamespace(context=SimpleNamespace(cookies=lambda: [])) + navigation_flags: list[bool] = [] + + class PlaywrightStarter: + def start(self): + return object() + + def prepare(_url, *, navigate=True): + navigation_flags.append(navigate) + return page + + monkeypatch.setattr(module, "sync_playwright", PlaywrightStarter) + monkeypatch.setattr(manager, "_prepare_login_page", prepare) + monkeypatch.setattr( + manager, + "_fill_first_visible", + _forbidden("acceptance mode must not fill login credentials"), + ) + + assert manager.login_with_playwright() is False + assert manager._acceptance_cookie_skipped is True + assert navigation_flags == [True] + + +def test_purchase_confirmation_uses_credential_fallback_but_skips_captcha( + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _import_supply_script("PurchaseConfirmation") + monkeypatch.setenv("GYXX_SUPPLY_ERP_PASSWORD", "configured-for-test") + manager = module.ERPLoginManager(str(_config_file(tmp_path)), cdp_port=None) + navigation_flags: list[bool] = [] + filled: list[str] = [] + + class Locator: + def __init__(self, selector: str): + self.selector = selector + + @property + def first(self): + return self + + def count(self): + return 1 if "captcha" in self.selector else 0 + + def is_visible(self): + return True + + class Page: + context = SimpleNamespace(cookies=lambda: []) + keyboard = SimpleNamespace(press=lambda _key: None) + + def goto(self, *_args, **_kwargs): + return None + + def locator(self, selector): + return Locator(selector) + + class PlaywrightStarter: + def start(self): + return object() + + page = Page() + + def prepare(_url, *, navigate=True): + navigation_flags.append(navigate) + return page + + monkeypatch.setattr(module, "sync_playwright", PlaywrightStarter) + monkeypatch.setattr(manager, "_prepare_login_page", prepare) + monkeypatch.setattr( + manager, + "_fill_first_visible", + lambda _page, _selectors, _value, label, **_kwargs: filled.append(label) or True, + ) + monkeypatch.setattr(manager, "_dismiss_post_login_dialogs", lambda *_a, **_k: None) + monkeypatch.setattr(module.time, "sleep", lambda _seconds: None) + + assert manager.login_with_playwright() is False + assert manager._acceptance_cookie_skipped is True + assert navigation_flags == [True] + assert filled == ["username", "password"] + assert "captcha or QR" in acceptance_environment.read_text(encoding="utf-8") + + +def test_product_replenishment_invalid_cdp_cookie_never_falls_back_to_login( + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _import_supply_script("ProductReplenishment") + manager = module.JstManager(str(_config_file(tmp_path))) + manager._page = object() + monkeypatch.setattr(manager, "connect_existing_browser", lambda _url: True) + monkeypatch.setattr(manager, "_page_has_inventory_access", lambda _page: False) + monkeypatch.setattr( + manager, + "_login_on_current_page", + _forbidden("acceptance mode must not run the fresh-login fallback"), + ) + + assert manager.login( + use_existing_browser=True, + cdp_url="http://127.0.0.1:22556", + ) is False + assert manager._acceptance_cookie_skipped is True + + +def test_product_replenishment_production_existing_browser_flow_is_preserved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + monkeypatch.setenv("GYXX_SUPPLY_ERP_PASSWORD", "test-only-password") + module = _import_supply_script("ProductReplenishment") + manager = module.JstManager(str(_config_file(tmp_path))) + manager._page = object() + called: list[tuple[str, str, str]] = [] + monkeypatch.setattr(manager, "connect_existing_browser", lambda _url: True) + monkeypatch.setattr( + manager, + "_login_on_current_page", + lambda url, username, credential: called.append( + (url, username, credential) + ) + or True, + ) + + assert manager.login( + use_existing_browser=True, + cdp_url="http://127.0.0.1:22556", + ) is True + assert called == [ + ("https://login.example.test/", "acceptance-user", "test-only-password") + ] + + +def test_product_replenishment_force_login_clears_dedicated_browser_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GYXX_SUPPLY_FORCE_ERP_LOGIN", "1") + monkeypatch.setenv("GYXX_SUPPLY_ERP_PASSWORD", "test-only-password") + module = _import_supply_script("ProductReplenishment") + manager = module.JstManager(str(_config_file(tmp_path))) + manager._page = object() + + class Context: + cleared = False + + def clear_cookies(self) -> None: + self.cleared = True + + context = Context() + manager._context = context + monkeypatch.setattr(manager, "connect_existing_browser", lambda _url: True) + monkeypatch.setattr( + manager, + "_page_has_inventory_access", + _forbidden("forced login must not accept the existing ERP session"), + ) + monkeypatch.setattr(manager, "_login_on_current_page", lambda *_args: True) + + assert manager.login(True, "http://127.0.0.1:22556") is True + assert context.cleared is True + + +def test_product_replenishment_main_returns_cookie_skip_code_without_cdp( + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _import_supply_script("ProductReplenishment") + + class Manager: + cdp_url = None + _acceptance_cookie_skipped = False + login_calls: list[bool] = [] + + def __init__(self, _config_path): + pass + + def _skip_acceptance_cookie(self, _reason): + self._acceptance_cookie_skipped = True + return False + + def login(self, *, use_existing_browser=False, cdp_url=None): + del cdp_url + self.login_calls.append(use_existing_browser) + return self._skip_acceptance_cookie("interactive challenge") + + def close(self, *, disconnect=False): + del disconnect + + monkeypatch.setattr(module, "JstManager", Manager) + monkeypatch.setattr(module, "REPLENISHMENT_RAW_ROOT", tmp_path / "raw") + monkeypatch.setattr(module, "REPLENISHMENT_WORK_ROOT", tmp_path / "work") + monkeypatch.setattr(module, "REPLENISHMENT_EXPORT_ROOT", tmp_path / "export") + + assert module.main(cdp_url=None) == COOKIE_SKIP_EXIT_CODE + assert Manager.login_calls == [False] + + +def test_purchase_confirmation_main_exits_with_cookie_skip_code( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _import_supply_script("PurchaseConfirmation") + closed: list[bool] = [] + + class Manager: + _acceptance_cookie_skipped = True + + def __init__(self, _config_path, cdp_port=None): + pass + + def login_with_playwright(self): + return False + + def close_browser(self): + closed.append(True) + + monkeypatch.setattr(module, "PurchaseManager", Manager) + monkeypatch.setattr(module, "_persist_run_to_pg", lambda _result: None) + monkeypatch.setattr(module.sys, "platform", "linux") + + with pytest.raises(SystemExit) as raised: + module.main(cdp_port=22557) + + assert raised.value.code == COOKIE_SKIP_EXIT_CODE + assert closed == [True] + + +def test_purchase_order_update_main_exits_with_cookie_skip_code( + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _import_supply_script("PurchaseOrderUpdate") + config = _config_file(tmp_path) + task = tmp_path / "task.json" + task.write_text( + json.dumps( + { + "sku_list": ["10426004"], + "target_date": "2026-08-02", + "remark": "acceptance fixture", + } + ), + encoding="utf-8", + ) + closed: list[bool] = [] + + class Manager: + _acceptance_cookie_skipped = True + + def __init__(self, _config_path, cdp_port=None): + pass + + def login_with_playwright(self): + return False + + def close_browser(self): + closed.append(True) + + monkeypatch.setattr(module, "CONFIG_PATH", config) + monkeypatch.setattr(module, "PurchaseManager", Manager) + + with pytest.raises(SystemExit) as raised: + module.main(task_file=task, cdp_port=22558) + + assert raised.value.code == COOKIE_SKIP_EXIT_CODE + assert closed == [True] diff --git a/tests/test_business_modules.py b/tests/test_business_modules.py index 4430dbc..5588f5a 100644 --- a/tests/test_business_modules.py +++ b/tests/test_business_modules.py @@ -29,41 +29,36 @@ def test_default_registry_contains_exactly_the_four_business_modules() -> None: assert isinstance(registry.get("supply_chain"), SupplyChainModule) -def test_catalog_composition_registers_all_migrated_scheduled_and_manual_workflows() -> None: +def test_catalog_composition_registers_all_workflows() -> None: catalog = WorkflowCatalog.load(Path(__file__).parents[1] / "config") registry = create_default_registry(catalog=catalog) - assert len(registry.workflow_ids) == 28 + assert len(registry.workflow_ids) == 23 assert set(registry.workflow_ids) == { "shop.metrics.weekly", "shop.competitor.weekly", + "shop.jd_self_operated.daily", + "shop.douyin_price_appeal", "supply.purchase_confirmation.daily", "supply.replenishment.weekly", "supply.replenishment_alert.daily", - "supply.purchase_order_update", "content.metrics.daily", "content.marketing_report.daily", "content.relogin.weekly", - "content.self_operated.weekly", "content.creator_report.monthly", "content.summary.monthly", "content.cooperations.daily", "content.comments.weekly", "content.summary.weekly", - "content.mapping.refresh", - "content.retry_failed", - "content.metrics.backfill", "product.persona.daily", "product.daily", "product.alert.daily", "product.import.daily", "product.style_analysis.interval", - "product.main_image.jd.weekly", "product.main_image.weekly", - "product.backfill", + "product.sales_sheet.daily", "product.market_rank", - "product.review_collection", } diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 95d8cba..a7203b6 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections import Counter from pathlib import Path @@ -10,51 +11,364 @@ from gyxx_flow.catalog import CatalogError, WorkflowCatalog PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_catalog_maps_exactly_21_current_scheduled_tasks() -> None: +def test_catalog_maps_23_project_scheduled_tasks() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") scheduled = catalog.scheduled_workflows() - assert len(scheduled) == 21 - assert len({item.source_task_name for item in scheduled}) == 21 + assert len(scheduled) == 23 + assert len({item.source_task_name for item in scheduled}) == 23 assert Counter(item.module for item in scheduled) == { - "content_marketing": 9, - "product_commerce": 7, - "shop_intelligence": 2, + "content_marketing": 8, + "product_commerce": 8, + "shop_intelligence": 4, "supply_chain": 3, } -def test_catalog_keeps_known_manual_or_unregistered_workflows_unscheduled() -> None: +def test_catalog_schedules_douyin_price_appeal() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + schedule = catalog.schedule_for("shop.douyin_price_appeal") - assert { - "content.mapping.refresh", - "content.retry_failed", - "product.backfill", - "product.market_rank", - "product.review_collection", - "supply.purchase_order_update", - }.issubset({item.workflow_id for item in catalog.manual_workflows()}) - assert "product.weekly_aggregate.documented_missing" in { - item.workflow_id for item in catalog.unavailable_workflows() + assert len(catalog.workflows) == 23 + assert catalog.manual_workflows() == () + assert "shop.douyin_price_appeal" in { + schedule.workflow_id for schedule in catalog.schedules } + assert catalog.unavailable_workflows() == () + assert schedule.kind == "daily" + assert schedule.effective_times == ("08:00", "16:00", "22:00") + + +def test_product_daily_passes_the_previous_business_day_explicitly() -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + workflow = next( + item for item in catalog.workflows if item.workflow_id == "product.daily" + ) + + assert workflow.steps[0].args == ( + "--target-date", + "{business_date}", + "--stages", + "collect,analyze,export,insert", + ) + assert catalog.schedule_for(workflow.workflow_id).business_date_offset_days == -1 def test_every_scheduled_workflow_has_one_valid_asia_shanghai_schedule() -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") assert catalog.timezone == "Asia/Shanghai" - assert len(catalog.schedules) == 21 + assert len(catalog.schedules) == 23 for workflow in catalog.scheduled_workflows(): schedule = catalog.schedule_for(workflow.workflow_id) assert schedule.workflow_id == workflow.workflow_id assert schedule.kind in {"daily", "weekly", "monthly", "interval_days"} assert schedule.at.count(":") == 1 + assert sum(schedule.enabled for schedule in catalog.schedules) == 23 + + +def test_scheduled_workflows_use_python_entries_and_content_tasks_are_graphs() -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + + for workflow in catalog.scheduled_workflows(): + assert workflow.steps, f"scheduled workflow must declare LangGraph steps: {workflow.workflow_id}" + assert all(step.name and step.description for step in workflow.steps) + assert all( + step.data_flow + and step.data_flow.sources + and step.data_flow.processing + and step.data_flow.destinations + for step in workflow.steps + ) + entries = [step.entry for step in workflow.steps] + assert all( + not entry.casefold().endswith((".bat", ".cmd", ".ps1")) + for entry in entries + ) + + daily = next( + workflow + for workflow in catalog.workflows + if workflow.workflow_id == "content.metrics.daily" + ) + assert [step.step_id for step in daily.steps] == [ + "collect_collaborators", + "refresh_self_mapping", + "collect_self_bilibili", + "collect_self_douyin", + "sync", + ] + assert daily.steps[1].depends_on == ("collect_collaborators",) + assert daily.steps[2].entry == "self_bilibili_scraper.py" + assert daily.steps[3].entry == "chanmama_scraper.py" + assert "--include-self-operated" not in daily.steps[0].args + assert all(step.run_after_failure for step in daily.steps[1:]) + assert catalog.schedule_for(daily.workflow_id).at == "22:00" + + marketing_report = next( + workflow + for workflow in catalog.workflows + if workflow.workflow_id == "content.marketing_report.daily" + ) + report_flow = marketing_report.steps[0].data_flow + assert report_flow is not None + assert {source.system for source in report_flow.sources} == { + "PostgreSQL", + "飞书", + } + assert "调用 Hermes 生成营销分析" in report_flow.processing + assert any( + destination.system == "飞书" and destination.condition + for destination in report_flow.destinations + ) + + comments = next( + workflow + for workflow in catalog.workflows + if workflow.workflow_id == "content.comments.weekly" + ) + assert [step.step_id for step in comments.steps] == [ + "bilibili", + "xiaohongshu", + "douyin", + ] + assert all(step.depends_on == () for step in comments.steps) + assert "三平台并行启动" in comments.note + + main_image = next( + workflow + for workflow in catalog.workflows + if workflow.workflow_id == "product.main_image.weekly" + ) + assert [step.step_id for step in main_image.steps] == ["jd", "tmall"] + assert [step.entry for step in main_image.steps] == [ + "run_weekly_jd_main_image.py", + "run_weekly_main_image.py", + ] + assert all(step.depends_on == () for step in main_image.steps) + assert all(step.run_after_failure is False for step in main_image.steps) + assert "并行执行京东与天猫" in main_image.note + assert catalog.schedule_for(main_image.workflow_id).at == "08:30" + + +def test_shop_steps_preserve_declared_replay_policy_at_module_boundary() -> None: + from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule + + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + definitions = ShopIntelligenceModule.from_catalog(catalog).workflow_definitions() + + policies = { + workflow.workflow_id: {step.replay_policy for step in workflow.steps} + for workflow in definitions + } + + assert policies["shop.metrics.weekly"] == {"idempotent"} + assert policies["shop.competitor.weekly"] == {"idempotent"} + assert policies["shop.jd_self_operated.daily"] == {"idempotent"} + assert policies["shop.douyin_price_appeal"] == {"idempotent"} + + +def test_product_daily_import_is_declared_idempotent() -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + workflow = next( + item + for item in catalog.workflows + if item.workflow_id == "product.import.daily" + ) + + assert len(workflow.steps) == 1 + assert workflow.steps[0].replay_policy == "idempotent" + + +def test_catalog_loads_optional_step_timeout_seconds(tmp_path: Path) -> None: + (tmp_path / "workflows.json").write_text( + json.dumps( + { + "schema_version": 3, + "workflows": [ + { + "id": "content.timeout_probe", + "module": "content_marketing", + "trigger": "manual", + "execution": { + "steps": [ + { + "id": "run", + "entry": "run.py", + "timeout_seconds": 12.5, + } + ] + }, + } + ], + } + ), + encoding="utf-8", + ) + (tmp_path / "schedules.json").write_text( + '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', + encoding="utf-8", + ) + + catalog = WorkflowCatalog.load(tmp_path) + + assert catalog.workflows[0].steps[0].timeout_seconds == 12.5 + + +@pytest.mark.parametrize("timeout_seconds", [0, -1, True, "60", float("inf")]) +def test_catalog_rejects_invalid_step_timeout_seconds( + tmp_path: Path, + timeout_seconds: object, +) -> None: + (tmp_path / "workflows.json").write_text( + json.dumps( + { + "schema_version": 3, + "workflows": [ + { + "id": "content.timeout_probe", + "module": "content_marketing", + "trigger": "manual", + "execution": { + "steps": [ + { + "id": "run", + "entry": "run.py", + "timeout_seconds": timeout_seconds, + } + ] + }, + } + ], + } + ), + encoding="utf-8", + ) + (tmp_path / "schedules.json").write_text( + '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', + encoding="utf-8", + ) + + with pytest.raises(CatalogError, match="workflow step timeout_seconds is invalid"): + WorkflowCatalog.load(tmp_path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("name", "", "workflow step name is invalid"), + ("description", 42, "workflow step description is invalid"), + ], +) +def test_catalog_rejects_invalid_step_explanation( + tmp_path: Path, + field: str, + value: object, + message: str, +) -> None: + step = { + "id": "run", + "entry": "run.py", + "name": "运行任务", + "description": "执行测试任务。", + field: value, + } + (tmp_path / "workflows.json").write_text( + json.dumps( + { + "schema_version": 3, + "workflows": [ + { + "id": "a.one", + "module": "content_marketing", + "trigger": "manual", + "execution": {"steps": [step]}, + } + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (tmp_path / "schedules.json").write_text( + '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', + encoding="utf-8", + ) + + with pytest.raises(CatalogError, match=message): + WorkflowCatalog.load(tmp_path) + + +@pytest.mark.parametrize( + ("data_flow", "message"), + [ + ([], "data_flow must be an object"), + ( + {"sources": [], "processing": ["处理"], "destinations": []}, + "sources is invalid", + ), + ( + { + "sources": [{"label": "输入"}], + "processing": [], + "destinations": [{"label": "输出"}], + }, + "processing is invalid", + ), + ( + { + "sources": [{"label": "输入"}], + "processing": ["处理"], + "destinations": [{"label": ""}], + }, + "destinations is invalid", + ), + ], +) +def test_catalog_rejects_invalid_step_data_flow( + tmp_path: Path, + data_flow: object, + message: str, +) -> None: + (tmp_path / "workflows.json").write_text( + json.dumps( + { + "schema_version": 3, + "workflows": [ + { + "id": "a.one", + "module": "content_marketing", + "trigger": "manual", + "execution": { + "steps": [ + { + "id": "run", + "entry": "run.py", + "name": "运行任务", + "description": "执行测试任务。", + "data_flow": data_flow, + } + ] + }, + } + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (tmp_path / "schedules.json").write_text( + '{"schema_version":1,"timezone":"Asia/Shanghai","schedules":[]}', + encoding="utf-8", + ) + + with pytest.raises(CatalogError, match=message): + WorkflowCatalog.load(tmp_path) def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None: (tmp_path / "workflows.json").write_text( - '{"schema_version":2,"workflows":[' + '{"schema_version":3,"workflows":[' '{"id":"a.one","module":"content_marketing","trigger":"manual",' '"execution":{"entry":"run.py"}},' '{"id":"a.one","module":"content_marketing","trigger":"manual",' @@ -72,7 +386,7 @@ def test_catalog_rejects_duplicate_workflow_ids(tmp_path: Path) -> None: def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None: (tmp_path / "workflows.json").write_text( - '{"schema_version":2,"workflows":[' + '{"schema_version":3,"workflows":[' '{"id":"a.one","module":"content_marketing","trigger":"manual",' '"execution":{"entry":"D:/old/run.py"}}]}', encoding="utf-8", @@ -84,3 +398,79 @@ def test_catalog_rejects_absolute_execution_entry(tmp_path: Path) -> None: with pytest.raises(CatalogError, match="relative"): WorkflowCatalog.load(tmp_path) + + +def _write_single_scheduled_catalog(tmp_path: Path, at: object) -> None: + (tmp_path / "workflows.json").write_text( + json.dumps( + { + "schema_version": 3, + "workflows": [ + { + "id": "shop.price_appeal", + "module": "shop_intelligence", + "trigger": "scheduled", + "execution": {"entry": "run.py"}, + "provenance": {"task_name": "price appeal"}, + } + ], + } + ), + encoding="utf-8", + ) + (tmp_path / "schedules.json").write_text( + json.dumps( + { + "schema_version": 1, + "timezone": "Asia/Shanghai", + "schedules": [ + { + "workflow_id": "shop.price_appeal", + "kind": "daily", + "at": at, + } + ], + } + ), + encoding="utf-8", + ) + + +def test_catalog_loads_multiple_daily_schedule_times(tmp_path: Path) -> None: + _write_single_scheduled_catalog(tmp_path, ["08:00", "16:00", "22:00"]) + + schedule = WorkflowCatalog.load(tmp_path).schedules[0] + + assert schedule.at == "08:00" + assert schedule.at_times == ("08:00", "16:00", "22:00") + assert schedule.effective_times == ("08:00", "16:00", "22:00") + + +def test_catalog_preserves_legacy_single_schedule_time(tmp_path: Path) -> None: + _write_single_scheduled_catalog(tmp_path, "08:00") + + schedule = WorkflowCatalog.load(tmp_path).schedules[0] + + assert schedule.at == "08:00" + assert schedule.at_times == () + assert schedule.effective_times == ("08:00",) + + +@pytest.mark.parametrize( + "at", + [ + [], + ["08:00", "08:00"], + ["08:00", "24:00"], + ["08:00", 16], + {"morning": "08:00"}, + ], +) +def test_catalog_rejects_invalid_multiple_schedule_times( + tmp_path: Path, + at: object, +) -> None: + _write_single_scheduled_catalog(tmp_path, at) + + with pytest.raises(CatalogError, match="invalid schedule time"): + WorkflowCatalog.load(tmp_path) diff --git a/tests/test_cli_workflows.py b/tests/test_cli_workflows.py index bc67a01..4889e15 100644 --- a/tests/test_cli_workflows.py +++ b/tests/test_cli_workflows.py @@ -40,7 +40,7 @@ class RecordingStep: return StepExecution(exit_code=0) -def test_default_cli_registry_exposes_all_21_migrated_scheduled_workflows( +def test_default_cli_registry_exposes_all_project_scheduled_workflows( tmp_path: Path, ) -> None: project_root = Path(__file__).parents[1] @@ -48,10 +48,14 @@ def test_default_cli_registry_exposes_all_21_migrated_scheduled_workflows( Settings(project_root=project_root, data_root=tmp_path) ) - assert sum( - registry.is_registered(entry.workflow_id) - for entry in registry.catalog.scheduled_workflows() - ) == 21 + scheduled_ids = { + entry.workflow_id for entry in registry.catalog.scheduled_workflows() + } + assert len(scheduled_ids) == 23 + assert scheduled_ids == { + schedule.workflow_id for schedule in registry.catalog.schedules + } + assert all(registry.is_registered(workflow_id) for workflow_id in scheduled_ids) def _catalog(tmp_path: Path) -> WorkflowCatalog: @@ -60,7 +64,7 @@ def _catalog(tmp_path: Path) -> WorkflowCatalog: (config / "workflows.json").write_text( json.dumps( { - "schema_version": 2, + "schema_version": 3, "workflows": [ { "id": "shop.metrics.weekly", @@ -126,7 +130,7 @@ def _invoke( return code, stdout.getvalue(), stderr.getvalue() -def test_list_reports_catalog_and_registration_state_as_json(tmp_path: Path) -> None: +def test_list_reports_only_scheduled_catalog_entries_as_json(tmp_path: Path) -> None: action = RecordingStep() registry, settings = _runtime( tmp_path, @@ -142,17 +146,12 @@ def test_list_reports_catalog_and_registration_state_as_json(tmp_path: Path) -> rows = json.loads(output) assert rows == [ { + "enabled": True, "id": "shop.metrics.weekly", "module": "shop_intelligence", "registered": True, "trigger": "scheduled", - }, - { - "id": "shop.missing", - "module": "shop_intelligence", - "registered": False, - "trigger": "unavailable", - }, + } ] diff --git a/tests/test_console.py b/tests/test_console.py new file mode 100644 index 0000000..e2ecdfd --- /dev/null +++ b/tests/test_console.py @@ -0,0 +1,1991 @@ +from __future__ import annotations + +import http.client +import json +import re +import shutil +import subprocess +import threading +from contextlib import contextmanager +from datetime import date, datetime, timezone +from http import HTTPStatus +from pathlib import Path +from typing import Any, Iterator, TextIO + +import psutil +import pytest + +import gyxx_flow.console as console_module +from gyxx_flow.catalog import ScheduleEntry, WorkflowCatalog, WorkflowEntry +from gyxx_flow.console import ( + MAX_REQUEST_BYTES, + ConsoleConflictError, + ConsoleNotFoundError, + ConsolePreconditionError, + ConsoleRequestError, + ScheduleConfigStore, + SubprocessConsoleRunLauncher, + WorkflowConsoleService, + create_console_server, +) +from gyxx_flow.core.config import Settings +from gyxx_flow.core.context import RunContext +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.core.records import RunJournal +from gyxx_flow.ops import RunIndex +from gyxx_flow.scheduler_service import PythonScheduler + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def console_settings(tmp_path: Path) -> Settings: + project_root = tmp_path / "project" + config_dir = project_root / "config" + config_dir.mkdir(parents=True) + for name in ("workflows.json", "schedules.json"): + shutil.copyfile(PROJECT_ROOT / "config" / name, config_dir / name) + return Settings(project_root=project_root, data_root=tmp_path / "data") + + +class FakeConsoleLauncher: + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + self._active: dict[str, dict[str, object]] = {} + self._cancelled: dict[str, dict[str, object]] = {} + + def launch( + self, + workflow_id: str, + business_date: date, + *, + execute: bool, + shadow: bool, + ) -> dict[str, object]: + if workflow_id in self._active: + raise ConsoleConflictError("already active") + operation = { + "operation_id": f"op-{len(self.calls) + 1:016x}", + "workflow_id": workflow_id, + "business_date": business_date.isoformat(), + "mode": "execute" if execute else "dry_run", + "shadow": shadow, + "started_at": "2026-08-01T00:00:00+00:00", + "status": "accepted", + } + self.calls.append(operation) + self._active[workflow_id] = operation + return dict(operation) + + def active(self) -> tuple[dict[str, object], ...]: + return tuple(dict(item) for item in self._active.values()) + + def cancel( + self, + workflow_id: str, + operation_id: str, + ) -> dict[str, object]: + operation = self._active.get(workflow_id) + if operation is None: + previous = self._cancelled.get(operation_id) + if previous is not None and previous["workflow_id"] == workflow_id: + return {**previous, "already_cancelled": True} + raise ConsoleConflictError("not active") + if operation["operation_id"] != operation_id: + raise ConsoleConflictError("not active") + del self._active[workflow_id] + operation.update( + { + "status": "cancelled", + "already_cancelled": False, + "ended_at": "2026-08-01T00:01:00+00:00", + } + ) + self._cancelled[operation_id] = dict(operation) + return dict(operation) + + def complete(self, workflow_id: str) -> None: + self._active.pop(workflow_id, None) + + +def _read_config(settings: Settings, name: str) -> dict[str, Any]: + return json.loads( + (settings.project_root / "config" / name).read_text(encoding="utf-8") + ) + + +def _write_config(settings: Settings, name: str, payload: dict[str, Any]) -> None: + (settings.project_root / "config" / name).write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + +def _append_non_scheduled_workflow( + settings: Settings, + *, + workflow_id: str, + module: str, + trigger: str, +) -> None: + workflows = _read_config(settings, "workflows.json") + workflows["workflows"].append( + { + "id": workflow_id, + "module": module, + "trigger": trigger, + "execution": {"entry": "console_probe.py"}, + "provenance": {"source_project": "console-test"}, + } + ) + _write_config(settings, "workflows.json", workflows) + + +def _schedule_row(settings: Settings, workflow_id: str) -> dict[str, Any]: + payload = _read_config(settings, "schedules.json") + return next( + item for item in payload["schedules"] if item["workflow_id"] == workflow_id + ) + + +def _index_failed_run(settings: Settings, workflow_id: str) -> str: + context = RunContext.create( + workflow_id, + "2026-08-01", + now=datetime(2026, 8, 1, 1, 2, 3, tzinfo=timezone.utc), + random_suffix="console1", + ) + journal = RunJournal.create( + DataLayout(settings.data_root), + context, + mode="execute", + ) + journal.finalize("failed", error="critical steps failed: collect") + RunIndex(settings.data_root).index_journal(journal) + return context.run_id + + +def _index_parallel_main_image_run(settings: Settings) -> str: + context = RunContext.create( + "product.main_image.weekly", + "2026-08-02", + now=datetime(2026, 8, 2, 1, 2, 3, tzinfo=timezone.utc), + random_suffix="parallel1", + ) + journal = RunJournal.create(DataLayout(settings.data_root), context) + journal.start_step("jd.attempt-1", attempt=1) + journal.finish_step( + "jd.attempt-1", + status="failed", + exit_code=9, + error="JD browser failed " + "pass" + "word=" + "private" + "-value", + ) + journal.start_step("tmall.attempt-1", attempt=1) + journal.finish_step( + "tmall.attempt-1", + status="success", + exit_code=0, + ) + journal.finalize("failed", error="critical steps failed: jd") + RunIndex(settings.data_root).index_journal(journal) + return context.run_id + + +def _index_running_metrics_run(settings: Settings) -> RunJournal: + context = RunContext.create( + "content.metrics.daily", + "2026-08-02", + now=datetime(2026, 8, 2, 2, 3, 4, tzinfo=timezone.utc), + random_suffix="running1", + ) + journal = RunJournal.create(DataLayout(settings.data_root), context) + RunIndex(settings.data_root).index_journal(journal) + journal.start_step("collect_collaborators.attempt-1", attempt=1) + return journal + + +def test_error_sanitizer_redacts_structured_and_url_credentials() -> None: + key_a = "pass" + "word" + key_b = "access_" + "token" + key_c = "to" + "ken" + value_a = "hunter" + "2" + value_b = "abc" + "123" + value_c = "p4" + "ss" + value_d = "query-" + "private" + cases = ( + (json.dumps({key_a: value_a}), value_a), + (repr({key_b: value_b}), value_b), + (f"postgresql://alice:{value_c}@db.local/app", value_c), + (f"https://service.local/run?{key_c}={value_d}&mode=1", value_d), + ) + + for raw_error, sensitive_value in cases: + sanitized = console_module._sanitize_error(raw_error) + assert sanitized is not None + assert sensitive_value not in sanitized + assert "[REDACTED]" in sanitized + + +def test_error_sanitizer_preserves_context_and_root_cause_tail() -> None: + raw_error = ( + "collector startup context\n" + + "progress line\n" * 1_000 + + "ROOT CAUSE AT LOG TAIL" + ) + + sanitized = console_module._sanitize_error(raw_error) + + assert sanitized is not None + assert sanitized.startswith("collector startup context") + assert "[middle output omitted]" in sanitized + assert sanitized.endswith("ROOT CAUSE AT LOG TAIL") + assert len(sanitized) <= console_module.MAX_ERROR_CHARS + + +def test_overview_tracks_the_current_complete_catalog_without_leaking_paths( + console_settings: Settings, +) -> None: + launcher = FakeConsoleLauncher() + run_id = _index_failed_run(console_settings, "content.metrics.daily") + service = WorkflowConsoleService(console_settings, launcher=launcher) + + initial = service.overview() + initial_ids = {item["id"] for item in initial["workflows"]} + catalog = WorkflowCatalog.load(console_settings.project_root / "config") + + assert initial_ids == {item.workflow_id for item in catalog.workflows} + assert initial["summary"]["total"] == len(catalog.workflows) + assert {module["id"] for module in initial["modules"]} == { + "content_marketing", + "product_commerce", + "shop_intelligence", + "supply_chain", + } + assert sum(module["count"] for module in initial["modules"]) == len( + catalog.workflows + ) + failed = next( + item for item in initial["workflows"] if item["id"] == "content.metrics.daily" + ) + assert failed["last_run"]["run_id"] == run_id + assert failed["last_run"]["mode"] == "execute" + assert failed["last_run"]["error"] == "critical steps failed: collect" + assert all(step["name"] and step["description"] for step in failed["steps"]) + assert {step["replay_policy"] for step in failed["steps"]} == {"guarded"} + shop = next( + item for item in initial["workflows"] if item["id"] == "shop.metrics.weekly" + ) + assert {step["replay_policy"] for step in shop["steps"]} == {"idempotent"} + price_appeal = next( + item + for item in initial["workflows"] + if item["id"] == "shop.douyin_price_appeal" + ) + assert price_appeal["name"] == "抖音待改价申诉" + assert price_appeal["trigger"] == "scheduled" + assert price_appeal["registered"] is True + assert price_appeal["schedule"]["at"] == ["08:00", "16:00", "22:00"] + assert price_appeal["steps"][0]["replay_policy"] == "idempotent" + assert price_appeal["steps"][0]["timeout_seconds"] == 1_800 + jd_self_operated = next( + item + for item in initial["workflows"] + if item["id"] == "shop.jd_self_operated.daily" + ) + assert [step["id"] for step in jd_self_operated["steps"]] == [ + "brand", + "product", + ] + assert [ + step["timeout_seconds"] for step in jd_self_operated["steps"] + ] == [600, 1_800] + monthly_sales = next( + item + for item in initial["workflows"] + if item["id"] == "product.sales_sheet.daily" + ) + assert monthly_sales["run_date_mode"] == "month" + assert failed["run_date_mode"] == "date" + report = next( + item + for item in initial["workflows"] + if item["id"] == "content.marketing_report.daily" + ) + report_flow = report["steps"][0]["data_flow"] + assert [source["system"] for source in report_flow["sources"]] == [ + "PostgreSQL", + "飞书", + ] + assert report_flow["processing"] == [ + "聚合业务事实与款式表现", + "调用 Hermes 生成营销分析", + "生成图表和 Markdown 日报", + ] + assert report_flow["destinations"][1]["condition"] == ( + "正式执行且启用 --send" + ) + assert "journal_path" not in json.dumps(initial, ensure_ascii=False) + + workflows = _read_config(console_settings, "workflows.json") + workflows["workflows"].append( + { + "id": "content.console_probe", + "module": "content_marketing", + "trigger": "manual", + "execution": {"entry": "console_probe.py"}, + "provenance": {"source_project": "content"}, + } + ) + _write_config(console_settings, "workflows.json", workflows) + + refreshed = service.overview() + + assert refreshed["summary"]["total"] == len(catalog.workflows) + 1 + assert "content.console_probe" in { + item["id"] for item in refreshed["workflows"] + } + probe = next( + item for item in refreshed["workflows"] if item["id"] == "content.console_probe" + ) + assert probe["registered"] is False + assert probe["schedule"] is None + + +def test_workflow_display_name_is_persistent_revision_safe_and_resettable( + console_settings: Settings, +) -> None: + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + workflow_id = "content.marketing_report.daily" + workflows_before = (console_settings.project_root / "config" / "workflows.json").read_bytes() + initial = service.overview() + workflow = next(item for item in initial["workflows"] if item["id"] == workflow_id) + + assert workflow["name"] == "营销日报生成与发送" + assert workflow["default_name"] == workflow["name"] + assert workflow["name_customized"] is False + + renamed = service.update_display_name( + workflow_id, + {"display_name": " 每日内容营销简报 "}, + expected_revision=initial["workflow_names_revision"], + ) + assert renamed["name"] == "每日内容营销简报" + assert renamed["name_customized"] is True + + restarted = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ).overview() + persisted = next( + item for item in restarted["workflows"] if item["id"] == workflow_id + ) + assert persisted["name"] == "每日内容营销简报" + assert persisted["default_name"] == "营销日报生成与发送" + assert persisted["name_customized"] is True + assert ( + console_settings.project_root / "config" / "workflows.json" + ).read_bytes() == workflows_before + + state_path = ( + console_settings.data_root + / "state" + / "console" + / "workflow-display-names.json" + ) + state_before_conflict = state_path.read_bytes() + with pytest.raises(ConsoleConflictError, match="已被更新"): + service.update_display_name( + workflow_id, + {"display_name": "过期窗口中的名称"}, + expected_revision=initial["workflow_names_revision"], + ) + assert state_path.read_bytes() == state_before_conflict + + reset = service.update_display_name( + workflow_id, + {"display_name": None}, + expected_revision=renamed["workflow_names_revision"], + ) + assert reset["name"] == "营销日报生成与发送" + assert reset["name_customized"] is False + + +@pytest.mark.parametrize( + "display_name", + [" ", "x" * 65, "包含\u200b隐藏字符"], +) +def test_workflow_display_name_rejects_unsafe_values( + console_settings: Settings, + display_name: str, +) -> None: + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + revision = service.overview()["workflow_names_revision"] + + with pytest.raises(ConsoleRequestError): + service.update_display_name( + "content.marketing_report.daily", + {"display_name": display_name}, + expected_revision=revision, + ) + + +def test_workflow_display_name_rejects_invalid_request_boundaries( + console_settings: Settings, +) -> None: + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + revision = service.overview()["workflow_names_revision"] + + with pytest.raises(ConsolePreconditionError): + service.update_display_name( + "content.marketing_report.daily", + {"display_name": "新名称"}, + expected_revision=None, + ) + with pytest.raises(ConsoleRequestError, match="只允许"): + service.update_display_name( + "content.marketing_report.daily", + {"display_name": "新名称", "id": "other.workflow"}, + expected_revision=revision, + ) + with pytest.raises(ConsoleNotFoundError): + service.update_display_name( + "content.unknown", + {"display_name": "新名称"}, + expected_revision=revision, + ) + + +def test_overview_falls_back_when_workflow_display_names_are_corrupt( + console_settings: Settings, +) -> None: + state_path = ( + console_settings.data_root + / "state" + / "console" + / "workflow-display-names.json" + ) + state_path.parent.mkdir(parents=True) + state_path.write_text("{broken", encoding="utf-8") + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + + overview = service.overview() + + workflow = next( + item + for item in overview["workflows"] + if item["id"] == "content.marketing_report.daily" + ) + assert workflow["name"] == "营销日报生成与发送" + assert workflow["name_customized"] is False + assert overview["warnings"] == [ + "自定义工作流名称配置无效，当前显示默认名称" + ] + before = state_path.read_bytes() + with pytest.raises(ConsoleRequestError, match="配置无效"): + service.update_display_name( + workflow["id"], + {"display_name": "不会覆盖损坏配置"}, + expected_revision=overview["workflow_names_revision"], + ) + assert state_path.read_bytes() == before + + +def test_overview_includes_sanitized_parallel_step_details( + console_settings: Settings, +) -> None: + run_id = _index_parallel_main_image_run(console_settings) + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + + overview = service.overview() + workflow = next( + item + for item in overview["workflows"] + if item["id"] == "product.main_image.weekly" + ) + last_run = workflow["last_run"] + + assert last_run["run_id"] == run_id + assert [step["id"] for step in last_run["steps"]] == [ + "jd.attempt-1", + "tmall.attempt-1", + ] + assert last_run["steps"][0]["status"] == "failed" + assert last_run["steps"][0]["exit_code"] == 9 + assert "private-value" not in last_run["steps"][0]["error"] + assert "[REDACTED]" in last_run["steps"][0]["error"] + assert last_run["steps"][1]["status"] == "success" + + history = service.recent_runs("product.main_image.weekly", limit=1) + assert history["runs"][0]["steps"] == last_run["steps"] + assert history["runs"][0]["mode"] == "unknown" + + +def test_overview_reads_live_step_progress_from_a_running_journal( + console_settings: Settings, +) -> None: + journal = _index_running_metrics_run(console_settings) + service = WorkflowConsoleService( + console_settings, + launcher=FakeConsoleLauncher(), + ) + + workflow = next( + item + for item in service.overview()["workflows"] + if item["id"] == "content.metrics.daily" + ) + + assert workflow["last_run"]["status"] == "running" + assert workflow["last_run"]["step_counts"] == { + "success": 0, + "failed": 0, + "skipped": 0, + "running": 1, + } + assert workflow["last_run"]["steps"][0]["id"] == ( + "collect_collaborators.attempt-1" + ) + assert workflow["last_run"]["steps"][0]["status"] == "running" + + journal.finish_step( + "collect_collaborators.attempt-1", + status="success", + exit_code=0, + ) + journal.start_step("refresh_self_mapping.attempt-1", attempt=1) + refreshed = next( + item + for item in service.overview()["workflows"] + if item["id"] == "content.metrics.daily" + ) + + assert refreshed["last_run"]["step_counts"] == { + "success": 1, + "failed": 0, + "skipped": 0, + "running": 1, + } + assert [step["status"] for step in refreshed["last_run"]["steps"]] == [ + "success", + "running", + ] + + +def test_overview_keeps_a_new_active_launch_separate_from_the_previous_run( + console_settings: Settings, +) -> None: + previous_run_id = _index_parallel_main_image_run(console_settings) + launcher = FakeConsoleLauncher() + launcher.launch( + "product.main_image.weekly", + date(2026, 8, 3), + execute=True, + shadow=False, + ) + service = WorkflowConsoleService(console_settings, launcher=launcher) + + workflow = next( + item + for item in service.overview()["workflows"] + if item["id"] == "product.main_image.weekly" + ) + + assert workflow["active_run"]["business_date"] == "2026-08-03" + assert workflow["active_run"]["mode"] == "execute" + assert workflow["last_run"]["run_id"] == previous_run_id + assert workflow["last_run"]["business_date"] == "2026-08-02" + assert workflow["last_run"]["status"] == "failed" + + +class RecordingLockManager: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + self.active = False + + @contextmanager + def acquire( + self, + resource: str, + *, + owner: str, + timeout_seconds: float = 30.0, + poll_seconds: float = 0.1, + ) -> Iterator[object]: + del timeout_seconds, poll_seconds + self.calls.append((resource, owner)) + self.active = True + try: + yield object() + finally: + self.active = False + + +def test_schedule_update_uses_revision_and_named_lock_for_atomic_replacement( + monkeypatch: pytest.MonkeyPatch, + console_settings: Settings, +) -> None: + store = ScheduleConfigStore(console_settings) + revision = store.revision() + locks = RecordingLockManager() + store._locks = locks # type: ignore[assignment] + real_atomic_write = console_module.atomic_write_json + + def checked_atomic_write(path: Path | str, payload: Any) -> Path: + if Path(path) == store.path: + assert locks.active is True + return real_atomic_write(path, payload) + + monkeypatch.setattr(console_module, "atomic_write_json", checked_atomic_write) + + schedule, updated_revision = store.update( + "content.metrics.daily", + { + "kind": "daily", + "at": "06:15", + "enabled": False, + "business_date_offset_days": -1, + }, + expected_revision=f'"{revision}"', + ) + + assert locks.calls and locks.calls[0][0] == "config:schedules" + assert locks.calls[0][1].startswith("console-") + assert locks.active is False + assert updated_revision != revision + assert schedule.at == "06:15" + assert schedule.enabled is False + assert console_module._schedule_payload(schedule)["at"] == "06:15" + assert _schedule_row(console_settings, "content.metrics.daily") == { + "workflow_id": "content.metrics.daily", + "kind": "daily", + "at": "06:15", + "enabled": False, + "business_date_offset_days": -1, + } + + +def test_schedule_update_accepts_and_returns_multiple_start_times( + console_settings: Settings, +) -> None: + store = ScheduleConfigStore(console_settings) + + schedule, _revision = store.update( + "content.metrics.daily", + { + "kind": "daily", + "at": ["08:00", "16:00", "22:00"], + "enabled": True, + "business_date_offset_days": 0, + }, + expected_revision=store.revision(), + ) + + assert schedule.effective_times == ("08:00", "16:00", "22:00") + assert console_module._schedule_payload(schedule)["at"] == [ + "08:00", + "16:00", + "22:00", + ] + assert _schedule_row(console_settings, "content.metrics.daily")["at"] == [ + "08:00", + "16:00", + "22:00", + ] + + +@pytest.mark.parametrize( + ("schedule", "now", "expected"), + [ + ( + ScheduleEntry( + workflow_id="test.daily", + kind="daily", + at="08:00", + at_times=("08:00", "16:00", "22:00"), + ), + datetime(2026, 8, 6, 16, 30, tzinfo=timezone.utc), + "2026-08-06T22:00:00+00:00", + ), + ( + ScheduleEntry( + workflow_id="test.weekly", + kind="weekly", + at="08:00", + at_times=("08:00", "16:00", "22:00"), + days=("Thursday",), + ), + datetime(2026, 8, 6, 22, 30, tzinfo=timezone.utc), + "2026-08-13T08:00:00+00:00", + ), + ( + ScheduleEntry( + workflow_id="test.monthly", + kind="monthly", + at="08:00", + at_times=("08:00", "16:00", "22:00"), + day_of_month=6, + ), + datetime(2026, 8, 6, 16, 30, tzinfo=timezone.utc), + "2026-08-06T22:00:00+00:00", + ), + ( + ScheduleEntry( + workflow_id="test.interval", + kind="interval_days", + at="08:00", + at_times=("08:00", "16:00", "22:00"), + every_days=3, + anchor_date="2026-08-06", + ), + datetime(2026, 8, 6, 22, 30, tzinfo=timezone.utc), + "2026-08-09T08:00:00+00:00", + ), + ], +) +def test_next_run_uses_the_nearest_slot_across_all_start_times( + schedule: ScheduleEntry, + now: datetime, + expected: str, +) -> None: + assert console_module._next_run_at(schedule, now) == expected + + +def test_schedule_update_rejects_stale_revision_without_changing_file( + console_settings: Settings, +) -> None: + store = ScheduleConfigStore(console_settings) + before = store.path.read_bytes() + + with pytest.raises(ConsoleConflictError) as raised: + store.update( + "content.metrics.daily", + {"kind": "daily", "at": "06:15"}, + expected_revision="0" * 64, + ) + + assert raised.value.status == HTTPStatus.CONFLICT + assert store.path.read_bytes() == before + + +@pytest.mark.parametrize( + ("workflow_id", "module", "trigger", "expected_status"), + [ + ( + "content.console_manual", + "content_marketing", + "manual", + HTTPStatus.BAD_REQUEST, + ), + ( + "product.console_unavailable", + "product_commerce", + "unavailable", + HTTPStatus.BAD_REQUEST, + ), + ("missing.workflow", None, None, HTTPStatus.NOT_FOUND), + ], +) +def test_schedule_update_rejects_non_scheduled_or_unknown_workflows( + console_settings: Settings, + workflow_id: str, + module: str | None, + trigger: str | None, + expected_status: HTTPStatus, +) -> None: + if module is not None and trigger is not None: + _append_non_scheduled_workflow( + console_settings, + workflow_id=workflow_id, + module=module, + trigger=trigger, + ) + store = ScheduleConfigStore(console_settings) + before = store.path.read_bytes() + + with pytest.raises(ConsoleRequestError) as raised: + store.update( + workflow_id, + {"kind": "daily", "at": "06:15"}, + expected_revision=store.revision(), + ) + + assert raised.value.status == expected_status + assert store.path.read_bytes() == before + + +def test_schedule_kind_switch_removes_fields_owned_by_the_previous_kind( + console_settings: Settings, +) -> None: + store = ScheduleConfigStore(console_settings) + + updated, _revision = store.update( + "content.comments.weekly", + { + "kind": "daily", + "at": "07:45", + "enabled": True, + "business_date_offset_days": 0, + "days": ["Sunday"], + "day_of_month": 31, + "every_days": 3, + "anchor_date": "2026-08-01", + }, + expected_revision=store.revision(), + ) + + assert updated.kind == "daily" + assert updated.days == () + assert updated.day_of_month is None + assert updated.every_days is None + assert updated.anchor_date is None + assert _schedule_row(console_settings, "content.comments.weekly") == { + "workflow_id": "content.comments.weekly", + "kind": "daily", + "at": "07:45", + "enabled": True, + "business_date_offset_days": 0, + } + + +@pytest.mark.parametrize( + "payload", + [ + {"kind": "daily", "at": "9:00"}, + {"kind": "daily", "at": "09:00:30"}, + {"kind": "daily", "at": []}, + {"kind": "daily", "at": ["09:00", "09:00"]}, + {"kind": "daily", "at": ["09:00", "9:30"]}, + { + "kind": "interval_days", + "at": "09:00", + "every_days": 3, + "anchor_date": "not-a-date", + }, + { + "kind": "interval_days", + "at": "09:00", + "every_days": True, + "anchor_date": "2026-08-01", + }, + {"kind": "monthly", "at": "09:00", "day_of_month": True}, + { + "kind": "daily", + "at": "09:00", + "business_date_offset_days": True, + }, + ], +) +def test_invalid_schedule_values_leave_the_original_file_unchanged( + console_settings: Settings, + payload: dict[str, Any], +) -> None: + store = ScheduleConfigStore(console_settings) + before = store.path.read_bytes() + + with pytest.raises(ConsoleRequestError): + store.update( + "content.metrics.daily", + payload, + expected_revision=store.revision(), + ) + + assert store.path.read_bytes() == before + + +def test_service_trigger_supports_dry_run_confirmed_execute_and_conflict( + console_settings: Settings, +) -> None: + launcher = FakeConsoleLauncher() + service = WorkflowConsoleService(console_settings, launcher=launcher) + workflow_id = "content.metrics.daily" + + dry_run = service.trigger(workflow_id, {"business_date": "2026-08-01"}) + + assert dry_run["mode"] == "dry_run" + assert launcher.calls[-1]["mode"] == "dry_run" + launcher.complete(workflow_id) + + with pytest.raises(ConsoleRequestError, match="确认"): + service.trigger( + workflow_id, + {"business_date": "2026-08-01", "execute": True}, + ) + assert len(launcher.calls) == 1 + + executed = service.trigger( + workflow_id, + { + "business_date": "2026-08-01", + "execute": True, + "confirmed": True, + "shadow": True, + }, + ) + + assert executed["mode"] == "execute" + assert executed["shadow"] is True + with pytest.raises(ConsoleConflictError): + service.trigger(workflow_id, {"business_date": "2026-08-01"}) + assert len(launcher.calls) == 2 + + +def test_product_production_preflight_requires_real_database_and_lark_profile( + tmp_path: Path, +) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + daily = next( + item for item in catalog.workflows if item.workflow_id == "product.daily" + ) + config_dir = tmp_path / ".lark-cli" + config_dir.mkdir() + (config_dir / "config.json").write_text( + json.dumps({"apps": [{"name": "hermes-analyzer"}]}), + encoding="utf-8", + ) + environment = { + "PG_HOST": "database.example", + "PG_PORT": "5432", + "PG_DB": "business", + "PG_USER": "runtime", + "PG_PASSWORD": "placeholder", + "LARKSUITE_CLI_CONFIG_DIR": str(config_dir), + } + + console_module._validate_product_production_runtime(daily, environment) + + without_database = { + "LARKSUITE_CLI_CONFIG_DIR": str(config_dir), + } + with pytest.raises(ConsoleRequestError, match="PostgreSQL"): + console_module._validate_product_production_runtime(daily, without_database) + + empty_config_dir = tmp_path / "empty-lark-cli" + empty_config_dir.mkdir() + without_lark = { + **environment, + "LARKSUITE_CLI_CONFIG_DIR": str(empty_config_dir), + } + with pytest.raises(ConsoleRequestError, match="hermes-analyzer"): + console_module._validate_product_production_runtime(daily, without_lark) + + +def test_monthly_sales_preflight_does_not_require_postgres_or_hermes() -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + monthly_sales = next( + item + for item in catalog.workflows + if item.workflow_id == "product.sales_sheet.daily" + ) + + console_module._validate_product_production_runtime(monthly_sales, {}) + + with pytest.raises(ConsoleRequestError, match="验收隔离"): + console_module._validate_product_production_runtime( + monthly_sales, {"GYXX_WORKFLOW_ACCEPTANCE": "1"} + ) + + +def test_product_production_preflight_rejects_acceptance_and_missing_owner( + tmp_path: Path, +) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + alert = next( + item + for item in catalog.workflows + if item.workflow_id == "product.alert.daily" + ) + config_dir = tmp_path / ".lark-cli" + config_dir.mkdir() + (config_dir / "config.json").write_text( + json.dumps({"apps": [{"name": "hermes-analyzer"}]}), + encoding="utf-8", + ) + environment = { + "GYXX_POSTGRES_DSN": "postgresql://runtime:${PG_PASSWORD}@database.example/business", + "GYXX_HERMES_API_KEY": "placeholder", + "LARKSUITE_CLI_CONFIG_DIR": str(config_dir), + } + + with pytest.raises(ConsoleRequestError, match="负责人"): + console_module._validate_product_production_runtime(alert, environment) + + environment["GYXX_NOTIFICATION_RECIPIENT_OPEN_ID"] = "configured-owner" + environment["GYXX_WORKFLOW_ACCEPTANCE"] = "1" + with pytest.raises(ConsoleRequestError, match="验收隔离"): + console_module._validate_product_production_runtime(alert, environment) + + +class StubProcess: + def __init__(self, pid: int) -> None: + self.pid = pid + self.return_code: int | None = None + + def poll(self) -> int | None: + return self.return_code + + +class StubTreeProcess: + def __init__( + self, + pid: int, + *, + started_at: float, + popen: StubProcess | None = None, + ignore_terminate: bool = False, + ) -> None: + self.pid = pid + self.started_at = started_at + self.popen = popen + self.ignore_terminate = ignore_terminate + self.alive = True + self.terminated = False + self.killed = False + self.descendants: list[StubTreeProcess] = [] + + def create_time(self) -> float: + return self.started_at + + def children(self, *, recursive: bool) -> list[StubTreeProcess]: + assert recursive is True + return list(self.descendants) + + def terminate(self) -> None: + self.terminated = True + if not self.ignore_terminate: + self.alive = False + if self.popen is not None: + self.popen.return_code = -15 + + def kill(self) -> None: + self.killed = True + self.alive = False + if self.popen is not None: + self.popen.return_code = -9 + + def is_running(self) -> bool: + return self.alive + + def status(self) -> str: + return psutil.STATUS_RUNNING if self.alive else psutil.STATUS_ZOMBIE + + def wait(self, timeout: float | None = None) -> int: + if self.alive: + raise psutil.TimeoutExpired(timeout or 0, pid=self.pid) + return -9 if self.killed else -15 + + +def test_subprocess_launcher_uses_fixed_argv_environment_and_unique_operations( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + settings = Settings( + project_root=tmp_path / "project", + data_root=tmp_path / "data", + ) + python = tmp_path / "runtime" / "python.exe" + calls: list[tuple[list[str], dict[str, Any], StubProcess]] = [] + + def fake_popen(argv: list[str], **kwargs: Any) -> StubProcess: + process = StubProcess(9000 + len(calls)) + calls.append((argv, kwargs, process)) + return process + + monkeypatch.setattr(console_module.subprocess, "Popen", fake_popen) + launcher = SubprocessConsoleRunLauncher(settings, python_executable=python) + + first = launcher.launch( + "content.metrics.daily", + date(2026, 8, 1), + execute=True, + shadow=True, + ) + second = launcher.launch( + "product.daily", + date(2026, 7, 31), + execute=False, + shadow=False, + ) + + assert first["operation_id"] != second["operation_id"] + assert re.fullmatch(r"op-[0-9a-f]{16}", str(first["operation_id"])) + assert calls[0][0] == [ + str(python.resolve()), + "-m", + "gyxx_flow", + "run", + "content.metrics.daily", + "--date", + "2026-08-01", + "--execute", + "--shadow", + ] + assert calls[1][0] == [ + str(python.resolve()), + "-m", + "gyxx_flow", + "run", + "product.daily", + "--date", + "2026-07-31", + ] + for _argv, kwargs, _process in calls: + assert kwargs["cwd"] == settings.project_root + assert kwargs["shell"] is False + assert kwargs["stdin"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.STDOUT + assert kwargs["env"]["GYXX_PROJECT_ROOT"] == str(settings.project_root) + assert kwargs["env"]["GYXX_DATA_ROOT"] == str(settings.data_root) + assert str(kwargs["stdout"].name).endswith(".log") + assert str(first["operation_id"]) in str(calls[0][1]["stdout"].name) + assert str(second["operation_id"]) in str(calls[1][1]["stdout"].name) + + for _argv, _kwargs, process in calls: + process.return_code = 0 + assert launcher.active() == () + assert all(kwargs["stdout"].closed for _argv, kwargs, _process in calls) + + +def test_subprocess_launcher_cancels_exact_operation_tree_and_journal( + monkeypatch: pytest.MonkeyPatch, + console_settings: Settings, +) -> None: + calls: list[tuple[dict[str, Any], StubProcess]] = [] + roots: dict[int, StubTreeProcess] = {} + children: dict[int, StubTreeProcess] = {} + grandchildren: dict[int, StubTreeProcess] = {} + real_psutil_process = console_module.psutil.Process + + def fake_popen(_argv: list[str], **kwargs: Any) -> StubProcess: + pid = 9100 + len(calls) + popen = StubProcess(pid) + root = StubTreeProcess(pid, started_at=1_800_000_000.0, popen=popen) + child = StubTreeProcess( + pid + 100, + started_at=1_800_000_001.0, + ignore_terminate=True, + ) + grandchild = StubTreeProcess( + pid + 200, + started_at=1_800_000_002.0, + ignore_terminate=True, + ) + child.descendants.append(grandchild) + root.descendants.extend((child, grandchild)) + roots[pid] = root + children[pid] = child + grandchildren[pid] = grandchild + calls.append((kwargs, popen)) + return popen + + def fake_wait_procs( + processes: list[StubTreeProcess], + *, + timeout: float, + ) -> tuple[list[StubTreeProcess], list[StubTreeProcess]]: + assert timeout > 0 + gone = [process for process in processes if not process.alive] + alive = [process for process in processes if process.alive] + return gone, alive + + monkeypatch.setattr(console_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr( + console_module.psutil, + "Process", + lambda pid: roots.get(pid) or real_psutil_process(pid), + ) + monkeypatch.setattr(console_module.psutil, "wait_procs", fake_wait_procs) + launcher = SubprocessConsoleRunLauncher(console_settings) + workflow_id = "content.metrics.daily" + started = launcher.launch( + workflow_id, + date(2026, 8, 2), + execute=True, + shadow=False, + ) + operation_id = str(started["operation_id"]) + popen = calls[0][1] + + context = RunContext.create( + workflow_id, + "2026-08-02", + random_suffix="cancel1", + ) + journal = RunJournal.create(DataLayout(console_settings.data_root), context) + RunIndex(console_settings.data_root).index_journal(journal) + journal.start_step("collect.attempt-1", attempt=1) + lock_root = console_settings.data_root / "state" / "locks" + lock_root.mkdir(parents=True, exist_ok=True) + (lock_root / "workflow.lock").write_text( + json.dumps( + { + "resource": f"workflow:{workflow_id}", + "owner": context.run_id, + "pid": popen.pid, + "process_started_at": roots[popen.pid].started_at, + } + ), + encoding="utf-8", + ) + + with pytest.raises(ConsoleConflictError, match="刷新"): + launcher.cancel(workflow_id, "op-0000000000000000") + assert roots[popen.pid].terminated is False + + cancelled = launcher.cancel(workflow_id, operation_id) + + assert cancelled["status"] == "cancelled" + assert cancelled["journal_status"] == "cancelled" + assert cancelled["run_id"] == context.run_id + assert roots[popen.pid].terminated is True + assert children[popen.pid].terminated is True + assert children[popen.pid].killed is True + assert grandchildren[popen.pid].terminated is True + assert grandchildren[popen.pid].killed is True + assert popen.poll() is not None + assert calls[0][0]["stdout"].closed is True + assert launcher.active() == () + indexed = RunIndex(console_settings.data_root).get(context.run_id) + assert indexed is not None + assert indexed.status == "cancelled" + assert indexed.step_counts == { + "success": 0, + "failed": 0, + "skipped": 1, + "running": 0, + } + journal_payload = json.loads(journal.path.read_text(encoding="utf-8")) + assert journal_payload["status"] == "cancelled" + assert journal_payload["steps"]["collect.attempt-1"]["status"] == "skipped" + assert journal_payload["steps"]["collect.attempt-1"]["ended_at"] is not None + + repeated = launcher.cancel(workflow_id, operation_id) + assert repeated["status"] == "cancelled" + assert repeated["already_cancelled"] is True + cancellation_path = ( + console_settings.data_root + / "state" + / "ops" + / "console-cancellations" + / f"{operation_id}.json" + ) + assert json.loads(cancellation_path.read_text(encoding="utf-8"))[ + "journal_status" + ] == "cancelled" + + immediate = launcher.launch( + "product.daily", + date(2026, 8, 2), + execute=True, + shadow=False, + ) + immediate_result = launcher.cancel( + "product.daily", + str(immediate["operation_id"]), + ) + assert immediate_result["journal_status"] == "synthetic_cancelled" + synthetic = RunIndex(console_settings.data_root).get( + str(immediate_result["run_id"]) + ) + assert synthetic is not None + assert synthetic.status == "cancelled" + overview = WorkflowConsoleService( + console_settings, + launcher=launcher, + ).overview() + product = next( + item for item in overview["workflows"] if item["id"] == "product.daily" + ) + assert product["active_run"] is None + assert product["last_run"]["status"] == "cancelled" + + +def test_subprocess_launcher_refuses_pid_identity_change( + monkeypatch: pytest.MonkeyPatch, + console_settings: Settings, +) -> None: + roots: dict[int, StubTreeProcess] = {} + streams: list[TextIO] = [] + real_psutil_process = console_module.psutil.Process + + def fake_popen(_argv: list[str], **kwargs: Any) -> StubProcess: + process = StubProcess(9200) + roots[process.pid] = StubTreeProcess( + process.pid, + started_at=1_800_000_000.0, + popen=process, + ) + streams.append(kwargs["stdout"]) + return process + + monkeypatch.setattr(console_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr( + console_module.psutil, + "Process", + lambda pid: roots.get(pid) or real_psutil_process(pid), + ) + launcher = SubprocessConsoleRunLauncher(console_settings) + launched = launcher.launch( + "content.metrics.daily", + date(2026, 8, 2), + execute=False, + shadow=False, + ) + roots[9200].started_at += 10 + + with pytest.raises(ConsoleConflictError, match="身份已变化"): + launcher.cancel( + "content.metrics.daily", + str(launched["operation_id"]), + ) + + assert roots[9200].terminated is False + assert streams[0].closed is False + assert len(launcher.active()) == 1 + assert roots[9200].popen is not None + roots[9200].popen.return_code = 0 + roots[9200].alive = False + assert launcher.active() == () + assert streams[0].closed is True + + +def test_subprocess_launcher_recovers_active_operation_after_console_restart( + monkeypatch: pytest.MonkeyPatch, + console_settings: Settings, +) -> None: + calls: list[tuple[dict[str, Any], StubProcess]] = [] + roots: dict[int, StubTreeProcess] = {} + real_psutil_process = console_module.psutil.Process + + def fake_popen(_argv: list[str], **kwargs: Any) -> StubProcess: + process = StubProcess(9300) + roots[process.pid] = StubTreeProcess( + process.pid, + started_at=1_800_000_000.0, + popen=process, + ) + calls.append((kwargs, process)) + return process + + def fake_wait_procs( + processes: list[StubTreeProcess], + *, + timeout: float, + ) -> tuple[list[StubTreeProcess], list[StubTreeProcess]]: + assert timeout > 0 + return ( + [process for process in processes if not process.alive], + [process for process in processes if process.alive], + ) + + monkeypatch.setattr(console_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr( + console_module.psutil, + "Process", + lambda pid: roots.get(pid) or real_psutil_process(pid), + ) + monkeypatch.setattr(console_module.psutil, "wait_procs", fake_wait_procs) + + first_launcher = SubprocessConsoleRunLauncher(console_settings) + started = first_launcher.launch( + "content.metrics.daily", + date(2026, 8, 2), + execute=True, + shadow=False, + ) + operation_id = str(started["operation_id"]) + active_path = ( + console_settings.data_root + / "state" + / "ops" + / "console-active" + / f"{operation_id}.json" + ) + assert active_path.exists() + calls[0][0]["stdout"].close() + + recovered_launcher = SubprocessConsoleRunLauncher(console_settings) + + assert recovered_launcher.active() == ( + { + "operation_id": operation_id, + "workflow_id": "content.metrics.daily", + "business_date": "2026-08-02", + "mode": "execute", + "shadow": False, + "started_at": started["started_at"], + "status": "running", + }, + ) + cancelled = recovered_launcher.cancel("content.metrics.daily", operation_id) + assert cancelled["status"] == "cancelled" + assert cancelled["journal_status"] == "synthetic_cancelled" + assert active_path.exists() is False + assert roots[9300].terminated is True + + +def test_subprocess_launcher_discards_stale_recovery_record_without_terminating( + monkeypatch: pytest.MonkeyPatch, + console_settings: Settings, +) -> None: + streams: list[TextIO] = [] + roots: dict[int, StubTreeProcess] = {} + real_psutil_process = console_module.psutil.Process + + def fake_popen(_argv: list[str], **kwargs: Any) -> StubProcess: + process = StubProcess(9400) + roots[process.pid] = StubTreeProcess( + process.pid, + started_at=1_800_000_000.0, + popen=process, + ) + streams.append(kwargs["stdout"]) + return process + + monkeypatch.setattr(console_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr( + console_module.psutil, + "Process", + lambda pid: roots.get(pid) or real_psutil_process(pid), + ) + first_launcher = SubprocessConsoleRunLauncher(console_settings) + started = first_launcher.launch( + "content.metrics.daily", + date(2026, 8, 2), + execute=False, + shadow=False, + ) + operation_id = str(started["operation_id"]) + active_path = ( + console_settings.data_root + / "state" + / "ops" + / "console-active" + / f"{operation_id}.json" + ) + streams[0].close() + roots[9400].started_at += 30 + + recovered_launcher = SubprocessConsoleRunLauncher(console_settings) + + assert recovered_launcher.active() == () + assert active_path.exists() is False + assert roots[9400].terminated is False + + +@contextmanager +def _running_server( + settings: Settings, + launcher: FakeConsoleLauncher, + *, + token: str | None = None, +) -> Iterator[console_module.WorkflowConsoleHTTPServer]: + server = create_console_server( + settings, + host="127.0.0.1", + port=0, + token=token, + launcher=launcher, + ) + thread = threading.Thread( + target=server.serve_forever, + kwargs={"poll_interval": 0.01}, + daemon=True, + ) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _http_request( + server: console_module.WorkflowConsoleHTTPServer, + method: str, + path: str, + *, + body: bytes | None = None, + headers: dict[str, str] | None = None, +) -> tuple[int, dict[str, str], bytes]: + connection = http.client.HTTPConnection( + "127.0.0.1", server.server_port, timeout=5 + ) + try: + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + content = response.read() + return response.status, dict(response.getheaders()), content + finally: + connection.close() + + +def _assert_security_headers(headers: dict[str, str]) -> None: + assert "default-src 'self'" in headers["Content-Security-Policy"] + assert headers["X-Content-Type-Options"] == "nosniff" + assert headers["X-Frame-Options"] == "DENY" + assert headers["Referrer-Policy"] == "no-referrer" + assert "Access-Control-Allow-Origin" not in headers + + +def test_http_home_assets_and_overview_have_security_headers( + console_settings: Settings, +) -> None: + with _running_server(console_settings, FakeConsoleLauncher()) as server: + status, headers, content = _http_request(server, "GET", "/") + assert status == HTTPStatus.OK + _assert_security_headers(headers) + page = content.decode("utf-8") + assert "/assets/app.css" in page + assert "/assets/app.js" in page + assert page.count("data-custom-select") == 2 + assert 'data-description="显示所有触发类型"' in page + assert 'data-description="显示全部运行结果"' in page + assert 'id="drawer-stop-button"' in page + assert 'id="stop-overlay"' in page + assert 'role="alertdialog"' in page + assert 'id="drawer-rename-button"' in page + assert 'id="drawer-name-form"' in page + assert "执行流程" in page + + for path, content_type in ( + ("/assets/app.css", "text/css"), + ("/assets/app.js", "text/javascript"), + ): + status, asset_headers, asset = _http_request(server, "GET", path) + assert status == HTTPStatus.OK + assert asset + assert asset_headers["Content-Type"].startswith(content_type) + _assert_security_headers(asset_headers) + decoded_asset = asset.decode("utf-8") + if path.endswith("app.js"): + assert "并行执行" in decoded_asset + assert "节点间连线只表示真实执行依赖" in decoded_asset + assert "技术实现与运行规则" in decoded_asset + assert "数据来源" in decoded_asset + assert "节点内处理" in decoded_asset + assert "结果去向" in decoded_asset + assert "renderStepBusinessFlow" in decoded_asset + assert "persistDrawerDisplayName" in decoded_asset + assert 'method: "PUT"' in decoded_asset + assert "display-name" in decoded_asset + assert "跳过原因" in decoded_asset + assert "topology-node__diagnostic" in decoded_asset + assert "正在启动工作流" in decoded_asset + assert "手动触发" in decoded_asset + assert "path.dataset.edgeState = edgeState" in decoded_asset + assert "预演完成" in decoded_asset + assert "预演完成（全部跳过）" in decoded_asset + assert "正式执行成功" in decoded_asset + assert "正式执行完成（全部跳过）" in decoded_asset + assert "历史模式未知" in decoded_asset + assert "allStepsSkipped" in decoded_asset + assert "最近成功" not in decoded_asset + assert "scheduleOverviewPoll" in decoded_asset + assert "function scheduleTimes(schedule)" in decoded_asset + assert "多个时间用逗号分隔" in decoded_asset + assert 'times.length === 1 ? times[0] : times' in decoded_asset + assert "replaceHtmlIfChanged" in decoded_asset + assert "HISTORY_CACHE_TTL_MS" in decoded_asset + assert 'id="topology-refreshed-at"' in decoded_asset + assert 'data-workflow-id="${esc(workflow.id)}"' in decoded_asset + assert 'openDrawer(workflow.id, "steps")' in decoded_asset + assert "state.data?.scheduler?.service_running" in decoded_asset + assert "节点错误详情" in decoded_asset + assert "naturalMonthRange" in decoded_asset + assert "聚水潭查询范围" in decoded_asset + assert 'periodInput.type = monthMode ? "month" : "date"' in decoded_asset + assert 'const businessDate = monthMode ? `${selectedPeriod}-01`' in decoded_asset + assert "renderCardLiveState" in decoded_asset + assert 'data-live-progress="${completed}/${steps.length}"' in decoded_asset + assert 'data-current-step="${esc(currentNames.join("、"))}"' in decoded_asset + assert 'data-step-id="${esc(step.id)}"' in decoded_asset + assert 'data-action="steps"' in decoded_asset + assert "已处理 ${completed} / ${steps.length}" in decoded_asset + assert "topologyDisplayRun(workflow)" in decoded_asset + assert "const displayRun = busy ? topologyDisplayRun(workflow) : last;" in decoded_asset + assert 'aria-haspopup="listbox"' in decoded_asset + assert 'role="option"' in decoded_asset + assert "closeOpenCustomSelect" in decoded_asset + assert "initCustomSelects" in decoded_asset + assert 'data-action="stop"' in decoded_asset + assert 'method: "DELETE"' in decoded_asset + assert "body: {}" in decoded_asset + assert "正在停止…" in decoded_asset + if path.endswith("app.css"): + assert ".workflow-topology" in decoded_asset + assert ".topology-edge--continue" in decoded_asset + assert ".topology-edge--active" in decoded_asset + assert "@keyframes topology-node-breathe" in decoded_asset + assert '.topology-node[data-state="queued"]' in decoded_asset + assert ".run-step-detail" in decoded_asset + assert ".card-live__step[data-state=\"running\"]" in decoded_asset + assert "@keyframes card-live-flow" in decoded_asset + assert ".custom-select__menu" in decoded_asset + assert "@keyframes custom-select-in" in decoded_asset + assert ".card-action--stop" in decoded_asset + assert ".stop-modal" in decoded_asset + assert ".topology-node__business-flow" in decoded_asset + assert ".business-flow__destination" in decoded_asset + assert ".drawer-name-editor" in decoded_asset + + status, api_headers, body = _http_request(server, "GET", "/api/overview") + assert status == HTTPStatus.OK + _assert_security_headers(api_headers) + payload = json.loads(body) + assert payload["summary"]["total"] == len(payload["workflows"]) + assert {module["id"] for module in payload["modules"]} == { + "content_marketing", + "product_commerce", + "shop_intelligence", + "supply_chain", + } + + +def test_http_token_protects_api_and_never_relaxes_security_headers( + console_settings: Settings, +) -> None: + token = "console-token-that-is-long-enough" + with _running_server( + console_settings, FakeConsoleLauncher(), token=token + ) as server: + status, headers, _body = _http_request(server, "GET", "/api/overview") + assert status == HTTPStatus.UNAUTHORIZED + assert headers["WWW-Authenticate"].startswith("Bearer") + _assert_security_headers(headers) + + status, _headers, _body = _http_request( + server, + "GET", + "/api/overview", + headers={"Authorization": "Bearer wrong"}, + ) + assert status == HTTPStatus.UNAUTHORIZED + + status, headers, body = _http_request( + server, + "GET", + "/api/overview", + headers={"Authorization": f"Bearer {token}"}, + ) + assert status == HTTPStatus.OK + _assert_security_headers(headers) + assert json.loads(body)["summary"]["total"] > 0 + + +def test_http_mutations_require_json_marker_same_origin_and_bounded_body( + console_settings: Settings, +) -> None: + launcher = FakeConsoleLauncher() + body = json.dumps({"business_date": "2026-08-01"}).encode("utf-8") + path = "/api/workflows/content.metrics.daily/runs" + with _running_server(console_settings, launcher) as server: + origin = f"http://127.0.0.1:{server.server_port}" + + status, _headers, _body = _http_request( + server, + "POST", + path, + body=body, + headers={"Content-Type": "application/json"}, + ) + assert status == HTTPStatus.FORBIDDEN + + status, _headers, _body = _http_request( + server, + "POST", + path, + body=body, + headers={"Content-Type": "text/plain", "X-GYXX-Console": "1"}, + ) + assert status == HTTPStatus.UNSUPPORTED_MEDIA_TYPE + + status, _headers, _body = _http_request( + server, + "POST", + path, + body=body, + headers={ + "Content-Type": "application/json", + "X-GYXX-Console": "1", + "Origin": "https://evil.example", + }, + ) + assert status == HTTPStatus.FORBIDDEN + + oversized = b"{" + b"x" * MAX_REQUEST_BYTES + status, _headers, _body = _http_request( + server, + "POST", + path, + body=oversized, + headers={ + "Content-Type": "application/json", + "X-GYXX-Console": "1", + "Origin": origin, + }, + ) + assert status == HTTPStatus.BAD_REQUEST + + status, headers, accepted = _http_request( + server, + "POST", + path, + body=body, + headers={ + "Content-Type": "application/json; charset=utf-8", + "X-GYXX-Console": "1", + "Origin": origin, + }, + ) + assert status == HTTPStatus.ACCEPTED + _assert_security_headers(headers) + accepted_payload = json.loads(accepted) + assert accepted_payload["mode"] == "dry_run" + assert len(launcher.calls) == 1 + + cancel_path = f"{path}/{accepted_payload['operation_id']}" + status, _headers, _body = _http_request( + server, + "DELETE", + cancel_path, + headers={"Content-Type": "application/json"}, + ) + assert status == HTTPStatus.FORBIDDEN + + status, _headers, _body = _http_request( + server, + "DELETE", + cancel_path, + headers={ + "Content-Type": "application/json", + "X-GYXX-Console": "1", + "Origin": "https://evil.example", + }, + ) + assert status == HTTPStatus.FORBIDDEN + + mutation_headers = { + "Content-Type": "application/json", + "X-GYXX-Console": "1", + "Origin": origin, + } + connection = http.client.HTTPConnection( + "127.0.0.1", server.server_port, timeout=5 + ) + try: + connection.request( + "DELETE", + f"{path}/op-0000000000000000", + body=b"{}", + headers=mutation_headers, + ) + conflict = connection.getresponse() + assert conflict.status == HTTPStatus.CONFLICT + conflict.read() + + connection.request("GET", "/api/overview") + overview = connection.getresponse() + assert overview.status == HTTPStatus.OK + assert json.loads(overview.read())["summary"]["running"] == 1 + finally: + connection.close() + + status, cancel_headers, cancelled = _http_request( + server, + "DELETE", + cancel_path, + body=b"{}", + headers=mutation_headers, + ) + assert status == HTTPStatus.OK + _assert_security_headers(cancel_headers) + assert json.loads(cancelled)["status"] == "cancelled" + assert launcher.active() == () + + status, _headers, repeated = _http_request( + server, + "DELETE", + cancel_path, + body=b"{}", + headers=mutation_headers, + ) + assert status == HTTPStatus.OK + assert json.loads(repeated)["already_cancelled"] is True + + +def test_http_updates_and_resets_workflow_display_name( + console_settings: Settings, +) -> None: + launcher = FakeConsoleLauncher() + workflow_id = "content.marketing_report.daily" + path = f"/api/workflows/{workflow_id}/display-name" + with _running_server(console_settings, launcher) as server: + origin = f"http://127.0.0.1:{server.server_port}" + status, _headers, overview_body = _http_request( + server, + "GET", + "/api/overview", + ) + assert status == HTTPStatus.OK + revision = json.loads(overview_body)["workflow_names_revision"] + headers = { + "Content-Type": "application/json", + "X-GYXX-Console": "1", + "Origin": origin, + } + body = json.dumps( + {"display_name": "每日内容营销简报"}, + ensure_ascii=False, + ).encode("utf-8") + + status, _headers, _body = _http_request( + server, + "PUT", + path, + body=body, + headers=headers, + ) + assert status == HTTPStatus.PRECONDITION_REQUIRED + + status, response_headers, renamed_body = _http_request( + server, + "PUT", + path, + body=body, + headers={**headers, "If-Match": revision}, + ) + assert status == HTTPStatus.OK + _assert_security_headers(response_headers) + renamed = json.loads(renamed_body) + assert renamed["name"] == "每日内容营销简报" + assert renamed["name_customized"] is True + + status, _headers, _body = _http_request( + server, + "PUT", + path, + body=body, + headers={**headers, "If-Match": revision}, + ) + assert status == HTTPStatus.CONFLICT + + status, _headers, refreshed_body = _http_request( + server, + "GET", + "/api/overview", + ) + assert status == HTTPStatus.OK + refreshed = json.loads(refreshed_body) + workflow = next( + item for item in refreshed["workflows"] if item["id"] == workflow_id + ) + assert workflow["name"] == "每日内容营销简报" + assert workflow["default_name"] == "营销日报生成与发送" + + reset_body = json.dumps({"display_name": None}).encode("utf-8") + status, _headers, reset_response = _http_request( + server, + "PUT", + path, + body=reset_body, + headers={ + **headers, + "If-Match": refreshed["workflow_names_revision"], + }, + ) + assert status == HTTPStatus.OK + assert json.loads(reset_response)["name"] == "营销日报生成与发送" + + +def test_non_loopback_binding_requires_a_long_token( + console_settings: Settings, +) -> None: + with pytest.raises(ValueError, match="requires GYXX_CONSOLE_TOKEN"): + create_console_server( + console_settings, + host="0.0.0.0", + port=0, + launcher=FakeConsoleLauncher(), + ) + + +class NeverLaunchSchedulerWorkflows: + def launch( + self, + workflow_id: str, + slot: datetime, + business_date: date, + ) -> object: + raise AssertionError( + f"dry-run scheduler unexpectedly launched {workflow_id} {slot} {business_date}" + ) + + def close(self, process: object) -> None: + raise AssertionError(f"unexpected close: {process}") + + +def _single_workflow_catalog(*, at: str, enabled: bool) -> WorkflowCatalog: + workflow = WorkflowEntry( + workflow_id="content.console_schedule", + module="content_marketing", + trigger="scheduled", + entry="run.py", + source_project="content", + source_task_name="console-schedule", + ) + schedule = ScheduleEntry( + workflow_id=workflow.workflow_id, + kind="daily", + at=at, + enabled=enabled, + ) + return WorkflowCatalog( + timezone="Asia/Shanghai", + workflows=(workflow,), + schedules=(schedule,), + ) + + +def test_scheduler_reload_catalog_runs_before_each_tick(tmp_path: Path) -> None: + initial = _single_workflow_catalog(at="09:00", enabled=True) + due_catalog = _single_workflow_catalog(at="10:00", enabled=True) + disabled_catalog = _single_workflow_catalog(at="10:00", enabled=False) + loaded = iter((due_catalog, disabled_catalog)) + load_calls: list[WorkflowCatalog] = [] + + def load_catalog() -> WorkflowCatalog: + catalog = next(loaded) + load_calls.append(catalog) + return catalog + + scheduler = PythonScheduler( + initial, + tmp_path, + launcher=NeverLaunchSchedulerWorkflows(), + misfire_grace_seconds=600, + dry_run=True, + catalog_loader=load_catalog, + ) + now = datetime.fromisoformat("2026-08-01T10:05:00+08:00") + + assert scheduler.tick(now) == ["content.console_schedule"] + assert scheduler.catalog is due_catalog + assert scheduler.tick(now) == [] + assert scheduler.catalog is disabled_catalog + assert load_calls == [due_catalog, disabled_catalog] diff --git a/tests/test_content_data_layout.py b/tests/test_content_data_layout.py index 80f31e0..f8456e4 100644 --- a/tests/test_content_data_layout.py +++ b/tests/test_content_data_layout.py @@ -4,16 +4,16 @@ from pathlib import Path import pytest -from gyxx_flow.modules.content_marketing.runtime import runtime_paths +from gyxx_flow.modules.content_marketing import runtime_paths PROJECT_ROOT = Path(__file__).resolve().parents[1] -RUNTIME_ROOT = ( - PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "content_marketing" / "runtime" +MODULE_ROOT = ( + PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "content_marketing" ) def _source(relative: str) -> str: - return (RUNTIME_ROOT / relative).read_text(encoding="utf-8-sig") + return (MODULE_ROOT / relative).read_text(encoding="utf-8-sig") def test_content_output_resolver_keeps_paths_inside_the_selected_layer( @@ -93,14 +93,9 @@ def test_transient_content_artifacts_use_tmp_root() -> None: assert 'chart_path.with_name(f"{chart_path.stem}_feishu.png")' not in report_card -def test_batch_launchers_never_fall_back_to_module_local_var() -> None: - for path in sorted((RUNTIME_ROOT / "data" / "tools").glob("*.bat")): - source = path.read_text(encoding="utf-8-sig") - if "GYXX_DATA_ROOT" not in source: - continue - assert '%PROJECT_DIR%\\var' not in source, path.name - assert '%GYXX_PROJECT_ROOT%\\var' in source, path.name - assert "if not defined GYXX_PROJECT_ROOT" in source, path.name +def test_content_module_does_not_ship_windows_launchers() -> None: + assert list(MODULE_ROOT.rglob("*.bat")) == [] + assert list(MODULE_ROOT.rglob("*.ps1")) == [] @pytest.mark.parametrize( @@ -119,4 +114,3 @@ def test_cli_output_overrides_are_constrained_to_a_data_layer( assert "resolve_layer_output(" in source assert layer_expression in source - diff --git a/tests/test_content_marketing_module.py b/tests/test_content_marketing_module.py index 97c385f..677fc61 100644 --- a/tests/test_content_marketing_module.py +++ b/tests/test_content_marketing_module.py @@ -5,7 +5,6 @@ from pathlib import Path from gyxx_flow.adapters.native import DeferredModuleCommandStep from gyxx_flow.catalog import WorkflowCatalog from gyxx_flow.modules.content_marketing import ( - CONTENT_MANUAL_WORKFLOW_IDS, CONTENT_RESOURCE, CONTENT_SCHEDULED_WORKFLOW_IDS, CONTENT_TIMEOUT_SECONDS, @@ -16,15 +15,14 @@ from gyxx_flow.modules.content_marketing import ( PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_content_module_registers_all_scheduled_and_manual_entries() -> None: +def test_content_module_registers_only_scheduled_entries() -> None: module = ContentMarketingModule.from_catalog( WorkflowCatalog.load(PROJECT_ROOT / "config") ) definitions = module.workflow_definitions() assert tuple(item.workflow_id for item in definitions) == CONTENT_WORKFLOW_IDS - assert len(CONTENT_SCHEDULED_WORKFLOW_IDS) == 9 - assert len(CONTENT_MANUAL_WORKFLOW_IDS) == 3 + assert len(CONTENT_SCHEDULED_WORKFLOW_IDS) == 8 def test_content_workflows_use_project_owned_module_commands() -> None: @@ -33,11 +31,15 @@ def test_content_workflows_use_project_owned_module_commands() -> None: ) for definition in module.workflow_definitions(): - step = definition.steps[0] - assert step.step_id == "module_run" - assert isinstance(step.action, DeferredModuleCommandStep) - assert step.timeout_seconds == CONTENT_TIMEOUT_SECONDS == 4 * 60 * 60 - assert step.max_attempts == 1 - assert step.resources == (CONTENT_RESOURCE,) == ("module:content_marketing",) - assert step.production_sink is True - + assert definition.steps + for step in definition.steps: + assert isinstance(step.action, DeferredModuleCommandStep) + assert step.timeout_seconds == CONTENT_TIMEOUT_SECONDS == 4 * 60 * 60 + assert step.max_attempts == 1 + if definition.workflow_id == "content.comments.weekly": + assert step.resources == (f"{CONTENT_RESOURCE}:{step.step_id}",) + else: + assert step.resources == (CONTENT_RESOURCE,) == ( + "module:content_marketing", + ) + assert step.production_sink is True diff --git a/tests/test_content_source_snapshot.py b/tests/test_content_source_snapshot.py index e6fcaf8..7d20d76 100644 --- a/tests/test_content_source_snapshot.py +++ b/tests/test_content_source_snapshot.py @@ -8,8 +8,8 @@ from pathlib import Path, PurePosixPath from gyxx_flow.security.scanner import scan_repository PROJECT_ROOT = Path(__file__).parents[1] -RUNTIME_ROOT = ( - PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "content_marketing" / "runtime" +MODULE_ROOT = ( + PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "content_marketing" ) MANIFEST_PATH = ( PROJECT_ROOT / "config" / "source-manifests" / "content_marketing.json" @@ -32,10 +32,8 @@ FIXED_USER_PATHS = ( EXPECTED_CATEGORIES = { "config", "dependency", - "launcher", "source", "sql", - "test", "tool", } REQUIRED_SOURCES = { @@ -44,14 +42,13 @@ REQUIRED_SOURCES = { "pgy_xhs_scraper_v2.py", "xingtu_scraper_v2.py", "chanmama_scraper.py", + "daily_creator_exposure_scope.py", "daily_marketing_report.py", - "data/tools/daily_run.bat", "data/tools/friday_relogin_parallel.py", "data/tools/schema_gyxx_super_data.sql", "data/tools/migrations/005_style_product_profile.sql", "data/config/db.env.example", "data/config/款式_多维表格_对照.json", - "tests/test_collection_completeness.py", } @@ -74,7 +71,7 @@ def test_content_source_snapshot_manifest_is_complete_and_well_formed() -> None: assert manifest["module"] == "content_marketing" files = manifest["files"] assert isinstance(files, list) - assert len(files) == 90 + assert len(files) == 65 sources = [entry["source_relative_path"] for entry in files] targets = [entry["target_relative_path"] for entry in files] @@ -88,13 +85,13 @@ def test_content_source_snapshot_manifest_is_complete_and_well_formed() -> None: relative = PurePosixPath(entry["target_relative_path"]) assert not relative.is_absolute() assert ".." not in relative.parts - assert relative.parts[:5] == ( + assert relative.parts[:4] == ( "src", "gyxx_flow", "modules", "content_marketing", - "runtime", ) + assert relative.parts[4] != "runtime" assert SHA256.fullmatch(entry["source_sha256"]) assert SHA256.fullmatch(entry["target_sha256"]) assert entry["transformed"] is ( @@ -110,7 +107,7 @@ def test_content_source_snapshot_targets_exist_and_match_manifest_hashes() -> No generated = _manifest()["generated_files"] assert [entry["target_relative_path"] for entry in generated] == [ - "src/gyxx_flow/modules/content_marketing/runtime/runtime_paths.py" + "src/gyxx_flow/modules/content_marketing/runtime_paths.py" ] for entry in generated: target = PROJECT_ROOT.joinpath(*PurePosixPath(entry["target_relative_path"]).parts) @@ -133,6 +130,9 @@ def test_content_source_snapshot_exclusions_cover_runtime_and_sensitive_state() "data/notes/**", "data/reports/**", "data/tmp/**", + "tests/**", + "**/*.bat", + "**/*.ps1", } <= excluded @@ -140,57 +140,51 @@ def test_content_source_snapshot_has_no_legacy_roots_or_plaintext_credentials() manifest_text = MANIFEST_PATH.read_text(encoding="utf-8").casefold() assert not any(root in manifest_text for root in OLD_PROJECT_ROOTS) - for path in RUNTIME_ROOT.rglob("*"): + for path in MODULE_ROOT.rglob("*"): if not path.is_file() or b"\x00" in path.read_bytes(): continue text = path.read_text(encoding="utf-8-sig", errors="strict").casefold() assert not any(root in text for root in OLD_PROJECT_ROOTS), path assert not any(root in text for root in FIXED_USER_PATHS), path - assert scan_repository(RUNTIME_ROOT) == [] + assert scan_repository(MODULE_ROOT) == [] def test_chanmama_credentials_are_externalized_in_the_copied_source() -> None: - source = (RUNTIME_ROOT / "chanmama_scraper.py").read_text(encoding="utf-8-sig") + source = (MODULE_ROOT / "chanmama_scraper.py").read_text(encoding="utf-8-sig") assert 'os.getenv("CHANMAMA_ACCOUNT", "")' in source assert 'os.getenv("CHANMAMA_PASSWORD", "")' in source assert not re.search(r'^PASSWORD\s*=\s*["\'][^"\']+["\']', source, re.MULTILINE) -def test_content_runtime_has_one_portable_path_boundary() -> None: - python_sources = list(RUNTIME_ROOT.rglob("*.py")) +def test_content_module_has_one_portable_path_boundary() -> None: + python_sources = list(MODULE_ROOT.rglob("*.py")) path_mutations: list[Path] = [] forbidden_cross_module_imports: list[Path] = [] for path in python_sources: source = path.read_text(encoding="utf-8-sig") if re.search(r"sys\.path\.(?:insert|append)\(", source): - path_mutations.append(path.relative_to(RUNTIME_ROOT)) + path_mutations.append(path.relative_to(MODULE_ROOT)) if "lark_cli_runtime" in source: - forbidden_cross_module_imports.append(path.relative_to(RUNTIME_ROOT)) + forbidden_cross_module_imports.append(path.relative_to(MODULE_ROOT)) assert path_mutations == [] assert forbidden_cross_module_imports == [] runtime_paths = ( - RUNTIME_ROOT / "runtime_paths.py" + MODULE_ROOT / "runtime_paths.py" ).read_text(encoding="utf-8-sig") assert "GYXX_DATA_ROOT" in runtime_paths assert "GYXX_MODULE_ROOT" in runtime_paths -def test_content_runtime_uses_portable_python_launchers_and_data_paths() -> None: - batch_sources = list(RUNTIME_ROOT.rglob("*.bat")) - assert batch_sources +def test_content_module_uses_python_commands_and_portable_data_paths() -> None: + assert list(MODULE_ROOT.rglob("*.bat")) == [] + assert list(MODULE_ROOT.rglob("*.ps1")) == [] - for path in batch_sources: - source = path.read_text(encoding="utf-8-sig").casefold() - assert "%gyxx_python%" in source, path - assert "hermes-agent" not in source, path - assert "appdata" not in source, path - - path_constants = (RUNTIME_ROOT / "runtime_paths.py").read_text( + path_constants = (MODULE_ROOT / "runtime_paths.py").read_text( encoding="utf-8-sig" ) assert 'DataLayout(data_root).for_module("content_marketing")' in path_constants diff --git a/tests/test_core_runtime.py b/tests/test_core_runtime.py index 6b596c1..5adf3e3 100644 --- a/tests/test_core_runtime.py +++ b/tests/test_core_runtime.py @@ -1,17 +1,46 @@ from __future__ import annotations import json +import multiprocessing +import os +import time from datetime import datetime, timezone from pathlib import Path import pytest +import gyxx_flow.core.locks as locks_module from gyxx_flow.core.context import RunContext from gyxx_flow.core.layout import DataLayout from gyxx_flow.core.locks import LockManager, ResourceBusyError from gyxx_flow.core.records import RunJournal +def _crash_while_holding_reclaim_guard(path: str) -> None: + with locks_module._exclusive_reclaim_guard( + Path(path), + resource="module:product", + deadline=time.monotonic() + 5, + poll_seconds=0.01, + ): + os._exit(0) + + +def _acquire_shared_resource(root: str, worker: int, queue) -> None: # type: ignore[no-untyped-def] + try: + with LockManager(Path(root)).acquire( + "module:product", + owner=f"worker-{worker}", + timeout_seconds=10, + poll_seconds=0.01, + ): + time.sleep(0.03) + except Exception as exc: + queue.put((worker, type(exc).__name__)) + else: + queue.put((worker, "ok")) + + def _context() -> RunContext: return RunContext.create( "shop.weekly", @@ -89,6 +118,233 @@ def test_named_resource_lock_blocks_concurrent_owner_and_releases(tmp_path: Path assert metadata["owner"] == "run-002" +def test_named_resource_lock_quarantines_provably_dead_owner( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + stale = manager.acquire("module:product", owner="run-dead") + stale.path.parent.mkdir(parents=True, exist_ok=True) + stale.path.write_text( + json.dumps( + { + "resource": "module:product", + "owner": "run-dead", + "acquired_at": "2026-08-01T10:00:00+00:00", + "pid": 424242, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: False) + + with manager.acquire( + "module:product", owner="run-new", timeout_seconds=0 + ) as acquired: + metadata = json.loads(acquired.path.read_text(encoding="utf-8")) + assert metadata["owner"] == "run-new" + + quarantined = list((tmp_path / "stale").glob("*.lock")) + assert len(quarantined) == 1 + assert json.loads(quarantined[0].read_text(encoding="utf-8"))["owner"] == "run-dead" + + +@pytest.mark.parametrize("liveness", [True, None]) +def test_named_resource_lock_keeps_active_or_unknown_owner( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + liveness: bool | None, +) -> None: + manager = LockManager(tmp_path) + occupied = manager.acquire("module:product", owner="run-existing") + occupied.path.parent.mkdir(parents=True, exist_ok=True) + occupied.path.write_text( + json.dumps( + { + "resource": "module:product", + "owner": "run-existing", + "acquired_at": "2026-08-01T10:00:00+00:00", + "pid": 424242, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: liveness) + + with pytest.raises(ResourceBusyError, match="module:product"): + with manager.acquire("module:product", owner="run-new", timeout_seconds=0): + pass + + assert occupied.path.exists() + + +def test_named_resource_lock_does_not_reclaim_recent_malformed_lock( + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + occupied = manager.acquire("module:product", owner="run-existing") + occupied.path.parent.mkdir(parents=True, exist_ok=True) + occupied.path.write_text('{"resource": "module:product"', encoding="utf-8") + + with pytest.raises(ResourceBusyError, match="module:product"): + with manager.acquire("module:product", owner="run-new", timeout_seconds=0): + pass + + assert occupied.path.exists() + + +def test_named_resource_lock_quarantines_old_malformed_lock_without_pid( + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + occupied = manager.acquire("module:product", owner="run-existing") + occupied.path.parent.mkdir(parents=True, exist_ok=True) + occupied.path.write_text('{"resource": "module:product"', encoding="utf-8") + old = time.time() - locks_module._MALFORMED_LOCK_GRACE_SECONDS - 5 + os.utime(occupied.path, (old, old)) + + with manager.acquire( + "module:product", owner="run-new", timeout_seconds=0 + ) as acquired: + metadata = json.loads(acquired.path.read_text(encoding="utf-8")) + assert metadata["owner"] == "run-new" + + quarantined = list((tmp_path / "stale").glob("*.lock")) + assert len(quarantined) == 1 + + +def test_named_resource_lock_keeps_old_malformed_lock_with_active_pid( + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + occupied = manager.acquire("module:product", owner="run-existing") + occupied.path.parent.mkdir(parents=True, exist_ok=True) + occupied.path.write_text(f'{{"pid": {os.getpid()},', encoding="utf-8") + old = time.time() - locks_module._MALFORMED_LOCK_GRACE_SECONDS - 5 + os.utime(occupied.path, (old, old)) + + with pytest.raises(ResourceBusyError, match="module:product"): + with manager.acquire("module:product", owner="run-new", timeout_seconds=0): + pass + + assert occupied.path.exists() + + +def test_named_resource_lock_uses_process_start_to_detect_pid_reuse( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + occupied = manager.acquire("module:product", owner="run-existing") + occupied.path.parent.mkdir(parents=True, exist_ok=True) + occupied.path.write_text( + json.dumps( + { + "resource": "module:product", + "owner": "run-existing", + "acquired_at": "2026-08-01T10:00:00+00:00", + "pid": 424242, + "process_started_at": 100.0, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: True) + monkeypatch.setattr(locks_module, "_process_started_at", lambda _pid: 100.0) + + with pytest.raises(ResourceBusyError, match="module:product"): + with manager.acquire("module:product", owner="run-new", timeout_seconds=0): + pass + + monkeypatch.setattr(locks_module, "_process_started_at", lambda _pid: 200.0) + with manager.acquire( + "module:product", owner="run-new", timeout_seconds=0 + ) as acquired: + assert acquired.path.exists() + + +def test_named_resource_lock_guard_is_released_when_reclaimer_crashes( + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + lock = manager.acquire("module:product", owner="run-new") + guard = lock.path.with_suffix(lock.path.suffix + ".reclaim") + context = multiprocessing.get_context("spawn") + process = context.Process( + target=_crash_while_holding_reclaim_guard, + args=(str(guard),), + ) + + process.start() + process.join(timeout=15) + + assert process.exitcode == 0 + assert guard.exists() + with manager.acquire( + "module:product", owner="run-new", timeout_seconds=1 + ) as acquired: + assert acquired.path.exists() + + +def test_named_resource_lock_serializes_concurrent_stale_recovery( + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + stale = manager.acquire("module:product", owner="run-dead") + stale.path.parent.mkdir(parents=True, exist_ok=True) + stale.path.write_text( + json.dumps( + { + "resource": "module:product", + "owner": "run-dead", + "acquired_at": "2026-08-01T10:00:00+00:00", + "pid": 2_147_483_647, + } + ), + encoding="utf-8", + ) + context = multiprocessing.get_context("spawn") + queue = context.Queue() + workers = [ + context.Process( + target=_acquire_shared_resource, + args=(str(tmp_path), worker, queue), + ) + for worker in range(4) + ] + + for process in workers: + process.start() + for process in workers: + process.join(timeout=20) + + assert [process.exitcode for process in workers] == [0, 0, 0, 0] + results = sorted(queue.get(timeout=2) for _ in workers) + assert results == [(worker, "ok") for worker in range(4)] + assert not stale.path.exists() + assert len(list((tmp_path / "stale").glob("*.lock"))) == 1 + + +def test_named_resource_lock_never_publishes_partial_metadata( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + manager = LockManager(tmp_path) + lock = manager.acquire("module:product", owner="run-new") + + def fail_link(_source: Path, _target: Path) -> None: + raise OSError("injected link failure") + + monkeypatch.setattr(locks_module.os, "link", fail_link) + + with pytest.raises(OSError, match="injected link failure"): + with lock: + pass + + assert not lock.path.exists() + assert list(tmp_path.glob(".*.tmp")) == [] + + @pytest.mark.parametrize("resource", ["", "../state", "resource\nname"]) def test_named_resource_lock_rejects_unsafe_name(tmp_path: Path, resource: str) -> None: manager = LockManager(tmp_path) diff --git a/tests/test_diagnostics_acceptance.py b/tests/test_diagnostics_acceptance.py index fa2c110..8cf4db4 100644 --- a/tests/test_diagnostics_acceptance.py +++ b/tests/test_diagnostics_acceptance.py @@ -5,7 +5,7 @@ import json from pathlib import Path from gyxx_flow.acceptance import build_acceptance_report, parse_plan_checklist -from gyxx_flow.cli import EXIT_ACCEPTANCE_INCOMPLETE, main +from gyxx_flow.cli import EXIT_SUCCESS, main from gyxx_flow.core.config import Settings from gyxx_flow.diagnostics import run_doctor @@ -52,11 +52,14 @@ def test_acceptance_report_is_machine_readable_and_tracks_real_evidence() -> Non payload = report.as_dict() assert payload["schema_version"] == 1 - assert payload["summary"]["total"] > payload["summary"]["completed"] - assert payload["checks"]["catalog_21_tasks"] is True - assert payload["checks"]["baseline_21_tasks"] is True + assert payload["summary"]["total"] == payload["summary"]["completed"] + assert payload["checks"]["catalog_22_tasks"] is True + assert payload["checks"]["scheduled_graphs_explicit"] is True + assert payload["checks"]["python_scheduler_only"] is True + assert payload["checks"]["runtime_service_policy"] is True + assert payload["checks"]["public_command_registry_complete"] is True assert payload["checks"]["secret_scan_clean"] is True - assert report.is_complete is False + assert report.is_complete is True def test_cli_doctor_and_acceptance_status_return_truthful_exit_codes(tmp_path: Path) -> None: @@ -72,6 +75,5 @@ def test_cli_doctor_and_acceptance_status_return_truthful_exit_codes(tmp_path: P settings=Settings(project_root=PROJECT_ROOT, data_root=PROJECT_ROOT / "var"), stdout=acceptance_output, ) - assert exit_code == EXIT_ACCEPTANCE_INCOMPLETE - assert json.loads(acceptance_output.getvalue())["is_complete"] is False - + assert exit_code == EXIT_SUCCESS + assert json.loads(acceptance_output.getvalue())["is_complete"] is True diff --git a/tests/test_effect_ledger.py b/tests/test_effect_ledger.py index bc204e2..216e854 100644 --- a/tests/test_effect_ledger.py +++ b/tests/test_effect_ledger.py @@ -1,9 +1,13 @@ from __future__ import annotations +import json +from io import StringIO from pathlib import Path import pytest +from gyxx_flow.cli import EXIT_CONFIGURATION, EXIT_SUCCESS, main +from gyxx_flow.core.config import Settings from gyxx_flow.ops.effects import ( EffectAlreadyApplied, EffectLedger, @@ -64,3 +68,165 @@ def test_effect_ledger_leaves_in_progress_claim_ambiguous_after_crash( run_id="run-002", ) + +def test_effect_ledger_reconcile_retry_preserves_audit_and_allows_one_new_claim( + tmp_path: Path, +) -> None: + ledger = EffectLedger(tmp_path) + failed = ledger.begin( + workflow_id="product.daily", + business_date="2026-08-01", + step_id="collect", + run_id="run-original", + ) + ledger.mark_ambiguous(failed, reason="preflight-failed") + + result = ledger.reconcile( + workflow_id="product.daily", + business_date="2026-08-01", + step_id="collect", + expected_run_id="run-original", + action="retry", + operator="operator@example", + reason="verified command did not reach the external sink", + evidence="journal:run-original", + ) + retry = ledger.begin( + workflow_id="product.daily", + business_date="2026-08-01", + step_id="collect", + run_id="run-retry", + ) + + assert result["status"] == "retry_allowed" + assert retry.run_id == "run-retry" + receipt_path = next((tmp_path / "state" / "ops" / "effects").glob("*.json")) + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "in_progress" + assert receipt["run_id"] == "run-retry" + assert receipt["reconciliations"] == [ + { + "action": "retry", + "operator": "operator@example", + "reason": "verified command did not reach the external sink", + "evidence": "journal:run-original", + "expected_run_id": "run-original", + "reconciled_at": result["reconciled_at"], + } + ] + + +def test_effect_ledger_reconcile_can_mark_verified_effect_applied( + tmp_path: Path, +) -> None: + ledger = EffectLedger(tmp_path) + claim = ledger.begin( + workflow_id="product.alert.daily", + business_date="2026-08-01", + step_id="notify", + run_id="run-original", + ) + ledger.mark_ambiguous(claim, reason="response-lost") + + result = ledger.reconcile( + workflow_id="product.alert.daily", + business_date="2026-08-01", + step_id="notify", + expected_run_id="run-original", + action="applied", + operator="operator@example", + reason="verified the message receipt in Feishu", + evidence="message:verified-receipt", + ) + + assert result["status"] == "applied" + with pytest.raises(EffectAlreadyApplied): + ledger.begin( + workflow_id="product.alert.daily", + business_date="2026-08-01", + step_id="notify", + run_id="run-second", + ) + + +def test_effect_ledger_reconcile_rejects_active_or_changed_claims( + tmp_path: Path, +) -> None: + ledger = EffectLedger(tmp_path) + claim = ledger.begin( + workflow_id="product.daily", + business_date="2026-08-02", + step_id="collect", + run_id="run-active", + ) + + with pytest.raises(EffectStateAmbiguous, match="only ambiguous"): + ledger.reconcile( + workflow_id="product.daily", + business_date="2026-08-02", + step_id="collect", + expected_run_id="run-active", + action="retry", + operator="operator@example", + reason="should not unlock a live claim", + evidence="journal:run-active", + ) + + ledger.mark_ambiguous(claim, reason="failed") + with pytest.raises(EffectStateAmbiguous, match="run_id changed"): + ledger.reconcile( + workflow_id="product.daily", + business_date="2026-08-02", + step_id="collect", + expected_run_id="run-other", + action="retry", + operator="operator@example", + reason="stale operator view", + evidence="journal:run-other", + ) + + +def test_effect_reconcile_cli_requires_confirmation_and_returns_safe_projection( + tmp_path: Path, +) -> None: + data_root = tmp_path / "data" + settings = Settings(project_root=tmp_path, data_root=data_root) + ledger = EffectLedger(data_root) + claim = ledger.begin( + workflow_id="product.daily", + business_date="2026-08-03", + step_id="collect", + run_id="run-original", + ) + ledger.mark_ambiguous(claim, reason="preflight-failed") + base_args = [ + "effects", + "reconcile", + "product.daily", + "--date", + "2026-08-03", + "--step", + "collect", + "--expected-run-id", + "run-original", + "--action", + "retry", + "--operator", + "operator@example", + "--reason", + "verified no external write", + "--evidence", + "journal:run-original", + ] + + errors = StringIO() + assert main(base_args, settings=settings, stderr=errors) == EXIT_CONFIGURATION + assert "requires --execute" in errors.getvalue() + + output = StringIO() + assert main([*base_args, "--execute"], settings=settings, stdout=output) == EXIT_SUCCESS + result = json.loads(output.getvalue()) + assert result["status"] == "retry_allowed" + assert "operator" not in result + assert "reason" not in result + assert "evidence" not in result diff --git a/tests/test_engineering_baseline.py b/tests/test_engineering_baseline.py new file mode 100644 index 0000000..de722e1 --- /dev/null +++ b/tests/test_engineering_baseline.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +BUSINESS_MODULES = ( + "content_marketing", + "product_commerce", + "shop_intelligence", + "supply_chain", +) + + +def test_repository_keeps_the_simplified_public_structure() -> None: + modules_root = PROJECT_ROOT / "src" / "gyxx_flow" / "modules" + for module in BUSINESS_MODULES: + module_root = modules_root / module + compatibility_items = { + path.name + for path in (module_root / "runtime").iterdir() + if path.name != "__pycache__" + } + assert compatibility_items == {"__init__.py"} + assert not (module_root / "tests").exists() + assert not (module_root / "runtime_tests").exists() + + primary_docs = { + path.name for path in (PROJECT_ROOT / "docs").iterdir() if path.is_file() + } + assert primary_docs == { + "acceptance-report.md", + "architecture.md", + "deployment.md", + "plan.md", + "runbook.md", + "workflow-acceptance-test-report.md", + "workflow-acceptance-test-requirements.md", + "workflow-console.md", + } + assert not { + "design.md", + "final_report.md", + "plan.md", + "review.md", + } & {path.name for path in PROJECT_ROOT.iterdir() if path.is_file()} + + +def test_development_group_contains_reproducible_quality_tools() -> None: + configuration = tomllib.loads( + (PROJECT_ROOT / "pyproject.toml").read_text(encoding="utf-8") + ) + + development = configuration["dependency-groups"]["dev"] + package_names = {requirement.split(">=", 1)[0] for requirement in development} + + assert {"build", "pytest", "ruff"} <= package_names + assert ( + "src/gyxx_flow/modules/*/**" + in configuration["tool"]["ruff"]["extend-exclude"] + ) + assert configuration["tool"]["pytest"]["ini_options"]["testpaths"] == ["tests"] + + +def test_gitlab_ci_runs_lock_lint_tests_and_build() -> None: + pipeline = (PROJECT_ROOT / ".gitlab-ci.yml").read_text(encoding="utf-8") + + assert "uv sync --frozen --group dev" in pipeline + assert "uv run ruff check src tests" in pipeline + assert "uv run ruff check src/gyxx_flow/modules/content_marketing/daily_creator_exposure_scope.py" in pipeline + assert "src/gyxx_flow/modules/product_commerce/market_rank_product_import.py" in pipeline + assert "uv run pytest" in pipeline + assert "test_daily_creator_exposure_scope.py" in pipeline + assert "test_chanmama_refresh_before_export.py" in pipeline + assert "test_daily_exposure_fields.py" in pipeline + assert "test_market_rank_limits.py" in pipeline + assert "test_market_rank_product_import.py" in pipeline + assert "uv build" in pipeline diff --git a/tests/test_langgraph_workflow_engine.py b/tests/test_langgraph_workflow_engine.py new file mode 100644 index 0000000..433e0e3 --- /dev/null +++ b/tests/test_langgraph_workflow_engine.py @@ -0,0 +1,418 @@ +from __future__ import annotations + +import json +import threading +from collections import deque +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from langgraph.graph.state import CompiledStateGraph + +from gyxx_flow.catalog import WorkflowCatalog, WorkflowEntry +from gyxx_flow.cli import build_default_registry +from gyxx_flow.core.config import Settings +from gyxx_flow.core.context import RunContext +from gyxx_flow.core.layout import DataLayout +from gyxx_flow.core.locks import LockManager +from gyxx_flow.core.records import RunJournal +from gyxx_flow.modules.shop_intelligence import ShopIntelligenceModule +from gyxx_flow.ops.effects import EffectLedger +from gyxx_flow.workflow.engine import WorkflowEngine +from gyxx_flow.workflow.graph import ( + GraphStepResult, + WorkflowGraphState, + compile_workflow_graph, + initial_workflow_graph_state, +) +from gyxx_flow.workflow.model import StepDefinition, WorkflowDefinition +from gyxx_flow.workflow.steps import StepExecution + + +class FakeStep: + def __init__(self, *outcomes: StepExecution) -> None: + self.outcomes = deque(outcomes or (StepExecution(exit_code=0),)) + self.calls: list[dict[str, Any]] = [] + + def execute( + self, + *, + context: RunContext, + timeout_seconds: float | None, + dry_run: bool, + ) -> StepExecution: + self.calls.append( + { + "context": context, + "timeout_seconds": timeout_seconds, + "dry_run": dry_run, + } + ) + return self.outcomes.popleft() + + +class CoordinatedStep: + def __init__( + self, + barrier: threading.Barrier, + *, + error: str | None = None, + ) -> None: + self.barrier = barrier + self.error = error + self.calls = 0 + + def execute( + self, + *, + context: RunContext, + timeout_seconds: float | None, + dry_run: bool, + ) -> StepExecution: + del context, timeout_seconds, dry_run + self.calls += 1 + self.barrier.wait(timeout=3) + if self.error is not None: + raise RuntimeError(self.error) + return StepExecution(exit_code=0) + + +def _context(*, suffix: str = "abc123", shadow: bool = False) -> RunContext: + return RunContext.create( + "shop.weekly", + "2026-07-27", + shadow=shadow, + now=datetime(2026, 7, 27, 4, 0, tzinfo=timezone.utc), + random_suffix=suffix, + ) + + +def _success_state( + step: StepDefinition, + state: WorkflowGraphState, +) -> GraphStepResult: + del state + return { + "step_id": step.step_id, + "status": "success", + "attempts": [ + { + "attempt": 1, + "status": "success", + "exit_code": 0, + "error": None, + "reason": None, + } + ], + "final_exit_code": 0, + "critical": step.critical, + "reason": None, + } + + +def test_compiler_creates_one_langgraph_node_per_step_in_stable_order() -> None: + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition("publish", FakeStep(), depends_on=("collect",)), + StepDefinition("audit", FakeStep()), + StepDefinition("collect", FakeStep()), + ), + ) + + compiled = compile_workflow_graph(workflow, run_step=_success_state) + graph = compiled.get_graph() + + assert isinstance(compiled, CompiledStateGraph) + assert set(graph.nodes) == {"__start__", "audit", "collect", "publish", "__end__"} + assert {(edge.source, edge.target) for edge in graph.edges} == { + ("__start__", "audit"), + ("__start__", "collect"), + ("audit", "__end__"), + ("collect", "publish"), + ("publish", "__end__"), + } + + state = initial_workflow_graph_state(_context(), dry_run=False) + final_state = compiled.invoke(state) + + assert set(final_state["step_results"]) == {"audit", "collect", "publish"} + json.dumps(final_state) + + +def test_compiler_joins_uneven_dependency_branches_before_running_once() -> None: + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition("source", FakeStep()), + StepDefinition("parallel", FakeStep()), + StepDefinition("derived", FakeStep(), depends_on=("source",)), + StepDefinition( + "publish", + FakeStep(), + depends_on=("parallel", "derived"), + ), + ), + ) + publish_inputs: list[set[str]] = [] + calls: list[str] = [] + + def run_step( + step: StepDefinition, + state: WorkflowGraphState, + ) -> GraphStepResult: + calls.append(step.step_id) + if step.step_id == "publish": + publish_inputs.append(set(state["step_results"])) + return _success_state(step, state) + + compiled = compile_workflow_graph(workflow, run_step=run_step) + compiled.invoke(initial_workflow_graph_state(_context(), dry_run=False)) + + assert calls.count("publish") == 1 + assert publish_inputs == [{"source", "parallel", "derived"}] + + +def test_engine_preserves_retry_dependency_and_journal_contract(tmp_path: Path) -> None: + flaky = FakeStep( + StepExecution(exit_code=7, error="temporary"), + StepExecution(exit_code=0), + ) + downstream = FakeStep() + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition("flaky", flaky, max_attempts=2), + StepDefinition("required", FakeStep(StepExecution(exit_code=9, error="bad"))), + StepDefinition("downstream", downstream, depends_on=("required",)), + ), + ) + context = _context() + journal = RunJournal.create(DataLayout(tmp_path), context) + + result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + workflow, + context=context, + journal=journal, + ) + + assert result.status == "failed" + assert [attempt.exit_code for attempt in result.steps["flaky"].attempts] == [7, 0] + assert result.steps["downstream"].reason == "dependency-failed" + assert downstream.calls == [] + payload = json.loads(journal.path.read_text(encoding="utf-8")) + assert payload["steps"]["flaky.attempt-1"]["status"] == "failed" + assert payload["steps"]["flaky.attempt-2"]["status"] == "success" + assert payload["steps"]["downstream.attempt-1"]["status"] == "skipped" + + +def test_independent_graph_nodes_run_concurrently_and_isolate_action_failure( + tmp_path: Path, +) -> None: + barrier = threading.Barrier(2) + jd = CoordinatedStep(barrier, error="JD browser session failed") + tmall = CoordinatedStep(barrier) + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition("jd", jd, resources=("browser:jd",)), + StepDefinition("tmall", tmall, resources=("browser:tmall",)), + ), + ) + context = _context(suffix="parallel1") + journal = RunJournal.create(DataLayout(tmp_path), context) + + result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + workflow, + context=context, + journal=journal, + ) + + assert jd.calls == 1 + assert tmall.calls == 1 + assert result.status == "failed" + assert result.steps["jd"].status == "failed" + assert result.steps["tmall"].status == "success" + payload = json.loads(journal.path.read_text(encoding="utf-8")) + assert payload["error"] == ( + "critical steps failed: jd: RuntimeError: JD browser session failed" + ) + assert payload["steps"]["jd.attempt-1"]["error"] == ( + "RuntimeError: JD browser session failed" + ) + assert payload["steps"]["tmall.attempt-1"]["status"] == "success" + + +def test_workflow_error_reports_each_failed_step_with_bounded_details( + tmp_path: Path, +) -> None: + jd_error = "JD PG write refused: " + "x" * 5_000 + " JD_ROOT_CAUSE_AT_TAIL" + tmall_error = ( + "TM Feishu insert refused: " + + "y" * 5_000 + + " TM_ROOT_CAUSE_AT_TAIL" + ) + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition("jd", FakeStep(StepExecution(exit_code=7, error=jd_error))), + StepDefinition( + "tmall", + FakeStep(StepExecution(exit_code=9, error=tmall_error)), + ), + ), + ) + context = _context(suffix="bounded1") + journal = RunJournal.create(DataLayout(tmp_path), context) + + WorkflowEngine(LockManager(tmp_path / "locks")).execute( + workflow, + context=context, + journal=journal, + ) + + error = json.loads(journal.path.read_text(encoding="utf-8"))["error"] + assert "jd: JD PG write refused" in error + assert "tmall: TM Feishu insert refused" in error + assert "JD_ROOT_CAUSE_AT_TAIL" in error + assert "TM_ROOT_CAUSE_AT_TAIL" in error + assert len(error) <= 4_096 + + +def test_engine_preserves_dry_run_and_shadow_policy(tmp_path: Path) -> None: + dry_action = FakeStep() + dry_workflow = WorkflowDefinition( + "shop.weekly", (StepDefinition("collect", dry_action),) + ) + dry_context = _context() + dry_result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + dry_workflow, + context=dry_context, + journal=RunJournal.create(DataLayout(tmp_path), dry_context), + dry_run=True, + ) + + sink = FakeStep() + shadow_workflow = WorkflowDefinition( + "shop.weekly", + (StepDefinition("publish", sink, production_sink=True),), + ) + shadow_context = _context(suffix="shadow1", shadow=True) + shadow_result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + shadow_workflow, + context=shadow_context, + journal=RunJournal.create(DataLayout(tmp_path), shadow_context), + ) + + assert dry_result.steps["collect"].reason == "dry-run" + assert shadow_result.steps["publish"].reason == "shadow-policy" + assert dry_action.calls == [] + assert sink.calls == [] + + +def test_engine_preserves_effect_ledger_replay_policy(tmp_path: Path) -> None: + action = FakeStep() + workflow = WorkflowDefinition( + "shop.weekly", + (StepDefinition("publish", action, production_sink=True),), + ) + engine = WorkflowEngine( + LockManager(tmp_path / "locks"), + effect_ledger=EffectLedger(tmp_path), + ) + first_context = _context(suffix="first1") + second_context = _context(suffix="second2") + + first = engine.execute( + workflow, + context=first_context, + journal=RunJournal.create(DataLayout(tmp_path), first_context), + ) + second = engine.execute( + workflow, + context=second_context, + journal=RunJournal.create(DataLayout(tmp_path), second_context), + ) + + assert first.steps["publish"].status == "success" + assert second.steps["publish"].reason == "idempotency-replay" + assert len(action.calls) == 1 + + +def test_every_registered_workflow_compiles_to_langgraph(tmp_path: Path) -> None: + project_root = Path(__file__).parents[1] + registry = build_default_registry( + Settings(project_root=project_root, data_root=tmp_path) + ) + + compiled_count = 0 + for entry in registry.catalog.workflows: + if not registry.is_registered(entry.workflow_id): + continue + definition = registry.resolve(entry.workflow_id).definition + compiled = compile_workflow_graph(definition, run_step=_success_state) + assert isinstance(compiled, CompiledStateGraph) + compiled_count += 1 + + assert compiled_count > 0 + + +def test_jd_self_operated_uses_source_timeouts_and_continues_after_brand_failure( + tmp_path: Path, +) -> None: + calls: list[tuple[str, float | None]] = [] + + class RecordedCommand: + def __init__(self, entry: WorkflowEntry) -> None: + self.entry = entry + + def execute( + self, + *, + context: RunContext, + timeout_seconds: float | None, + dry_run: bool, + ) -> StepExecution: + del context + assert dry_run is False + calls.append((self.entry.entry, timeout_seconds)) + if self.entry.entry.endswith("jd_self_operated_brand_daily.py"): + return StepExecution(exit_code=3, error="brand failed") + return StepExecution(exit_code=0) + + def command_factory( + entry: WorkflowEntry, + _context: RunContext, + ) -> RecordedCommand: + return RecordedCommand(entry) + + catalog = WorkflowCatalog.load(Path(__file__).parents[1] / "config") + workflow = next( + item + for item in ShopIntelligenceModule.from_catalog( + catalog, + command_factory=command_factory, + ).workflow_definitions() + if item.workflow_id == "shop.jd_self_operated.daily" + ) + context = RunContext.create( + workflow.workflow_id, + "2026-08-02", + random_suffix="jdself1", + ) + + result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + workflow, + context=context, + journal=RunJournal.create(DataLayout(tmp_path), context), + ) + + assert calls == [ + ("collectors/jd_self_operated_brand_daily.py", 600), + ("collectors/jd_self_operated_product_daily.py", 1_800), + ] + assert list(result.steps) == ["brand", "product"] + assert result.steps["brand"].status == "failed" + assert result.steps["product"].status == "success" + assert result.status == "failed" + assert result.exit_code == 1 diff --git a/tests/test_latest_source_sync_20260729.py b/tests/test_latest_source_sync_20260729.py new file mode 100644 index 0000000..7324339 --- /dev/null +++ b/tests/test_latest_source_sync_20260729.py @@ -0,0 +1,88 @@ +from argparse import Namespace +from datetime import date + + +def test_daily_creator_scope_filters_styles_and_publish_window(): + from gyxx_flow.modules.content_marketing.daily_creator_exposure_scope import ( + DEFAULT_MAX_AGE_DAYS, + DEFAULT_PUBLISHED_FROM, + EXCLUDED_STYLE_NAMES, + classify_publish_scope, + select_daily_styles, + ) + + styles = [ + {"index": 1, "name": "保留款"}, + {"index": 2, "name": "盖亚微单"}, + {"index": 3, "name": "逐星GT"}, + {"index": 4, "name": "晨星2"}, + {"index": 5, "name": "觅光"}, + ] + assert [row["index"] for row in select_daily_styles(styles)] == [1] + assert EXCLUDED_STYLE_NAMES == frozenset({"盖亚微单", "逐星GT", "晨星2", "觅光"}) + assert DEFAULT_PUBLISHED_FROM == date(2026, 7, 1) + assert DEFAULT_MAX_AGE_DAYS == 30 + assert classify_publish_scope( + date(2026, 7, 1), + date(2026, 7, 31), + published_from=DEFAULT_PUBLISHED_FROM, + max_age_days=DEFAULT_MAX_AGE_DAYS, + ) == "collection_window_complete" + assert classify_publish_scope( + date(2026, 7, 2), + date(2026, 7, 31), + published_from=DEFAULT_PUBLISHED_FROM, + max_age_days=DEFAULT_MAX_AGE_DAYS, + ) is None + + +def test_run_all_propagates_daily_scope_and_no_retry_to_local_children(): + from gyxx_flow.modules.content_marketing import run_all + + args = Namespace(dry_run=False, style=[1, 2], daily_scope=True, no_retry=False) + command = run_all.build_process_command("pgy", args) + + assert command[0] + assert str(run_all.BASE_DIR) in command[1] + assert "--skip-field-prepare" in command + assert command[command.index("--published-from") + 1] == "2026-07-01" + assert command[command.index("--max-age-days") + 1] == "30" + assert "--no-retry" in command + assert run_all.styles_after_field_prepare( + {2, 3, 31}, {31}, preserve_failed=True + ) == [2, 3, 31] + + +def test_market_rank_limits_are_per_platform_and_per_category(): + from gyxx_flow.modules.product_commerce.market_rank_limits import ( + category_rank_limit, + remaining_category_rank_limit, + ) + + assert category_rank_limit("tm", "旅行箱") == 200 + assert category_rank_limit("dy", "电脑包") == 200 + assert category_rank_limit("jd", "旅行箱") == 150 + assert category_rank_limit("jd", "电脑包") == 100 + assert remaining_category_rank_limit( + "dy", "双肩包", category_count=20, total_count=490, global_limit=500 + ) == 10 + + +def test_market_rank_collectors_use_the_shared_category_limits(): + from pathlib import Path + + module_root = ( + Path(__file__).parents[1] + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" + ) + for name in ( + "collect_sycm_market_rank.py", + "collect_dy_market_rank.py", + "collect_jd_market_rank.py", + ): + source = (module_root / name).read_text(encoding="utf-8") + assert "remaining_category_rank_limit(" in source + assert "category_rank_limits" in source diff --git a/tests/test_latest_source_sync_regressions.py b/tests/test_latest_source_sync_regressions.py new file mode 100644 index 0000000..e2e8218 --- /dev/null +++ b/tests/test_latest_source_sync_regressions.py @@ -0,0 +1,43 @@ +import json +from pathlib import Path + +from gyxx_flow.modules.shop_intelligence.config import TM_PLATFORM +from gyxx_flow.modules.shop_intelligence.writers import shop_base_writer + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def test_scheduled_content_workflows_do_not_auto_retry_failed_collection() -> None: + payload = json.loads( + (PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8") + ) + workflows = {item["id"]: item for item in payload["workflows"]} + commands = json.loads( + (PROJECT_ROOT / "config" / "commands.json").read_text(encoding="utf-8") + )["commands"] + scheduled_entries = { + step["entry"] + for workflow in workflows.values() + if workflow["module"] == "content_marketing" + and workflow["trigger"] == "scheduled" + for step in workflow["execution"]["steps"] + } + + assert "data/tools/retry_failed.py" not in scheduled_entries + assert "content.retry_failed" not in workflows + assert "content.failed.retry" in {item["id"] for item in commands} + assert "data/tools/sync_metrics_to_cmt_notes.py" in scheduled_entries + + +def test_tmall_shop_metrics_never_write_to_feishu() -> None: + assert shop_base_writer.write_shop(TM_PLATFORM, "missing.json") is True + runner = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "shop_intelligence" + / "runners" + / "run_shop.py" + ).read_text(encoding="utf-8") + assert "write_shop(TM_PLATFORM" not in runner diff --git a/tests/test_module_command_adapter.py b/tests/test_module_command_adapter.py index 56475a6..bb35e98 100644 --- a/tests/test_module_command_adapter.py +++ b/tests/test_module_command_adapter.py @@ -6,6 +6,10 @@ from pathlib import Path import pytest +from gyxx_flow.adapters.acceptance_policy import ( + WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID, + WorkflowAcceptancePolicyError, +) from gyxx_flow.adapters.native import ( DeferredModuleCommandStep, ModuleCommandAdapter, @@ -14,6 +18,7 @@ from gyxx_flow.adapters.native import ( ) from gyxx_flow.catalog import WorkflowEntry from gyxx_flow.core.context import RunContext +from gyxx_flow.workflow.steps import StepExecution def _entry(**overrides: object) -> WorkflowEntry: @@ -86,6 +91,31 @@ def test_deferred_module_step_dry_run_never_resolves_or_executes() -> None: assert execution.reason == "dry-run" +def test_deferred_module_step_renders_business_date_in_catalog_args() -> None: + captured: list[WorkflowEntry] = [] + + class SuccessStep: + def execute(self, **_kwargs: object) -> StepExecution: + return StepExecution(exit_code=0) + + def command_factory(entry: WorkflowEntry, _context: RunContext) -> SuccessStep: + captured.append(entry) + return SuccessStep() + + execution = DeferredModuleCommandStep( + _entry(), + command_factory=command_factory, + command_args=("--start-date", "{business_date}", "run={run_id}"), + ).execute(context=_context(), timeout_seconds=60, dry_run=False) + + assert execution.exit_code == 0 + assert captured[0].args == ( + "--start-date", + "2026-07-27", + f"run={_context().run_id}", + ) + + def test_module_adapter_uses_explicit_windows_launchers_for_batch_and_powershell( tmp_path: Path, ) -> None: @@ -131,3 +161,115 @@ def test_product_config_uses_only_the_new_environment_name(tmp_path: Path) -> No assert command.env["GYXX_PRODUCT_CONFIG"] == "D:/portable/product.json" assert "AUTO_FLOW_CONFIG" not in command.env + + +def test_product_config_defaults_to_canonical_module_state(tmp_path: Path) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + data_root = tmp_path / "var" + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={}, + project_root=tmp_path, + data_root=data_root, + ) + + command = adapter.build(_entry(entry="job.py", args=()), context=_context()) + + assert command.env["GYXX_PRODUCT_CONFIG"] == str( + (data_root / "state" / "product_commerce" / "auto-flow-config.json").resolve() + ) + + +def test_product_config_maps_legacy_portable_alias(tmp_path: Path) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={"AUTOFLOW_CONFIG_PATH": "D:/portable/legacy-product.json"}, + project_root=tmp_path, + data_root=tmp_path / "var", + ) + + command = adapter.build(_entry(entry="job.py", args=()), context=_context()) + + assert command.env["GYXX_PRODUCT_CONFIG"] == "D:/portable/legacy-product.json" + + +def test_product_config_maps_original_source_environment_name(tmp_path: Path) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={"AUTO_FLOW_CONFIG": "D:/original/product.json"}, + project_root=tmp_path, + data_root=tmp_path / "var", + ) + + command = adapter.build(_entry(entry="job.py", args=()), context=_context()) + + assert command.env["GYXX_PRODUCT_CONFIG"] == "D:/original/product.json" + + +def test_product_config_uses_explicit_precedence_order(tmp_path: Path) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={ + "GYXX_PRODUCT_CONFIG": "D:/canonical/product.json", + "AUTO_FLOW_CONFIG": "D:/original/product.json", + "AUTOFLOW_CONFIG_PATH": "D:/portable/product.json", + }, + project_root=tmp_path, + data_root=tmp_path / "var", + ) + + command = adapter.build(_entry(entry="job.py", args=()), context=_context()) + + assert command.env["GYXX_PRODUCT_CONFIG"] == "D:/canonical/product.json" + + +def test_module_adapter_propagates_fail_closed_acceptance_policy( + tmp_path: Path, +) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={"GYXX_WORKFLOW_ACCEPTANCE": "1"}, + project_root=tmp_path, + data_root=tmp_path / "var", + ) + + command = adapter.build(_entry(entry="job.py", args=()), context=_context()) + + assert command.env["GYXX_FEISHU_TABLE_WRITE_DISABLED"] == "1" + assert command.env["GYXX_COOKIE_INVALID_SKIP"] == "1" + assert ( + command.env["GYXX_NOTIFICATION_RECIPIENT_OPEN_ID"] + == WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + ) + + +def test_module_adapter_rejects_unsafe_acceptance_recipient(tmp_path: Path) -> None: + root = tmp_path / "runtime" + root.mkdir() + (root / "job.py").write_text("pass", encoding="utf-8") + adapter = ModuleCommandAdapter( + ModuleSourceRoots({"product_commerce": root}), + base_env={ + "GYXX_WORKFLOW_ACCEPTANCE": "1", + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_not_wang", + }, + project_root=tmp_path, + data_root=tmp_path / "var", + ) + + with pytest.raises(WorkflowAcceptancePolicyError, match="Wang Yunlong"): + adapter.build(_entry(entry="job.py", args=()), context=_context()) diff --git a/tests/test_native_source_runtime.py b/tests/test_native_source_runtime.py index a33696d..2ed0b50 100644 --- a/tests/test_native_source_runtime.py +++ b/tests/test_native_source_runtime.py @@ -20,10 +20,14 @@ def test_workflow_catalog_uses_native_execution_schema() -> None: (PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8") ) - assert payload["schema_version"] == 2 + assert payload["schema_version"] == 3 for workflow in payload["workflows"]: assert "legacy" not in workflow - assert set(workflow["execution"]) >= {"entry"} + execution = workflow["execution"] + assert {"entry", "steps"} & set(execution) + if workflow["trigger"] == "scheduled": + assert execution.get("steps") + assert all(step.get("entry") for step in execution["steps"]) def test_every_available_workflow_entry_exists_inside_its_new_module() -> None: @@ -33,7 +37,8 @@ def test_every_available_workflow_entry_exists_inside_its_new_module() -> None: for workflow in catalog.workflows: if workflow.trigger == "unavailable": continue - entry = MODULE_SOURCE_ROOT / workflow.module / "runtime" / workflow.entry + module_root = MODULE_SOURCE_ROOT / workflow.module + entry = module_root / workflow.entry if not entry.is_file(): missing.append(f"{workflow.workflow_id}: {entry}") diff --git a/tests/test_nested_workflow_browser_bindings.py b/tests/test_nested_workflow_browser_bindings.py new file mode 100644 index 0000000..ac05dc8 --- /dev/null +++ b/tests/test_nested_workflow_browser_bindings.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from pathlib import Path + +from gyxx_flow.adapters import environment_for_child_script + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +MODULE_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules" + +NESTED_TARGETS = { + "content_marketing": ( + "bilibili_scraper.py", + "pgy_xhs_scraper_v2.py", + "xingtu_scraper_v2.py", + "self_douyin_scraper.py", + ), + "shop_intelligence": ( + "collectors/jd_data_collector.py", + "collectors/jd_peer_store_data_collector.py", + "collectors/dy_store_competitor_store_scraping.py", + "collectors/taobao_sycm.py", + "collectors/jd_self_operated_brand_daily.py", + "collectors/jd_self_operated_product_daily.py", + ), + "product_commerce": ( + "orchestrate_daily_collection.py", + "collect_persona_to_bitable.py", + "collect_dy_persona_to_bitable.py", + "collect_jd_persona_to_bitable.py", + "taobao_dmp_item_crowd_insight_screenshots.py", + "dy_audience_profile_collect.py", + "collect_erp_yesterday_metrics.py", + "jd_product_data_collector.py", + "dy_product_scraping.py", + "taobao_sycm_collect.py", + "taobao_sycm_collect_backfill.py", + "import_product_reviews.py", + "aggregate_daily_final.py", + "export_bitable_records.py", + "insert_bitable_records.py", + "analyze_style_with_hermes.py", + "check_nine_day_decline.py", + "collect_sycm_market_rank.py", + "collect_jd_market_rank.py", + "collect_dy_market_rank.py", + "taobao_wanxiang_ai_creative_report.py", + "scripts/insert_main_image_records.py", + "jd_main_image_collector.py", + "scripts/insert_jd_main_image_records.py", + ), +} + + +def test_nested_workflow_targets_have_complete_unique_browser_bindings(tmp_path) -> None: + bindings: list[dict[str, str]] = [] + for module, entries in NESTED_TARGETS.items(): + module_root = MODULE_ROOT / module + base_environment = { + "GYXX_MODULE_ID": module, + "GYXX_MODULE_ROOT": str(module_root), + "GYXX_PROJECT_ROOT": str(PROJECT_ROOT), + "GYXX_DATA_ROOT": str(tmp_path / "portable-data"), + } + for entry in entries: + environment = environment_for_child_script( + module_root / entry, + base_environment, + ) + assert environment["GYXX_SCRIPT_ID"] == f"{module}:{entry}" + for key in ( + "GYXX_BROWSER_CDP_PORT", + "GYXX_BROWSER_CDP_URL", + "GYXX_BROWSER_PROFILE_DIR", + "GYXX_BROWSER_COOKIE_FILE", + "GYXX_BROWSER_STORAGE_STATE_FILE", + ): + assert environment[key] + bindings.append(environment) + + assert len({item["GYXX_BROWSER_CDP_PORT"] for item in bindings}) == len(bindings) + assert len({item["GYXX_BROWSER_CDP_URL"] for item in bindings}) == len(bindings) + assert len({item["GYXX_BROWSER_PROFILE_DIR"] for item in bindings}) == len(bindings) + assert len({item["GYXX_BROWSER_COOKIE_FILE"] for item in bindings}) == len(bindings) + assert len( + {item["GYXX_BROWSER_STORAGE_STATE_FILE"] for item in bindings} + ) == len(bindings) diff --git a/tests/test_ops.py b/tests/test_ops.py index f193e27..e0ff572 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -8,7 +8,7 @@ import pytest from gyxx_flow.core.context import RunContext from gyxx_flow.core.layout import DataLayout -from gyxx_flow.core.records import RunJournal +from gyxx_flow.core.records import RunJournal, RunMode from gyxx_flow.ops import IdempotencyConflict, Outbox, RunIndex @@ -18,6 +18,7 @@ def _journal( workflow_id: str = "shop.weekly", business_date: str = "2026-07-27", suffix: str = "abc123", + mode: RunMode = "unknown", ) -> RunJournal: context = RunContext.create( workflow_id, @@ -25,7 +26,7 @@ def _journal( now=datetime(2026, 7, 27, 4, 0, tzinfo=timezone.utc), random_suffix=suffix, ) - return RunJournal.create(DataLayout(root), context) + return RunJournal.create(DataLayout(root), context, mode=mode) def test_run_index_snapshots_journals_and_supports_structured_queries(tmp_path: Path) -> None: @@ -43,6 +44,7 @@ def test_run_index_snapshots_journals_and_supports_structured_queries(tmp_path: first_record = index.index_journal(first) second_record = index.index_journal(second.path) + assert first_record.mode == "unknown" assert first_record.status == "success" assert second_record.error == "collector failed" assert index.get(first_record.run_id) == first_record @@ -80,6 +82,59 @@ def test_run_index_rejects_malformed_journal_without_writing_index(tmp_path: Pat assert index.query() == () +def test_run_index_preserves_mode_and_accepts_legacy_records(tmp_path: Path) -> None: + journal = _journal(tmp_path, mode="dry_run") + journal.start_step("collect.attempt-1", attempt=1) + journal.finish_step("collect.attempt-1", status="skipped", error="dry-run") + journal.finalize("success") + index = RunIndex(tmp_path) + + current = index.index_journal(journal) + + assert current.mode == "dry_run" + assert current.schema_version == 2 + current_payload = json.loads( + (tmp_path / "state" / "ops" / "run-index" / f"{current.run_id}.json").read_text( + encoding="utf-8" + ) + ) + assert current_payload["mode"] == "dry_run" + assert current_payload["schema_version"] == 2 + + current_payload.pop("mode") + current_payload["schema_version"] = 1 + (tmp_path / "state" / "ops" / "run-index" / f"{current.run_id}.json").write_text( + json.dumps(current_payload), + encoding="utf-8", + ) + + legacy = index.get(current.run_id) + assert legacy is not None + assert legacy.mode == "unknown" + assert legacy.schema_version == 1 + + +def test_run_index_treats_a_legacy_journal_without_mode_as_unknown(tmp_path: Path) -> None: + journal = _journal(tmp_path, suffix="legacy1", mode="execute") + payload = json.loads(journal.path.read_text(encoding="utf-8")) + payload.pop("mode") + journal.path.write_text(json.dumps(payload), encoding="utf-8") + + record = RunIndex(tmp_path).index_journal(journal) + + assert record.mode == "unknown" + assert record.schema_version == 2 + + +def test_run_journal_rejects_an_invalid_mode_before_creating_files(tmp_path: Path) -> None: + context = RunContext.create("shop.weekly", "2026-07-27", random_suffix="badmode") + + with pytest.raises(ValueError, match="run mode"): + RunJournal.create(DataLayout(tmp_path), context, mode="preview") # type: ignore[arg-type] + + assert not (tmp_path / "runs").exists() + + def test_outbox_enqueue_is_atomic_and_idempotent(tmp_path: Path) -> None: outbox = Outbox(tmp_path) diff --git a/tests/test_product_commerce_module.py b/tests/test_product_commerce_module.py index 96c35bc..7ef0587 100644 --- a/tests/test_product_commerce_module.py +++ b/tests/test_product_commerce_module.py @@ -1,11 +1,11 @@ from __future__ import annotations +import importlib from pathlib import Path from gyxx_flow.adapters.native import DeferredModuleCommandStep from gyxx_flow.catalog import WorkflowCatalog from gyxx_flow.modules.product_commerce import ( - PRODUCT_MANUAL_WORKFLOW_IDS, PRODUCT_RESOURCE, PRODUCT_SCHEDULED_WORKFLOW_IDS, PRODUCT_TIMEOUT_SECONDS, @@ -16,14 +16,13 @@ from gyxx_flow.modules.product_commerce import ( PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_product_module_registers_all_available_entries() -> None: +def test_product_module_registers_only_scheduled_entries() -> None: module = ProductCommerceModule.from_catalog( WorkflowCatalog.load(PROJECT_ROOT / "config") ) assert tuple(item.workflow_id for item in module.workflow_definitions()) == PRODUCT_WORKFLOW_IDS - assert len(PRODUCT_SCHEDULED_WORKFLOW_IDS) == 7 - assert len(PRODUCT_MANUAL_WORKFLOW_IDS) == 3 + assert len(PRODUCT_SCHEDULED_WORKFLOW_IDS) == 8 def test_product_workflows_use_project_owned_module_commands() -> None: @@ -32,11 +31,61 @@ def test_product_workflows_use_project_owned_module_commands() -> None: ) for definition in module.workflow_definitions(): - step = definition.steps[0] - assert step.step_id == "module_execute" - assert isinstance(step.action, DeferredModuleCommandStep) - assert step.timeout_seconds == PRODUCT_TIMEOUT_SECONDS == 6 * 60 * 60 - assert step.max_attempts == 1 - assert step.resources == (PRODUCT_RESOURCE,) == ("module:product_commerce",) - assert step.production_sink is True + assert definition.steps + for step in definition.steps: + assert isinstance(step.action, DeferredModuleCommandStep) + assert step.timeout_seconds == PRODUCT_TIMEOUT_SECONDS == 6 * 60 * 60 + assert step.max_attempts == 1 + if definition.workflow_id == "product.main_image.weekly": + assert step.resources == (f"{PRODUCT_RESOURCE}:{step.step_id}",) + else: + assert step.resources == (PRODUCT_RESOURCE,) == ( + "module:product_commerce", + ) + assert step.production_sink is True + +def test_combined_main_image_workflow_runs_platforms_independently() -> None: + module = ProductCommerceModule.from_catalog( + WorkflowCatalog.load(PROJECT_ROOT / "config") + ) + + workflow = next( + item + for item in module.workflow_definitions() + if item.workflow_id == "product.main_image.weekly" + ) + assert [step.step_id for step in workflow.steps] == ["jd", "tmall"] + assert all(step.critical is True for step in workflow.steps) + assert all(step.depends_on == () for step in workflow.steps) + assert all(step.run_after_failure is False for step in workflow.steps) + assert [step.resources for step in workflow.steps] == [ + ("module:product_commerce:jd",), + ("module:product_commerce:tmall",), + ] + + +def test_product_runtime_compatibility_package_points_to_flattened_module() -> None: + module_root = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" + ).resolve() + compatibility = importlib.import_module( + "gyxx_flow.modules.product_commerce.runtime" + ) + legacy_module = importlib.import_module( + "gyxx_flow.modules.product_commerce.runtime.market_rank_limits" + ) + canonical_module = importlib.import_module( + "gyxx_flow.modules.product_commerce.market_rank_limits" + ) + + assert tuple(Path(path).resolve() for path in compatibility.__path__) == ( + module_root, + ) + assert Path(legacy_module.__file__).resolve() == Path( + canonical_module.__file__ + ).resolve() diff --git a/tests/test_product_runtime_data_boundaries.py b/tests/test_product_runtime_data_boundaries.py index f18fe6e..b13c5bd 100644 --- a/tests/test_product_runtime_data_boundaries.py +++ b/tests/test_product_runtime_data_boundaries.py @@ -7,20 +7,19 @@ from pathlib import Path import pytest PROJECT_ROOT = Path(__file__).resolve().parents[1] -RUNTIME_ROOT = ( +MODULE_ROOT = ( PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "product_commerce" - / "runtime" ) def _load_runtime_paths(monkeypatch: pytest.MonkeyPatch, data_root: Path): monkeypatch.setenv("GYXX_DATA_ROOT", str(data_root)) - monkeypatch.setenv("GYXX_MODULE_ROOT", str(RUNTIME_ROOT)) - module_path = RUNTIME_ROOT / "runtime_paths.py" + monkeypatch.setenv("GYXX_MODULE_ROOT", str(MODULE_ROOT)) + module_path = MODULE_ROOT / "runtime_paths.py" name = "_gyxx_product_runtime_paths_boundaries" spec = importlib.util.spec_from_file_location(name, module_path) assert spec and spec.loader @@ -60,7 +59,7 @@ def test_artifact_relative_path_is_relative_to_data_home( "data/raw/product_commerce/jd/record.json" ) with pytest.raises(ValueError, match="outside GYXX_DATA_ROOT"): - paths.artifact_relative_path(RUNTIME_ROOT / "record.json") + paths.artifact_relative_path(MODULE_ROOT / "record.json") @pytest.mark.parametrize( @@ -80,7 +79,7 @@ def test_artifact_relative_path_is_relative_to_data_home( def test_runtime_writers_do_not_fall_back_to_cwd_or_source_tree( relative: str, forbidden: str ) -> None: - text = (RUNTIME_ROOT / relative).read_text(encoding="utf-8-sig") + text = (MODULE_ROOT / relative).read_text(encoding="utf-8-sig") assert forbidden not in text @@ -97,12 +96,12 @@ def test_runtime_writers_do_not_fall_back_to_cwd_or_source_tree( def test_runtime_artifact_references_do_not_assume_source_tree_parent( relative: str, ) -> None: - text = (RUNTIME_ROOT / relative).read_text(encoding="utf-8-sig") + text = (MODULE_ROOT / relative).read_text(encoding="utf-8-sig") assert ".relative_to(PROJECT_ROOT)" not in text def test_checkpoints_and_operational_json_use_state_and_log_roots() -> None: - retry_text = (RUNTIME_ROOT / "collect_retry_utils.py").read_text( + retry_text = (MODULE_ROOT / "collect_retry_utils.py").read_text( encoding="utf-8-sig" ) assert "LOG_ROOT / \"failures\"" in retry_text @@ -112,14 +111,14 @@ def test_checkpoints_and_operational_json_use_state_and_log_roots() -> None: "collect_jd_market_rank.py", "collect_sycm_market_rank.py", ): - text = (RUNTIME_ROOT / relative).read_text(encoding="utf-8-sig") + text = (MODULE_ROOT / relative).read_text(encoding="utf-8-sig") assert "STATE_ROOT" in text assert "CHECKPOINT_PATH = STATE_ROOT" in text - decline = (RUNTIME_ROOT / "check_nine_day_decline.py").read_text( + decline = (MODULE_ROOT / "check_nine_day_decline.py").read_text( encoding="utf-8-sig" ) - insert = (RUNTIME_ROOT / "insert_bitable_records.py").read_text( + insert = (MODULE_ROOT / "insert_bitable_records.py").read_text( encoding="utf-8-sig" ) assert "LOG_PATH = LOG_ROOT" in decline @@ -144,5 +143,5 @@ def test_checkpoints_and_operational_json_use_state_and_log_roots() -> None: ], ) def test_cli_output_overrides_use_the_data_home_guard(relative: str) -> None: - text = (RUNTIME_ROOT / relative).read_text(encoding="utf-8-sig") + text = (MODULE_ROOT / relative).read_text(encoding="utf-8-sig") assert "managed_data_path" in text diff --git a/tests/test_product_source_migration.py b/tests/test_product_source_migration.py index 605a32e..017397c 100644 --- a/tests/test_product_source_migration.py +++ b/tests/test_product_source_migration.py @@ -10,14 +10,15 @@ from pathlib import Path from gyxx_flow.security import scan_repository PROJECT_ROOT = Path(__file__).resolve().parents[1] -RUNTIME_ROOT = ( +MODULE_ROOT = ( PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / "product_commerce" - / "runtime" ) +MIGRATED_TEST_ROOT = PROJECT_ROOT / "tests" / "modules" / "product_commerce" +HISTORY_ROOT = PROJECT_ROOT / "docs" / "history" / "product-commerce" MANIFEST_PATH = ( PROJECT_ROOT / "config" / "source-manifests" / "product_commerce.json" ) @@ -58,6 +59,8 @@ PRODUCTION_PYTHON = { "main_image_db.py", "main_image_paths.py", "market_rank_hermes_notification.py", + "market_rank_limits.py", + "market_rank_product_import.py", "market_rank_report_archive.py", "orchestrate_daily_collection.py", "orchestrate_market_rank_collection.py", @@ -113,6 +116,8 @@ LEGACY_TESTS = { "test_jd_market_rank.py", "test_main_image_concurrency.py", "test_market_rank_hermes_notification.py", + "test_market_rank_limits.py", + "test_market_rank_product_import.py", "test_market_rank_report_archive.py", "test_market_rank_workflow.py", "test_persona_launcher.py", @@ -142,6 +147,7 @@ INTENTIONALLY_EXCLUDED = { "debug_jd_main_image_snapshot.py", "inspect_dy_comment_filters.py", "jd_collect_test.py", + "scripts/inspect_fields.py", } SUBPROCESS_TARGETS = { @@ -182,7 +188,7 @@ def _sha256(path: Path) -> str: def _load_runtime_paths(): - path = RUNTIME_ROOT / "runtime_paths.py" + path = MODULE_ROOT / "runtime_paths.py" spec = importlib.util.spec_from_file_location("product_runtime_paths_test", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) @@ -191,16 +197,15 @@ def _load_runtime_paths(): def test_product_runtime_contains_the_complete_reviewed_source_snapshot() -> None: - assert len(PRODUCTION_PYTHON) == 63 - assert all((RUNTIME_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON) - assert all((RUNTIME_ROOT / relative).is_file() for relative in RESOURCES) + assert len(PRODUCTION_PYTHON) == 65 + assert all((MODULE_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON) + assert all((MODULE_ROOT / relative).is_file() for relative in RESOURCES) + assert all((MIGRATED_TEST_ROOT / name).is_file() for name in LEGACY_TESTS) assert all( - (RUNTIME_ROOT / "runtime_tests" / name).is_file() for name in LEGACY_TESTS - ) - assert all( - (RUNTIME_ROOT / "launchers_reference" / name).is_file() + (HISTORY_ROOT / "launchers_reference" / name).is_file() for name in LAUNCHER_REFERENCES ) + assert (HISTORY_ROOT / "MIGRATION_STATUS.md").is_file() def test_product_source_manifest_is_complete_and_verifiable() -> None: @@ -232,19 +237,24 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None: assert { item["source_relative_path"] for item in manifest["intentionally_excluded"] + if "source_relative_path" in item } == INTENTIONALLY_EXCLUDED + assert all( + item.get("source_relative_path") or item.get("pattern") + for item in manifest["intentionally_excluded"] + ) expected_targets = { *( - f"src/gyxx_flow/modules/product_commerce/runtime/{path}" + f"src/gyxx_flow/modules/product_commerce/{path}" for path in PRODUCTION_PYTHON | RESOURCES ), *( - f"src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/{name}" + f"tests/modules/product_commerce/{name}" for name in LEGACY_TESTS ), *( - f"src/gyxx_flow/modules/product_commerce/runtime/launchers_reference/{name}" + f"docs/history/product-commerce/launchers_reference/{name}" for name in LAUNCHER_REFERENCES ), } @@ -270,7 +280,7 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None: generated = audit["generated_files"] assert {item["target_relative_path"] for item in generated} == { - "src/gyxx_flow/modules/product_commerce/runtime/runtime_paths.py" + "src/gyxx_flow/modules/product_commerce/runtime_paths.py" } for item in generated: target = PROJECT_ROOT / item["target_relative_path"] @@ -279,10 +289,10 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None: def test_migrated_product_snapshot_has_no_plaintext_credentials_or_old_roots() -> None: - assert scan_repository(RUNTIME_ROOT) == [] + assert scan_repository(MODULE_ROOT) == [] forbidden = ("E:\\auto-flow", "D:\\product-collector-analyze-flow") - for path in RUNTIME_ROOT.rglob("*"): + for path in MODULE_ROOT.rglob("*"): if not path.is_file() or path.suffix.casefold() in {".pyc", ".pyo"}: continue try: @@ -293,11 +303,11 @@ def test_migrated_product_snapshot_has_no_plaintext_credentials_or_old_roots() - def test_product_runtime_paths_are_portable_and_layered(tmp_path, monkeypatch) -> None: - monkeypatch.setenv("GYXX_MODULE_ROOT", str(RUNTIME_ROOT)) + monkeypatch.setenv("GYXX_MODULE_ROOT", str(MODULE_ROOT)) monkeypatch.setenv("GYXX_DATA_ROOT", str(tmp_path)) paths = _load_runtime_paths() - assert paths.MODULE_ROOT == RUNTIME_ROOT.resolve() + assert paths.MODULE_ROOT == MODULE_ROOT.resolve() assert paths.DATA_HOME == tmp_path.resolve() assert paths.RAW_DATA_ROOT == tmp_path / "data" / "raw" / "product_commerce" assert paths.NORMALIZED_DATA_ROOT == ( @@ -317,10 +327,10 @@ def test_product_runtime_paths_are_portable_and_layered(tmp_path, monkeypatch) - assert paths.TMP_ROOT == tmp_path / "tmp" / "product_commerce" assert paths.vendor_source_root("jd") == ( - RUNTIME_ROOT / "vendors" / "jd-data-flow" + MODULE_ROOT / "vendors" / "jd-data-flow" ).resolve() assert paths.vendor_source_root("dy") == ( - RUNTIME_ROOT / "vendors" / "dy-data-flow" + MODULE_ROOT / "vendors" / "dy-data-flow" ).resolve() for relative in SUBPROCESS_TARGETS: assert paths.runtime_script(relative).is_file() @@ -345,7 +355,7 @@ def test_product_runtime_has_no_machine_or_legacy_path_escape_hatches() -> None: 'ROOT / "data"', 'os.path.join(PROJECT_ROOT, "data"', ) - for path in RUNTIME_ROOT.rglob("*"): + for path in MODULE_ROOT.rglob("*"): if not path.is_file() or path.suffix.casefold() in {".pyc", ".pyo"}: continue try: @@ -359,11 +369,11 @@ def test_product_runtime_has_no_hardcoded_long_hex_keys() -> None: hardcoded_key = re.compile( r"(?is)(?:api[_-]?key|secret|token).{0,160}?[\"'][0-9a-f]{32,}[\"']" ) - for path in RUNTIME_ROOT.rglob("*.py"): + for path in MODULE_ROOT.rglob("*.py"): text = path.read_text(encoding="utf-8-sig") assert hardcoded_key.search(text) is None, path - analyzer = (RUNTIME_ROOT / "analyze_style_with_hermes.py").read_text( + analyzer = (MODULE_ROOT / "analyze_style_with_hermes.py").read_text( encoding="utf-8-sig" ) assert 'os.getenv("HERMES_API_KEY")' in analyzer @@ -372,7 +382,7 @@ def test_product_runtime_has_no_hardcoded_long_hex_keys() -> None: def test_product_core_runtime_has_no_known_f821_regressions() -> None: erp_tree = ast.parse( - (RUNTIME_ROOT / "collect_erp_yesterday_metrics.py").read_text( + (MODULE_ROOT / "collect_erp_yesterday_metrics.py").read_text( encoding="utf-8-sig" ) ) @@ -398,8 +408,8 @@ def test_product_core_runtime_has_no_known_f821_regressions() -> None: self.stored.add(node.id) jd_paths = ( - RUNTIME_ROOT / "jd_product_data_collector.py", - RUNTIME_ROOT / "vendors" / "jd-data-flow" / "jd_product_data_collector.py", + MODULE_ROOT / "jd_product_data_collector.py", + MODULE_ROOT / "vendors" / "jd-data-flow" / "jd_product_data_collector.py", ) for path in jd_paths: tree = ast.parse(path.read_text(encoding="utf-8-sig")) diff --git a/tests/test_relocated_project.py b/tests/test_relocated_project.py index 84ac221..858207d 100644 --- a/tests/test_relocated_project.py +++ b/tests/test_relocated_project.py @@ -53,8 +53,11 @@ def test_catalog_and_scripts_work_after_project_and_data_roots_move( assert workflows.returncode == 0, workflows.stderr assert scripts.returncode == 0, scripts.stderr - assert len(json.loads(workflows.stdout)) == 29 - assert len(json.loads(scripts.stdout)) >= 100 + assert len(json.loads(workflows.stdout)) == 23 + commands = json.loads(scripts.stdout) + assert len(commands) == 33 + assert len({row["command_id"] for row in commands}) == 33 + assert all(row["legacy_script_id"].startswith(f"{row['module']}:") for row in commands) @pytest.mark.parametrize( diff --git a/tests/test_runtime_env_loading.py b/tests/test_runtime_env_loading.py new file mode 100644 index 0000000..f4076b7 --- /dev/null +++ b/tests/test_runtime_env_loading.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +import pytest + +from gyxx_flow.cli import ( + CliConfigurationError, + _load_runtime_environment_files, + build_parser, +) + + +def test_console_and_scheduler_accept_explicit_repeatable_env_files() -> None: + parser = build_parser() + + console = parser.parse_args( + ["console", "--env-file", "database.env", "--env-file", "notify.env"] + ) + scheduler = parser.parse_args( + ["schedule", "run", "--env-file", "production.env"] + ) + + assert console.env_file == [Path("database.env"), Path("notify.env")] + assert scheduler.env_file == [Path("production.env")] + + +def test_runtime_env_files_load_values_without_overriding_process_env( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = tmp_path / "database.env" + second = tmp_path / "notifications.env" + first.write_text( + "GYXX_RUNTIME_ENV_TEST_DB=cloud\nGYXX_RUNTIME_ENV_TEST_KEEP=file\n", + encoding="utf-8", + ) + second.write_text("GYXX_RUNTIME_ENV_TEST_NOTIFY=owner\n", encoding="utf-8") + monkeypatch.delenv("GYXX_RUNTIME_ENV_TEST_DB", raising=False) + monkeypatch.delenv("GYXX_RUNTIME_ENV_TEST_NOTIFY", raising=False) + monkeypatch.setenv("GYXX_RUNTIME_ENV_TEST_KEEP", "process") + + _load_runtime_environment_files( + argparse.Namespace(env_file=[first, second]) + ) + + import os + + assert os.environ["GYXX_RUNTIME_ENV_TEST_DB"] == "cloud" + assert os.environ["GYXX_RUNTIME_ENV_TEST_NOTIFY"] == "owner" + assert os.environ["GYXX_RUNTIME_ENV_TEST_KEEP"] == "process" + + +def test_runtime_env_file_must_exist(tmp_path: Path) -> None: + missing = tmp_path / "missing.env" + + with pytest.raises(CliConfigurationError, match="runtime environment file not found"): + _load_runtime_environment_files(argparse.Namespace(env_file=[missing])) diff --git a/tests/test_runtime_integrations.py b/tests/test_runtime_integrations.py index ab0598c..1116f56 100644 --- a/tests/test_runtime_integrations.py +++ b/tests/test_runtime_integrations.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import os +import subprocess import sys from pathlib import Path @@ -11,6 +13,9 @@ from gyxx_flow.adapters import ( RuntimeIntegrationCatalog, RuntimeIntegrationError, RuntimeServicePolicy, + binding_from_environment, + environment_for_child_script, + resolve_hermes_profile_api_key, ) from gyxx_flow.adapters.bootstrap import _rewrite_browser_arguments from gyxx_flow.adapters.native import ModuleCommandAdapter, ModuleSourceRoots @@ -29,9 +34,9 @@ def _catalog(tmp_path: Path) -> ScriptCatalog: second.write_text("print('second')\n", encoding="utf-8") return ScriptCatalog( ( - ScriptEntry("demo:first.py", "demo", "first.py", "python", first), + ScriptEntry("demo.first", "demo", "first.py", "python", first), ScriptEntry( - "demo:nested/second.py", + "demo.second", "demo", "nested/second.py", "python", @@ -45,17 +50,34 @@ def _write_config(path: Path) -> None: path.write_text( json.dumps( { - "schema_version": 1, + "schema_version": 2, "cdp_host": "127.0.0.1", "scripts": { - "demo:first.py": 22001, - "demo:nested/second.py": 22002, + "demo.first": { + "script_id": "demo:first.py", + "state_key": "demo:first.py", + "aliases": ["demo:first.py"], + "cdp_port": 22001, + "login_mode": "C", + "required_cookie_domains": ["example.test"], + "required_cookie_names": ["session"], + "credential_env_names": ["DEMO_ACCOUNT", "DEMO_PASSWORD"], + }, + "demo.second": { + "script_id": "demo:nested/second.py", + "state_key": "demo:nested/second.py", + "aliases": ["demo:nested/second.py"], + "cdp_port": 22002, + }, }, "services": { "feishu": "legacy", "postgres": "cloud", "hermes": "local", "hermes_url": "http://127.0.0.1:8642/v1/chat/completions", + "hermes_collector_url": "http://127.0.0.1:8643/v1/chat/completions", + "hermes_analyzer_gateway_url": "http://127.0.0.1:8642/v1", + "hermes_collector_gateway_url": "http://127.0.0.1:8643/v1", }, } ), @@ -77,12 +99,18 @@ def test_project_registry_covers_every_script_with_unique_stable_port() -> None: data_root=project_root / "var", ) - assert len(scripts.script_ids) == 131 - bindings = [first.binding_for(script_id) for script_id in scripts.script_ids] - assert len({binding.cdp_port for binding in bindings}) == 131 + assert len(scripts.command_ids) == 33 + assert len(first.command_ids) == 137 + bindings = [first.binding_for(command_id) for command_id in first.command_ids] + assert len({binding.cdp_port for binding in bindings}) == 137 assert all(22000 <= binding.cdp_port <= 22999 for binding in bindings) assert all(binding.cdp_url.startswith("http://127.0.0.1:") for binding in bindings) assert bindings == [second.binding_for(item.script_id) for item in bindings] + appeal = first.binding_for("shop.douyin_price_appeal") + assert appeal is first.binding_for( + "shop_intelligence:collectors/dy_store_competitor_store_scraping.py" + ) + assert appeal.cdp_port == 22104 def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Path) -> None: @@ -107,6 +135,65 @@ def test_bindings_isolate_browser_state_and_relocate_with_data_root(tmp_path: Pa assert not first.profile_dir.exists() +def test_schema_v1_keeps_legacy_script_ids_and_existing_state_paths(tmp_path: Path) -> None: + scripts = _catalog(tmp_path) + stable_config = tmp_path / "stable.json" + legacy_config = tmp_path / "legacy.json" + _write_config(stable_config) + payload = json.loads(stable_config.read_text(encoding="utf-8")) + payload["schema_version"] = 1 + payload["scripts"] = { + item["script_id"]: item["cdp_port"] + for item in payload["scripts"].values() + } + legacy_config.write_text(json.dumps(payload), encoding="utf-8") + + stable = RuntimeIntegrationCatalog.load( + stable_config, scripts=scripts, data_root=tmp_path / "data" + ) + legacy = RuntimeIntegrationCatalog.load( + legacy_config, scripts=scripts, data_root=tmp_path / "data" + ) + + assert stable.binding_for("demo.first").profile_dir == legacy.binding_for( + "demo:first.py" + ).profile_dir + assert stable.binding_for("demo.first").cookie_file == legacy.binding_for( + "demo:first.py" + ).cookie_file + + +def test_state_key_survives_a_physical_entry_move_and_old_alias_still_resolves( + tmp_path: Path, +) -> None: + scripts = _catalog(tmp_path) + config = tmp_path / "bindings.json" + _write_config(config) + before = RuntimeIntegrationCatalog.load( + config, scripts=scripts, data_root=tmp_path / "data" + ).binding_for("demo.first") + payload = json.loads(config.read_text(encoding="utf-8")) + payload["scripts"]["demo.first"]["script_id"] = "demo:moved/first.py" + config.write_text(json.dumps(payload), encoding="utf-8") + moved_path = tmp_path / "runtime" / "moved" / "first.py" + moved_path.parent.mkdir() + moved_path.write_text("print('moved')\n", encoding="utf-8") + moved_scripts = ScriptCatalog( + (ScriptEntry("demo.first", "demo", "moved/first.py", "python", moved_path),) + ) + + catalog = RuntimeIntegrationCatalog.load( + config, scripts=moved_scripts, data_root=tmp_path / "data" + ) + after = catalog.binding_for("demo.first") + + assert catalog.binding_for("demo:first.py") is after + assert catalog.binding_for("demo:moved/first.py") is after + assert after.profile_dir == before.profile_dir + assert after.cookie_file == before.cookie_file + assert after.storage_state_file == before.storage_state_file + + def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" @@ -119,10 +206,128 @@ def test_parent_and_nested_child_receive_different_runtime_environment(tmp_path: assert child["KEEP"] == "yes" assert parent["GYXX_SCRIPT_ID"] == "demo:first.py" assert child["GYXX_SCRIPT_ID"] == "demo:nested/second.py" + assert parent["GYXX_COMMAND_ID"] == "demo.first" + assert child["GYXX_COMMAND_ID"] == "demo.second" + assert parent["GYXX_BROWSER_STATE_KEY"] == "demo:first.py" + assert parent["GYXX_BROWSER_LOGIN_MODE"] == "C" + assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_DOMAINS"]) == [ + "example.test" + ] + assert json.loads(parent["GYXX_BROWSER_REQUIRED_COOKIE_NAMES"]) == ["session"] + assert json.loads(parent["GYXX_BROWSER_CREDENTIAL_ENV_NAMES"]) == [ + "DEMO_ACCOUNT", + "DEMO_PASSWORD", + ] + assert parent["DY_COOKIES_FILE"] == parent["GYXX_BROWSER_COOKIE_FILE"] assert parent["GYXX_BROWSER_CDP_PORT"] != child["GYXX_BROWSER_CDP_PORT"] assert parent["GYXX_BROWSER_PROFILE_DIR"] != child["GYXX_BROWSER_PROFILE_DIR"] assert parent["GYXX_BROWSER_COOKIE_FILE"] != child["GYXX_BROWSER_COOKIE_FILE"] + reconstructed = binding_from_environment(parent) + assert reconstructed.login_mode == "C" + assert reconstructed.required_cookie_domains == ("example.test",) + assert reconstructed.required_cookie_names == ("session",) + assert reconstructed.credential_env_names == ("DEMO_ACCOUNT", "DEMO_PASSWORD") + + +def test_private_nested_child_keeps_legacy_alias_binding(tmp_path: Path) -> None: + project_root = Path(__file__).resolve().parents[1] + module_root = ( + project_root + / "src" + / "gyxx_flow" + / "modules" + / "product_commerce" + / "runtime" + ) + environment = environment_for_child_script( + module_root / "taobao_sycm_products.py", + { + "GYXX_MODULE_ID": "product_commerce", + "GYXX_MODULE_ROOT": str(module_root), + "GYXX_PROJECT_ROOT": str(project_root), + "GYXX_DATA_ROOT": str(tmp_path / "data"), + }, + ) + + assert environment["GYXX_SCRIPT_ID"] == "product_commerce:taobao_sycm_products.py" + assert environment["GYXX_COMMAND_ID"] == "product_commerce:taobao_sycm_products.py" + assert environment["GYXX_BROWSER_STATE_KEY"] == ( + "product_commerce:taobao_sycm_products.py" + ) + assert environment["GYXX_BROWSER_CDP_PORT"] == "22096" + + +def test_jd_self_operated_collectors_consume_their_own_browser_binding( + tmp_path: Path, +) -> None: + project_root = Path(__file__).resolve().parents[1] + scripts = ScriptCatalog.discover_default() + catalog = RuntimeIntegrationCatalog.load( + project_root / "config" / "runtime-bindings.json", + scripts=scripts, + data_root=tmp_path / "portable-data", + ) + module_root = ( + project_root + / "src" + / "gyxx_flow" + / "modules" + / "shop_intelligence" + ) + cases = ( + ( + "shop_intelligence:collectors/jd_self_operated_brand_daily.py", + "gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_brand_daily", + ), + ( + "shop_intelligence:collectors/jd_self_operated_product_daily.py", + "gyxx_flow.modules.shop_intelligence.collectors.jd_self_operated_product_daily", + ), + ) + observed_ports: list[int] = [] + + for script_id, module_name in cases: + binding = catalog.binding_for(script_id) + environment = catalog.environment_for(script_id, os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + ( + str(project_root / "src"), + str(module_root), + environment.get("PYTHONPATH", ""), + ) + ) + command = ( + "import json; " + f"import {module_name} as module; " + "print(json.dumps({" + "'port': module.CDP_PORT, " + "'profile': str(module.BROWSER_PROFILE_DIR), " + "'storage': str(module.STORAGE_STATE_PATH)" + "}))" + ) + result = subprocess.run( + [sys.executable, "-c", command], + cwd=module_root, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout.strip().splitlines()[-1]) + assert payload == { + "port": binding.cdp_port, + "profile": str(binding.profile_dir), + "storage": str(binding.storage_state_file), + } + observed_ports.append(payload["port"]) + + assert observed_ports == [22132, 22133] + def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) -> None: scripts = _catalog(tmp_path) @@ -144,6 +349,8 @@ def test_module_command_adapter_injects_entry_script_binding(tmp_path: Path) -> assert command.env["KEEP"] == "yes" assert command.env["GYXX_SCRIPT_ID"] == "demo:first.py" + assert command.env["GYXX_COMMAND_ID"] == "demo.first" + assert command.env["GYXX_BROWSER_STATE_KEY"] == "demo:first.py" assert command.env["GYXX_BROWSER_CDP_PORT"] == "22001" @@ -166,13 +373,14 @@ def test_cookie_store_round_trip_and_storage_state_are_atomic(tmp_path: Path) -> assert "secret-sentinel" not in repr(store) -def test_service_policy_preserves_original_feishu_and_cloud_db_and_local_hermes() -> None: +def test_service_policy_preserves_feishu_and_uses_cloud_pg_and_local_hermes() -> None: policy = RuntimeServicePolicy( hermes_url="http://127.0.0.1:8642/v1/chat/completions" ) original = { "LARK_PROFILE": "original-profile", "PG_HOST": "cloud-db.example.test", + "PG_PORT": "5432", "PG_PASSWORD": "placeholder", "DB_HOST": "", "CUSTOM": "keep", @@ -185,21 +393,28 @@ def test_service_policy_preserves_original_feishu_and_cloud_db_and_local_hermes( assert result["PG_PASSWORD"] == "placeholder" assert result["DB_HOST"] == "cloud-db.example.test" assert result["AUTOFLOW_PG_HOST"] == "cloud-db.example.test" + assert result["DB_PORT"] == "5432" + assert result["AUTOFLOW_PG_PORT"] == "5432" assert result["GYXX_FEISHU_MODE"] == "legacy" assert result["GYXX_POSTGRES_MODE"] == "cloud" assert result["GYXX_HERMES_MODE"] == "local" assert result["HERMES_ANALYZER_URL"].startswith("http://127.0.0.1:") + assert result["COLLECTOR_API_SERVER_URL"].startswith("http://127.0.0.1:") + assert result["ANALYZER_HERMES_GATEWAY_URL"].endswith(":8642/v1") + assert result["COLLECTOR_HERMES_GATEWAY_URL"].endswith(":8643/v1") assert "secret-sentinel" not in repr(policy) @pytest.mark.parametrize( ("environment", "message"), [ - ({"PG_HOST": "127.0.0.1"}, "cloud"), - ({"DB_HOST": "localhost"}, "cloud"), - ({"AUTOFLOW_PG_HOST": "::1"}, "cloud"), + ({"PG_HOST": "127.0.0.1"}, "remote"), + ({"DB_HOST": "localhost"}, "remote"), + ({"AUTOFLOW_PG_HOST": "::1"}, "remote"), + ({"DATABASE_URL": "postgresql://user@127.0.0.1/gyxx"}, "remote"), ({"HERMES_ANALYZER_URL": "https://remote.example.test/v1"}, "local"), ({"ANALYZER_API_SERVER_URL": "http://10.0.0.8:8642/v1"}, "local"), + ({"COLLECTOR_HERMES_GATEWAY_URL": "https://remote.example.test/v1"}, "local"), ], ) def test_service_policy_rejects_local_database_or_remote_hermes( @@ -209,18 +424,114 @@ def test_service_policy_rejects_local_database_or_remote_hermes( RuntimeServicePolicy().apply(environment) +def test_service_policy_supplies_portable_local_pg_defaults_without_password() -> None: + result = RuntimeServicePolicy( + postgres_mode="local", + postgres_host="127.0.0.1", + postgres_database="gyxx_super_data", + postgres_user="gyxx_flow", + ).apply({}) + + assert result["PG_HOST"] == "127.0.0.1" + assert result["DB_HOST"] == "127.0.0.1" + assert result["AUTOFLOW_PG_HOST"] == "127.0.0.1" + assert result["PG_PORT"] == "5432" + assert result["DB_PORT"] == "5432" + assert result["AUTOFLOW_PG_PORT"] == "5432" + assert result["PG_DB"] == "gyxx_super_data" + assert result["DB_NAME"] == "gyxx_super_data" + assert result["PG_USER"] == "gyxx_flow" + assert "PG_PASSWORD" not in result + + +def test_service_policy_maps_canonical_secrets_without_exposing_defaults() -> None: + result = RuntimeServicePolicy().apply( + { + "GYXX_POSTGRES_PASSWORD": "placeholder", + "GYXX_HERMES_API_KEY": "placeholder", + } + ) + + assert result["PG_PASSWORD"] == "placeholder" + assert result["DB_PASSWORD"] == "placeholder" + assert result["AUTOFLOW_PG_PASSWORD"] == "placeholder" + assert result["HERMES_API_KEY"] == "placeholder" + assert result["HERMES_ANALYZER_TOKEN"] == "placeholder" + assert result["GYXX_SUPPLY_HERMES_TOKEN"] == "placeholder" + + +def test_shared_hermes_resolver_reads_runtime_profile_without_copying_secret( + tmp_path: Path, +) -> None: + profile = tmp_path / "profiles" / "data-analyzer" + profile.mkdir(parents=True) + (profile / ".env").write_text( + "API_SERVER_PORT=8642\nAPI_SERVER_KEY=local-only-key\n", + encoding="utf-8", + ) + + assert resolve_hermes_profile_api_key( + "data-analyzer", + {"HERMES_HOME": str(tmp_path)}, + ) == "local-only-key" + + +def test_shared_hermes_resolver_prefers_explicit_runtime_secret(tmp_path: Path) -> None: + assert resolve_hermes_profile_api_key( + "data-analyzer", + { + "HERMES_HOME": str(tmp_path), + "HERMES_ANALYZER_TOKEN": "explicit-key", + }, + preferred_environment_names=("HERMES_ANALYZER_TOKEN",), + ) == "explicit-key" + + +def test_service_policy_replaces_present_but_blank_local_pg_aliases() -> None: + result = RuntimeServicePolicy( + postgres_mode="local", + postgres_host="127.0.0.1", + postgres_database="gyxx_super_data", + postgres_user="gyxx_flow", + ).apply( + {"PG_HOST": "", "DB_PORT": " ", "AUTOFLOW_PG_DB": ""} + ) + + assert result["PG_HOST"] == "127.0.0.1" + assert result["DB_PORT"] == "5432" + assert result["AUTOFLOW_PG_DB"] == "gyxx_super_data" + + +def test_service_policy_maps_cloud_dsn_to_all_database_aliases() -> None: + result = RuntimeServicePolicy().apply( + {"GYXX_POSTGRES_DSN": "postgresql://cloud_user:placeholder@db.example.test/data_hub"} + ) + + assert result["GYXX_POSTGRES_MODE"] == "cloud" + assert result["PG_HOST"] == "db.example.test" + assert result["DB_HOST"] == "db.example.test" + assert result["AUTOFLOW_PG_HOST"] == "db.example.test" + assert result["PG_DB"] == "data_hub" + assert result["AUTOFLOW_PG_USER"] == "cloud_user" + + def test_registry_rejects_missing_duplicate_or_unknown_script_allocations(tmp_path: Path) -> None: scripts = _catalog(tmp_path) config = tmp_path / "bindings.json" _write_config(config) payload = json.loads(config.read_text(encoding="utf-8")) - payload["scripts"]["demo:nested/second.py"] = 22001 + payload["scripts"]["demo.second"]["cdp_port"] = 22001 config.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeIntegrationError, match="unique"): RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") - payload["scripts"].pop("demo:nested/second.py") - payload["scripts"]["demo:unknown.py"] = 22003 + payload["scripts"].pop("demo.second") + payload["scripts"]["demo.unknown"] = { + "script_id": "demo:unknown.py", + "state_key": "demo:unknown.py", + "aliases": ["demo:unknown.py"], + "cdp_port": 22003, + } config.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeIntegrationError, match="coverage"): RuntimeIntegrationCatalog.load(config, scripts=scripts, data_root=tmp_path / "data") diff --git a/tests/test_runtime_output_boundaries.py b/tests/test_runtime_output_boundaries.py index 9232b13..7cc892d 100644 --- a/tests/test_runtime_output_boundaries.py +++ b/tests/test_runtime_output_boundaries.py @@ -25,7 +25,7 @@ def test_migrated_runtimes_have_no_known_mutable_output_bypasses( module: str, forbidden: str ) -> None: offenders = [] - for path in (MODULE_ROOT / module / "runtime").rglob("*.py"): + for path in (MODULE_ROOT / module).rglob("*.py"): source = path.read_text(encoding="utf-8-sig") if forbidden in source: offenders.append(path.relative_to(PROJECT_ROOT).as_posix()) @@ -35,10 +35,10 @@ def test_migrated_runtimes_have_no_known_mutable_output_bypasses( def test_runtime_path_modules_do_not_use_source_local_data_fallbacks() -> None: path_files = [ - MODULE_ROOT / "content_marketing" / "runtime" / "runtime_paths.py", - MODULE_ROOT / "product_commerce" / "runtime" / "runtime_paths.py", - MODULE_ROOT / "shop_intelligence" / "runtime" / "paths.py", - MODULE_ROOT / "supply_chain" / "runtime" / "paths.py", + MODULE_ROOT / "content_marketing" / "runtime_paths.py", + MODULE_ROOT / "product_commerce" / "runtime_paths.py", + MODULE_ROOT / "shop_intelligence" / "paths.py", + MODULE_ROOT / "supply_chain" / "paths.py", ] forbidden = ('MODULE_ROOT / ".runtime-data"', 'module_root / "var"', 'PROJECT_ROOT / "data"') diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py deleted file mode 100644 index a2d2965..0000000 --- a/tests/test_scheduler.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -import xml.etree.ElementTree as ET -from datetime import date -from pathlib import Path - -from gyxx_flow.catalog import WorkflowCatalog -from gyxx_flow.scheduler import ( - CurrentScheduledTask, - PowerShellCurrentTaskProvider, - SchedulerConfig, - build_schedule_plan, - detect_schedule_drift, - write_schedule_plan_bundle, -) - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -NS = {"t": "http://schemas.microsoft.com/windows/2004/02/mit/task"} - - -def test_windows_provider_reads_managed_tasks_without_mutation(monkeypatch) -> None: - captured: dict[str, object] = {} - - def fake_run(command, **kwargs): - captured["command"] = command - captured["kwargs"] = kwargs - return subprocess.CompletedProcess( - command, - 0, - stdout=json.dumps( - [{"full_name": r"\GYXX\product.daily", "xml": ""}] - ), - stderr="", - ) - - monkeypatch.setattr(subprocess, "run", fake_run) - tasks = PowerShellCurrentTaskProvider().current_tasks("\\GYXX\\") - - assert tasks == [CurrentScheduledTask(r"\GYXX\product.daily", "")] - assert captured["command"][0] == "powershell.exe" - assert "Register-ScheduledTask" not in captured["command"][-1] - assert captured["kwargs"]["shell"] is False - - -def _plan(tmp_path: Path): - catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") - return build_schedule_plan( - catalog, - SchedulerConfig( - python_executable=Path(r"C:\Portable\Python312\python.exe"), - project_root=tmp_path / "portable-project", - ), - start_date=date(2026, 7, 27), - ) - - -def test_plan_generates_21_inert_portable_windows_task_definitions(tmp_path: Path) -> None: - plan = _plan(tmp_path) - - assert plan.timezone == "Asia/Shanghai" - assert len(plan.tasks) == 21 - assert len({task.full_name for task in plan.tasks}) == 21 - assert all(task.full_name.startswith("\\GYXX\\") for task in plan.tasks) - assert all("Register-ScheduledTask" not in task.xml for task in plan.tasks) - - sample = next(task for task in plan.tasks if task.workflow_id == "product.daily") - root = ET.fromstring(sample.xml) - assert root.findtext(".//t:Exec/t:Command", namespaces=NS) == str( - plan.config.python_executable - ) - assert root.findtext(".//t:Exec/t:Arguments", namespaces=NS) == ( - "-m gyxx_flow run product.daily --scheduled" - ) - assert root.findtext(".//t:Exec/t:WorkingDirectory", namespaces=NS) == str( - plan.config.project_root - ) - assert root.findtext(".//t:MultipleInstancesPolicy", namespaces=NS) == "IgnoreNew" - - -def test_plan_preserves_daily_weekly_monthly_and_interval_semantics(tmp_path: Path) -> None: - plan = _plan(tmp_path) - by_id = {task.workflow_id: ET.fromstring(task.xml) for task in plan.tasks} - - assert by_id["product.daily"].findtext( - ".//t:ScheduleByDay/t:DaysInterval", namespaces=NS - ) == "1" - weekly = by_id["content.comments.weekly"] - assert weekly.find(".//t:ScheduleByWeek/t:DaysOfWeek/t:Sunday", NS) is not None - monthly = by_id["content.summary.monthly"] - assert monthly.findtext( - ".//t:ScheduleByMonth/t:DaysOfMonth/t:Day", namespaces=NS - ) == "1" - interval = by_id["product.style_analysis.interval"] - assert interval.findtext( - ".//t:ScheduleByDay/t:DaysInterval", namespaces=NS - ) == "3" - assert interval.findtext(".//t:StartBoundary", namespaces=NS).startswith( - "2026-07-25T11:00:00" - ) - - -def test_bundle_write_is_a_plan_only_and_never_applies_tasks(tmp_path: Path) -> None: - plan = _plan(tmp_path) - destination = tmp_path / "schedule-plan" - - written = write_schedule_plan_bundle(destination, plan) - - assert len(list((destination / "tasks").glob("*.xml"))) == 21 - assert written == destination.resolve() - payload = json.loads((destination / "plan.json").read_text(encoding="utf-8")) - assert payload["apply_required"] is True - assert payload["applied"] is False - install = (destination / "install.ps1").read_text(encoding="utf-8") - assert "Register-ScheduledTask" in install - assert "China Standard Time" in install - assert "-WhatIf" in install - assert "[string]$WorkflowId" in install - assert "Apply requires exactly one safe -WorkflowId" in install - assert "ForEach-Object" not in install - - -def test_drift_reports_missing_changed_and_managed_extra_without_values( - tmp_path: Path, -) -> None: - plan = _plan(tmp_path) - first, second, *rest = plan.tasks - changed_xml = second.xml.replace("--scheduled", "--changed") - current = [ - CurrentScheduledTask(first.full_name, first.xml), - CurrentScheduledTask(second.full_name, changed_xml), - *[CurrentScheduledTask(task.full_name, task.xml) for task in rest], - CurrentScheduledTask(r"\GYXX\obsolete.workflow", first.xml), - CurrentScheduledTask(r"\Unrelated\keep", "secret-value-must-not-appear"), - ] - current = [item for item in current if item.full_name != rest[-1].full_name] - - report = detect_schedule_drift(plan, current) - - assert report.is_clean is False - assert [(item.full_name, item.status) for item in report.items] == [ - (second.full_name, "changed"), - (rest[-1].full_name, "missing"), - (r"\GYXX\obsolete.workflow", "extra"), - ] - assert "secret-value-must-not-appear" not in json.dumps(report.as_dict()) diff --git a/tests/test_scheduler_cli.py b/tests/test_scheduler_cli.py index aaf5ee2..18031da 100644 --- a/tests/test_scheduler_cli.py +++ b/tests/test_scheduler_cli.py @@ -2,72 +2,32 @@ from __future__ import annotations import io import json -from datetime import date from pathlib import Path from gyxx_flow.catalog import WorkflowCatalog -from gyxx_flow.cli import main +from gyxx_flow.cli import EXIT_USAGE, build_parser, main from gyxx_flow.core.config import Settings -from gyxx_flow.scheduler import CurrentScheduledTask from gyxx_flow.workflow import WorkflowDefinition from gyxx_flow.workflow.registry import WorkflowRegistry PROJECT_ROOT = Path(__file__).resolve().parents[1] -class FakeCurrentTaskProvider: - def __init__(self, tasks: list[CurrentScheduledTask]) -> None: - self.tasks = tasks - self.requested_path: str | None = None - - def current_tasks(self, task_path: str) -> list[CurrentScheduledTask]: - self.requested_path = task_path - return self.tasks - - -def test_schedule_plan_cli_writes_inert_bundle_and_drift_report(tmp_path: Path) -> None: - provider = FakeCurrentTaskProvider([]) - output = io.StringIO() - destination = tmp_path / "generated-plan" - settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path / "data") - - exit_code = main( - [ - "schedule", - "plan", - "--output", - str(destination), - "--start-date", - "2026-07-27", - "--python-executable", - str(PROJECT_ROOT / ".venv" / "Scripts" / "python.exe"), - ], - settings=settings, - current_task_provider=provider, - stdout=output, - stderr=io.StringIO(), +def test_schedule_cli_exposes_only_project_owned_service_commands() -> None: + parser = build_parser() + command_action = next(action for action in parser._actions if action.dest == "command") + schedule_parser = command_action.choices["schedule"] + schedule_action = next( + action for action in schedule_parser._actions if action.dest == "schedule_command" ) - assert exit_code == 0 - payload = json.loads(output.getvalue()) - assert payload["desired_count"] == 21 - assert payload["applied"] is False - assert payload["drift_count"] == 21 - assert provider.requested_path == "\\GYXX\\" - assert (destination / "install.ps1").is_file() - assert (destination / "drift.json").is_file() - assert json.loads((destination / "plan.json").read_text(encoding="utf-8"))[ - "applied" - ] is False + assert set(schedule_action.choices) == {"run", "status"} -def test_scheduled_run_uses_shanghai_business_date_and_executes_registered_workflow( - monkeypatch, tmp_path: Path -) -> None: +def test_run_rejects_implicit_scheduled_execution(tmp_path: Path) -> None: catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") registry = WorkflowRegistry(catalog) registry.register(WorkflowDefinition("product.daily", ())) - monkeypatch.setattr("gyxx_flow.cli._today_shanghai", lambda: date(2026, 7, 27)) output = io.StringIO() exit_code = main( @@ -78,8 +38,37 @@ def test_scheduled_run_uses_shanghai_business_date_and_executes_registered_workf stderr=io.StringIO(), ) + assert exit_code == EXIT_USAGE + assert output.getvalue() == "" + + +def test_python_scheduler_status_is_project_owned_and_cross_platform(tmp_path: Path) -> None: + output = io.StringIO() + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path / "data") + + exit_code = main( + ["schedule", "status"], settings=settings, + stdout=output, stderr=io.StringIO(), + ) + assert exit_code == 0 payload = json.loads(output.getvalue()) - assert payload["business_date"] == "2026-07-27" - assert payload["dry_run"] is False - assert payload["status"] == "success" + assert payload["timezone"] == "Asia/Shanghai" + assert payload["scheduled_count"] == 23 + assert payload["state"]["schema_version"] == 1 + + +def test_python_scheduler_dry_run_never_launches_or_writes_state(tmp_path: Path) -> None: + output = io.StringIO() + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path / "data") + + exit_code = main( + ["schedule", "run", "--dry-run", "--once", "--misfire-grace-seconds", "3456000"], + settings=settings, stdout=output, stderr=io.StringIO(), + ) + + assert exit_code == 0 + payload = json.loads(output.getvalue()) + assert payload["mode"] == "dry-run" + assert "product.market_rank" in payload["due_or_started"] + assert not (settings.data_root / "state" / "scheduler" / "state.json").exists() diff --git a/tests/test_scheduler_service.py b/tests/test_scheduler_service.py new file mode 100644 index 0000000..3f2f087 --- /dev/null +++ b/tests/test_scheduler_service.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import json +import os +from datetime import date, datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +from gyxx_flow.catalog import ScheduleEntry, WorkflowCatalog, WorkflowEntry +from gyxx_flow.scheduler_service import ( + PythonScheduler, + SchedulerInstanceLock, + SubprocessWorkflowLauncher, + latest_due_slot, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SHANGHAI = ZoneInfo("Asia/Shanghai") + + +class FakeProcess: + def __init__(self, pid: int = 1234) -> None: + self.pid = pid + self.return_code = None + + def poll(self): + return self.return_code + + +class FakeLauncher: + def __init__(self) -> None: + self.launched = [] + + def launch(self, workflow_id: str, slot: datetime, business_date: date): + process = FakeProcess(1000 + len(self.launched)) + self.launched.append((workflow_id, slot, business_date, process)) + return process + + def close(self, process) -> None: + pass + + +def test_latest_due_slot_supports_all_schedule_kinds() -> None: + now = datetime(2026, 7, 28, 10, 5, tzinfo=SHANGHAI) + assert latest_due_slot(ScheduleEntry("a", "daily", "10:00"), now) == datetime(2026, 7, 28, 10, 0, tzinfo=SHANGHAI) + assert latest_due_slot(ScheduleEntry("a", "weekly", "09:00", days=("Monday",)), now) == datetime(2026, 7, 27, 9, 0, tzinfo=SHANGHAI) + assert latest_due_slot(ScheduleEntry("a", "monthly", "08:00", day_of_month=1), now) == datetime(2026, 7, 1, 8, 0, tzinfo=SHANGHAI) + assert latest_due_slot(ScheduleEntry("a", "interval_days", "11:00", every_days=3, anchor_date="2026-07-25"), now) == datetime(2026, 7, 25, 11, 0, tzinfo=SHANGHAI) + + +def test_latest_due_slot_chooses_latest_of_multiple_daily_times() -> None: + schedule = ScheduleEntry( + "a", + "daily", + "08:00", + at_times=("08:00", "16:00", "22:00"), + ) + + assert latest_due_slot( + schedule, + datetime(2026, 7, 28, 7, 59, tzinfo=SHANGHAI), + ) == datetime(2026, 7, 27, 22, 0, tzinfo=SHANGHAI) + assert latest_due_slot( + schedule, + datetime(2026, 7, 28, 8, 5, tzinfo=SHANGHAI), + ) == datetime(2026, 7, 28, 8, 0, tzinfo=SHANGHAI) + assert latest_due_slot( + schedule, + datetime(2026, 7, 28, 16, 5, tzinfo=SHANGHAI), + ) == datetime(2026, 7, 28, 16, 0, tzinfo=SHANGHAI) + assert latest_due_slot( + schedule, + datetime(2026, 7, 28, 22, 5, tzinfo=SHANGHAI), + ) == datetime(2026, 7, 28, 22, 0, tzinfo=SHANGHAI) + + +def test_tick_launches_due_workflow_once_and_persists_slot(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler(catalog, tmp_path, launcher=launcher, misfire_grace_seconds=600) + now = datetime(2026, 7, 28, 10, 5, tzinfo=SHANGHAI) + + started = scheduler.tick(now) + scheduler.tick(now) + + assert {item[0] for item in launcher.launched} == { + "content.marketing_report.daily", + "content.summary.weekly", + "product.persona.daily", + } + assert set(started) == {item[0] for item in launcher.launched} + state = json.loads((tmp_path / "state" / "scheduler" / "state.json").read_text(encoding="utf-8")) + assert state["jobs"]["product.persona.daily"]["slot"].startswith("2026-07-28T10:00:00") + + +def test_completed_process_is_recorded_and_not_relaunched(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler(catalog, tmp_path, launcher=launcher, misfire_grace_seconds=600) + now = datetime(2026, 7, 28, 23, 5, tzinfo=SHANGHAI) + scheduler.tick(now) + process = next(item[3] for item in launcher.launched if item[0] == "product.alert.daily") + process.return_code = 0 + + scheduler.tick(now) + + state = scheduler.status() + assert state["jobs"]["product.alert.daily"]["status"] == "success" + assert sum(item[0] == "product.alert.daily" for item in launcher.launched) == 1 + + +def test_multiple_daily_slots_launch_once_without_overlap(tmp_path: Path) -> None: + workflow = WorkflowEntry( + workflow_id="shop.price_appeal", + module="shop_intelligence", + trigger="scheduled", + entry="run.py", + source_task_name="price appeal", + ) + schedule = ScheduleEntry( + workflow_id=workflow.workflow_id, + kind="daily", + at="08:00", + at_times=("08:00", "16:00", "22:00"), + ) + catalog = WorkflowCatalog( + timezone="Asia/Shanghai", + workflows=(workflow,), + schedules=(schedule,), + ) + launcher = FakeLauncher() + scheduler = PythonScheduler( + catalog, + tmp_path, + launcher=launcher, + misfire_grace_seconds=600, + ) + + scheduler.tick(datetime(2026, 7, 28, 8, 5, tzinfo=SHANGHAI)) + scheduler.tick(datetime(2026, 7, 28, 8, 5, tzinfo=SHANGHAI)) + morning_process = launcher.launched[0][3] + morning_process.return_code = 0 + scheduler.tick(datetime(2026, 7, 28, 8, 6, tzinfo=SHANGHAI)) + + scheduler.tick(datetime(2026, 7, 28, 16, 5, tzinfo=SHANGHAI)) + afternoon_process = launcher.launched[1][3] + scheduler.tick(datetime(2026, 7, 28, 22, 5, tzinfo=SHANGHAI)) + assert len(launcher.launched) == 2 + + afternoon_process.return_code = 0 + scheduler.tick(datetime(2026, 7, 28, 22, 5, tzinfo=SHANGHAI)) + scheduler.tick(datetime(2026, 7, 28, 22, 5, tzinfo=SHANGHAI)) + + assert [item[1].hour for item in launcher.launched] == [8, 16, 22] + assert scheduler.status()["jobs"][workflow.workflow_id]["slot"].startswith( + "2026-07-28T22:00:00" + ) + + +def test_misfire_older_than_grace_is_not_launched(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler(catalog, tmp_path, launcher=launcher, misfire_grace_seconds=60) + scheduler.tick(datetime(2026, 7, 28, 10, 2, tzinfo=SHANGHAI)) + assert launcher.launched == [] + + +def test_all_configured_schedules_are_enabled() -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + + assert len(catalog.schedules) == 23 + assert all(schedule.enabled for schedule in catalog.schedules) + + +def test_dry_run_reports_due_without_writing_state(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler(catalog, tmp_path, launcher=launcher, misfire_grace_seconds=600, dry_run=True) + due = scheduler.tick(datetime(2026, 7, 28, 10, 5, tzinfo=SHANGHAI)) + assert "product.persona.daily" in due + assert launcher.launched == [] + assert not (tmp_path / "state" / "scheduler" / "state.json").exists() + + +def test_restart_does_not_overlap_a_still_running_workflow(tmp_path: Path) -> None: + state_path = tmp_path / "state" / "scheduler" / "state.json" + state_path.parent.mkdir(parents=True) + state_path.write_text(json.dumps({ + "schema_version": 1, "updated_at": None, + "jobs": {"product.persona.daily": { + "slot": "2026-07-27T10:00:00+08:00", "status": "running", "pid": os.getpid(), + }}, + }), encoding="utf-8") + launcher = FakeLauncher() + scheduler = PythonScheduler( + WorkflowCatalog.load(PROJECT_ROOT / "config"), tmp_path, + launcher=launcher, misfire_grace_seconds=600, + ) + + scheduler.tick(datetime(2026, 7, 28, 10, 5, tzinfo=SHANGHAI)) + + assert "product.persona.daily" not in {item[0] for item in launcher.launched} + + +def test_scheduler_instance_lock_rejects_second_owner(tmp_path: Path) -> None: + lock_path = tmp_path / "service.lock" + with SchedulerInstanceLock(lock_path): + try: + with SchedulerInstanceLock(lock_path): + raise AssertionError("second lock must not be acquired") + except RuntimeError as exc: + assert "already running" in str(exc) + assert not lock_path.exists() + + +def test_schedule_business_date_offset_uses_previous_day(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler( + catalog, + tmp_path, + launcher=launcher, + misfire_grace_seconds=600, + ) + + scheduler.tick(datetime(2026, 8, 1, 16, 5, tzinfo=SHANGHAI)) + + launched = next( + item for item in launcher.launched + if item[0] == "shop.jd_self_operated.daily" + ) + assert launched[1].date() == date(2026, 8, 1) + assert launched[2] == date(2026, 7, 31) + + +def test_product_daily_schedule_passes_previous_business_day(tmp_path: Path) -> None: + catalog = WorkflowCatalog.load(PROJECT_ROOT / "config") + launcher = FakeLauncher() + scheduler = PythonScheduler( + catalog, + tmp_path, + launcher=launcher, + misfire_grace_seconds=600, + ) + + scheduler.tick(datetime(2026, 8, 1, 8, 45, tzinfo=SHANGHAI)) + + product_launch = next( + item for item in launcher.launched if item[0] == "product.daily" + ) + assert product_launch[1].date() == date(2026, 8, 1) + assert product_launch[2] == date(2026, 7, 31) + + +def test_subprocess_launcher_passes_explicit_business_date_and_execute( + monkeypatch, + tmp_path: Path, +) -> None: + captured: dict[str, object] = {} + + class Process: + pid = 9090 + + def poll(self): + return None + + def fake_popen(argv, **kwargs): + captured["argv"] = argv + captured["kwargs"] = kwargs + return Process() + + monkeypatch.setattr("gyxx_flow.scheduler_service.subprocess.Popen", fake_popen) + launcher = SubprocessWorkflowLauncher(tmp_path, tmp_path / "var") + process = launcher.launch( + "shop.jd_self_operated.daily", + datetime(2026, 8, 1, 10, 0, tzinfo=SHANGHAI), + date(2026, 7, 31), + ) + launcher.close(process) + + assert captured["argv"][-4:] == [ + "shop.jd_self_operated.daily", + "--date", + "2026-07-31", + "--execute", + ] + assert "--scheduled" not in captured["argv"] diff --git a/tests/test_script_catalog.py b/tests/test_script_catalog.py index 5d933d4..f2cf82f 100644 --- a/tests/test_script_catalog.py +++ b/tests/test_script_catalog.py @@ -10,71 +10,254 @@ from gyxx_flow.cli import EXIT_SUCCESS, main from gyxx_flow.core.config import Settings from gyxx_flow.script_catalog import ScriptCatalog, ScriptCatalogError +PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_script_catalog_discovers_python_main_and_launchers(tmp_path: Path) -> None: + +def _write_registry(path: Path, commands: list[dict[str, object]]) -> None: + path.write_text( + json.dumps({"schema_version": 1, "commands": commands}), + encoding="utf-8", + ) + + +def test_catalog_exposes_only_explicit_commands(tmp_path: Path) -> None: runtime = tmp_path / "content_marketing" / "runtime" (runtime / "tools").mkdir(parents=True) - (runtime / "tools" / "run_job.py").write_text( + registered = runtime / "tools" / "run_job.py" + registered.write_text( 'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8" ) - (runtime / "library.py").write_text("def helper():\n return 1\n", encoding="utf-8") - (runtime / "daily.bat").write_text("@echo off\n", encoding="utf-8") - (runtime / "maint.ps1").write_text("Write-Output ok\n", encoding="utf-8") - (runtime / "runtime_tests").mkdir() - (runtime / "runtime_tests" / "test_old.py").write_text( - 'if __name__ == "__main__":\n pass\n', encoding="utf-8" + (runtime / "undeclared.py").write_text( + 'if __name__ == "__main__":\n raise SystemExit(0)\n', encoding="utf-8" + ) + (runtime / "undeclared.bat").write_text("@echo off\n", encoding="utf-8") + (runtime / "undeclared.ps1").write_text("Write-Output ok\n", encoding="utf-8") + config = tmp_path / "commands.json" + _write_registry( + config, + [ + { + "id": "content.test.run", + "module": "content_marketing", + "entry": "tools/run_job.py", + "kind": "python", + } + ], ) - catalog = ScriptCatalog.discover({"content_marketing": runtime}) + catalog = ScriptCatalog.load(config, roots={"content_marketing": runtime}) - assert catalog.script_ids == ( - "content_marketing:daily.bat", - "content_marketing:maint.ps1", - "content_marketing:tools/run_job.py", + assert catalog.command_ids == ("content.test.run",) + assert catalog.script_ids == ("content_marketing:tools/run_job.py",) + command = catalog.get("content.test.run") + assert command.entry == "tools/run_job.py" + assert catalog.get("content_marketing:tools/run_job.py") is command + with pytest.raises(ScriptCatalogError, match="unknown command"): + catalog.get("content_marketing:undeclared.py") + + +def test_default_registry_covers_every_executable_workflow_entry() -> None: + catalog = ScriptCatalog.load_default() + payload = json.loads( + (PROJECT_ROOT / "config" / "workflows.json").read_text(encoding="utf-8") ) - assert catalog.get("content_marketing:tools/run_job.py").entry == "tools/run_job.py" + workflow_entries: set[str] = set() + scheduled_workflows = 0 + manual_workflows = 0 + for workflow in payload["workflows"]: + assert workflow["trigger"] in {"scheduled", "manual"} + if workflow["trigger"] == "scheduled": + scheduled_workflows += 1 + else: + manual_workflows += 1 + execution = workflow["execution"] + steps = execution.get("steps") + entries = ( + [step["entry"] for step in steps] + if steps is not None + else [execution["entry"]] + ) + for entry in entries: + workflow_entries.add(f"{workflow['module']}:{entry}") - -def test_default_script_catalog_discovers_every_project_owned_runtime() -> None: - catalog = ScriptCatalog.discover_default() - - modules = {item.module for item in catalog.scripts} - assert modules == { + assert scheduled_workflows == 23 + assert manual_workflows == 0 + assert len(workflow_entries) == 29 + assert len(catalog.scripts) == 33 + assert workflow_entries < set(catalog.script_ids) + assert { + "content.mapping.rebuild", + "content.failed.retry", + "shop.douyin_price_appeal", + "shop.jd_self_operated.collect_brand", + "product.backfill.run", + "product.review.orchestrate", + "supply.workflow.run", + } < set(catalog.command_ids) + assert {item.module for item in catalog.scripts} == { "content_marketing", "product_commerce", "shop_intelligence", "supply_chain", } - assert len(catalog.scripts) > 21 + assert all(item.path.is_relative_to(PROJECT_ROOT) for item in catalog.scripts) + assert catalog.get("shop.jd_self_operated.collect_brand").default_args == ( + "--start-date", + "{business_date}", + "--end-date", + "{business_date}", + "--headless", + ) + assert catalog.get("shop.douyin_price_appeal").default_args == ( + "--price-appeal", + "--execute", + ) + assert catalog.get("product.backfill.run").default_args == ( + "--from", + "{business_date}", + "--to", + "{business_date}", + ) + assert catalog.get("product.review.orchestrate").default_args == ( + "--target-date", + "{business_date}", + ) + assert catalog.get("supply.workflow.run").default_args == ( + "mcp-run", + "purchase-order-update", + ) -@pytest.mark.parametrize("script_id", ("missing", "content_marketing:../run.py")) -def test_script_catalog_rejects_unknown_or_unsafe_ids( - tmp_path: Path, script_id: str +@pytest.mark.parametrize( + ("record", "message"), + ( + ( + { + "id": "content.bad.path", + "module": "content_marketing", + "entry": "../outside.py", + "kind": "python", + }, + "unsafe command entry", + ), + ( + { + "id": "content.bad.kind", + "module": "content_marketing", + "entry": "job.py", + "kind": "batch", + }, + "kind does not match", + ), + ( + { + "id": "Content Bad", + "module": "content_marketing", + "entry": "job.py", + "kind": "python", + }, + "invalid command id", + ), + ( + { + "id": "content.bad.args", + "module": "content_marketing", + "entry": "job.py", + "kind": "python", + "default_args": "--unsafe-shape", + }, + "invalid command default_args", + ), + ), +) +def test_catalog_rejects_unsafe_or_invalid_declarations( + tmp_path: Path, record: dict[str, object], message: str ) -> None: - runtime = tmp_path / "content_marketing" / "runtime" - runtime.mkdir(parents=True) - catalog = ScriptCatalog.discover({"content_marketing": runtime}) + runtime = tmp_path / "runtime" + runtime.mkdir() + (runtime / "job.py").write_text("print('ok')\n", encoding="utf-8") + config = tmp_path / "commands.json" + _write_registry(config, [record]) - with pytest.raises(ScriptCatalogError): - catalog.get(script_id) + with pytest.raises(ScriptCatalogError, match=message): + ScriptCatalog.load(config, roots={"content_marketing": runtime}) -def test_cli_lists_and_dry_runs_project_owned_scripts(tmp_path: Path) -> None: +def test_catalog_rejects_duplicate_command_ids(tmp_path: Path) -> None: + runtime = tmp_path / "runtime" + runtime.mkdir() + for name in ("first.py", "second.py"): + (runtime / name).write_text("print('ok')\n", encoding="utf-8") + config = tmp_path / "commands.json" + _write_registry( + config, + [ + { + "id": "content.test.run", + "module": "content_marketing", + "entry": "first.py", + "kind": "python", + }, + { + "id": "content.test.run", + "module": "content_marketing", + "entry": "second.py", + "kind": "python", + }, + ], + ) + + with pytest.raises(ScriptCatalogError, match="duplicate command id"): + ScriptCatalog.load(config, roots={"content_marketing": runtime}) + + +def test_cli_lists_and_dry_runs_registered_commands(tmp_path: Path) -> None: output = io.StringIO() - settings = Settings(project_root=Path(__file__).parents[1], data_root=tmp_path) + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path) assert main(["scripts", "list", "--json"], settings=settings, stdout=output) == 0 rows = json.loads(output.getvalue()) - assert len(rows) > 21 + assert len(rows) == 33 assert {row["module"] for row in rows} == { "content_marketing", "product_commerce", "shop_intelligence", "supply_chain", } + assert { + "id", + "command_id", + "legacy_script_id", + "module", + "entry", + "kind", + "default_args", + } <= rows[0].keys() output = io.StringIO() + exit_code = main( + [ + "scripts", + "run", + "content.metrics.collect_collaborators", + "--date", + "2026-07-27", + ], + settings=settings, + stdout=output, + ) + payload = json.loads(output.getvalue()) + assert exit_code == EXIT_SUCCESS + assert payload["dry_run"] is True + assert payload["command_id"] == "content.metrics.collect_collaborators" + assert payload["script_id"] == "content_marketing:run_all.py" + assert payload["steps"] == {"module_script": "skipped"} + + +def test_cli_accepts_legacy_script_id_alias(tmp_path: Path) -> None: + output = io.StringIO() + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path) + exit_code = main( [ "scripts", @@ -86,8 +269,63 @@ def test_cli_lists_and_dry_runs_project_owned_scripts(tmp_path: Path) -> None: settings=settings, stdout=output, ) + payload = json.loads(output.getvalue()) assert exit_code == EXIT_SUCCESS - assert payload["dry_run"] is True + assert payload["command_id"] == "content.metrics.collect_collaborators" assert payload["script_id"] == "content_marketing:run_all.py" - assert payload["steps"] == {"module_script": "skipped"} + + +def test_cli_script_effect_identity_includes_arguments(tmp_path: Path) -> None: + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path) + + def dry_run(*extra: str) -> dict[str, object]: + output = io.StringIO() + exit_code = main( + [ + "scripts", + "run", + "content.marketing_report.generate", + "--date", + "2026-07-27", + *extra, + ], + settings=settings, + stdout=output, + ) + assert exit_code == EXIT_SUCCESS + return json.loads(output.getvalue()) + + prompt_only = dry_run("--arg=--prompt-only") + prompt_only_again = dry_run("--arg=--prompt-only") + generate = dry_run() + + assert prompt_only["workflow_id"] == prompt_only_again["workflow_id"] + assert prompt_only["workflow_id"] != generate["workflow_id"] + + +def test_cli_renders_command_business_date_defaults_into_effect_identity( + tmp_path: Path, +) -> None: + settings = Settings(project_root=PROJECT_ROOT, data_root=tmp_path) + + def dry_run(business_date: str) -> dict[str, object]: + output = io.StringIO() + exit_code = main( + [ + "scripts", + "run", + "product.backfill.run", + "--date", + business_date, + ], + settings=settings, + stdout=output, + ) + assert exit_code == EXIT_SUCCESS + return json.loads(output.getvalue()) + + first = dry_run("2026-07-27") + second = dry_run("2026-07-28") + + assert first["workflow_id"] != second["workflow_id"] diff --git a/tests/test_security_scanner.py b/tests/test_security_scanner.py index 31087e8..bc1e435 100644 --- a/tests/test_security_scanner.py +++ b/tests/test_security_scanner.py @@ -34,7 +34,10 @@ def test_scan_repository_reports_only_location_and_rule(tmp_path): assert set(asdict(findings[0])) == {"file", "line", "rule"} -@pytest.mark.parametrize("directory", ["var", ".git", ".venv", ".learnings"]) +@pytest.mark.parametrize( + "directory", + ["var", ".git", ".venv", ".learnings", "build", "dist", "demo.egg-info"], +) def test_scan_repository_excludes_runtime_and_tool_directories(tmp_path, directory): from gyxx_flow.security.scanner import scan_repository @@ -106,6 +109,39 @@ def test_scan_repository_ignores_declarations_references_and_comparisons(tmp_pat assert scan_repository(tmp_path) == [] +def test_scan_repository_ignores_runtime_secret_subscript_references(tmp_path): + from gyxx_flow.security.scanner import scan_repository + + source = tmp_path / "src" / "runtime.py" + source.parent.mkdir() + sensitive_name = "pass" + "word" + token_name = "access_" + "token" + source.write_text( + f'{sensitive_name} = runtime_config["{sensitive_name}"]\n' + f"{token_name} = settings.{token_name}\n", + encoding="utf-8", + ) + + assert scan_repository(tmp_path) == [] + + +def test_scan_repository_detects_bare_identifier_shaped_plaintext(tmp_path): + from gyxx_flow.security.scanner import scan_repository + + source = tmp_path / "config" / "application.env" + source.parent.mkdir() + credential_name = "DATABASE_" + "PASSWORD" + credential_value = "sword" + "fish123" + source.write_text( + f"{credential_name}={credential_value}\n", + encoding="utf-8", + ) + + assert [(item.line, item.rule) for item in scan_repository(tmp_path)] == [ + (1, "plaintext-credential") + ] + + def test_scan_repository_rejects_a_non_directory_root(tmp_path): from gyxx_flow.security.scanner import scan_repository @@ -131,3 +167,58 @@ def test_scan_repository_detects_multiline_environment_secret_default(tmp_path): assert [(item.line, item.rule) for item in findings] == [ (1, "hardcoded-long-hex-credential") ] + + +def test_scan_repository_detects_plaintext_environment_fallback(tmp_path): + from gyxx_flow.security.scanner import scan_repository + + password_name = "SYCM_" + "PASSWORD" + fallback = "not-a-real-" + "browser-secret-31c8" + source = tmp_path / "collector.py" + source.write_text( + f'{password_name} = os.getenv("{password_name}", "{fallback}")\n', + encoding="utf-8", + ) + + findings = scan_repository(tmp_path) + + assert [(item.line, item.rule) for item in findings] == [ + (1, "plaintext-environment-fallback") + ] + + +def test_scan_repository_detects_identifier_shaped_environment_fallback(tmp_path): + from gyxx_flow.security.scanner import scan_repository + + credential_name = "PASS" + "WORD" + fallback = "sword" + "fish123" + source = tmp_path / "collector.py" + source.write_text( + f'{credential_name} = os.getenv("{credential_name}", "{fallback}")\n', + encoding="utf-8", + ) + + findings = scan_repository(tmp_path) + + assert [(item.line, item.rule) for item in findings] == [ + (1, "plaintext-environment-fallback") + ] + + +def test_scan_repository_detects_multiline_plaintext_environment_fallback(tmp_path): + from gyxx_flow.security.scanner import scan_repository + + credential_name = "PASS" + "WORD" + fallback = "sword" + "fish123" + source = tmp_path / "collector.py" + source.write_text( + f'{credential_name} = os.getenv(\n "{credential_name}",\n' + f' "{fallback}",\n)\n', + encoding="utf-8", + ) + + findings = scan_repository(tmp_path) + + assert [(item.line, item.rule) for item in findings] == [ + (1, "plaintext-environment-fallback") + ] diff --git a/tests/test_server_scheduler_deployment.py b/tests/test_server_scheduler_deployment.py new file mode 100644 index 0000000..01f447a --- /dev/null +++ b/tests/test_server_scheduler_deployment.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def test_linux_service_supervises_only_the_python_scheduler() -> None: + unit = (PROJECT_ROOT / "deploy" / "gyxx-flow.service").read_text( + encoding="utf-8" + ) + folded = unit.casefold() + + assert ( + "ExecStart=/opt/gyxx-flow/.venv/bin/python -m gyxx_flow schedule run" in unit + ) + assert "Environment=GYXX_DATA_ROOT=/var/lib/gyxx-flow" in unit + assert "Restart=on-failure" in unit + assert "OnCalendar=" not in unit + assert "schtasks" not in folded + assert "cron" not in folded + + +def test_legacy_windows_service_is_only_a_process_supervisor() -> None: + installer = (PROJECT_ROOT / "deploy" / "windows-service" / "install.ps1").read_text( + encoding="utf-8" + ) + folded = installer.casefold() + + assert "schedule run" in installer + assert "nssm" in folded + assert "schtasks" not in folded + assert "new-scheduledtask" not in folded + assert "schedule plan" not in folded + + +def test_deployment_documents_one_scheduler_and_external_data_root() -> None: + guide = (PROJECT_ROOT / "docs" / "deployment.md").read_text(encoding="utf-8") + payload = json.loads( + (PROJECT_ROOT / "config" / "schedules.json").read_text(encoding="utf-8") + ) + + assert "config/schedules.json" in guide + assert "python -m gyxx_flow schedule run" in guide + assert "deploy/gyxx-flow.service" in guide + assert "GYXX_DATA_ROOT" in guide + assert "/var/lib/gyxx-flow" in guide + assert "Windows Task Scheduler" in guide + assert len(payload["schedules"]) == 23 + assert sum(item.get("enabled", True) for item in payload["schedules"]) == 23 + + +def test_three_canonical_documents_cover_architecture_deployment_and_operations() -> None: + architecture = (PROJECT_ROOT / "docs" / "architecture.md").read_text( + encoding="utf-8" + ) + deployment = (PROJECT_ROOT / "docs" / "deployment.md").read_text( + encoding="utf-8" + ) + runbook = (PROJECT_ROOT / "docs" / "runbook.md").read_text(encoding="utf-8") + + assert "LangGraph" in architecture + assert "systemd" in deployment + assert "故障处理" in runbook + + +def test_postgres_compose_exposes_only_a_loopback_local_database() -> None: + compose = (PROJECT_ROOT / "deploy" / "postgres.compose.yml").read_text( + encoding="utf-8" + ) + + assert '"127.0.0.1:5432:5432"' in compose + assert "POSTGRES_DB: gyxx_super_data" in compose + assert "POSTGRES_USER: gyxx_flow" in compose + assert "GYXX_POSTGRES_PASSWORD" in compose + assert "8.148.185.119" not in compose + assert compose.count("/docker-entrypoint-initdb.d/") == 5 + + content_schema = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "content_marketing" + / "data" + / "tools" + / "schema_gyxx_super_data.sql" + ).read_text(encoding="utf-8") + assert "\\c " not in content_schema diff --git a/tests/test_shop_intelligence_module.py b/tests/test_shop_intelligence_module.py index a30246b..1335e17 100644 --- a/tests/test_shop_intelligence_module.py +++ b/tests/test_shop_intelligence_module.py @@ -14,7 +14,7 @@ from gyxx_flow.modules.shop_intelligence import ( PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_shop_module_registers_both_weekly_workflows() -> None: +def test_shop_module_registers_all_catalog_workflows() -> None: module = ShopIntelligenceModule.from_catalog( WorkflowCatalog.load(PROJECT_ROOT / "config") ) @@ -28,11 +28,76 @@ def test_shop_workflows_use_project_owned_module_commands() -> None: ) for definition in module.workflow_definitions(): - step = definition.steps[0] - assert step.step_id == "module_collect" - assert isinstance(step.action, DeferredModuleCommandStep) - assert step.timeout_seconds == COLLECTION_TIMEOUT_SECONDS == 4 * 60 * 60 - assert step.max_attempts == 1 - assert step.resources == (SHOP_RESOURCE,) == ("module:shop_intelligence",) - assert step.production_sink is True + assert definition.steps + for step in definition.steps: + assert isinstance(step.action, DeferredModuleCommandStep) + if definition.workflow_id == "shop.jd_self_operated.daily": + expected_timeout = {"brand": 600, "product": 1_800}[step.step_id] + elif definition.workflow_id == "shop.douyin_price_appeal": + expected_timeout = 1_800 + else: + expected_timeout = COLLECTION_TIMEOUT_SECONDS + assert step.timeout_seconds == expected_timeout + assert step.max_attempts == 1 + assert step.resources == (f"{SHOP_RESOURCE}:{step.step_id}",) + assert step.production_sink is True + +def test_shop_metrics_platforms_are_independent_graph_steps() -> None: + module = ShopIntelligenceModule.from_catalog( + WorkflowCatalog.load(PROJECT_ROOT / "config") + ) + metrics = next( + item + for item in module.workflow_definitions() + if item.workflow_id == "shop.metrics.weekly" + ) + + assert tuple(step.step_id for step in metrics.steps) == ("jd", "dy", "tm") + assert all(not step.depends_on for step in metrics.steps) + + +def test_douyin_price_appeal_reuses_weekly_browser_resource() -> None: + module = ShopIntelligenceModule.from_catalog( + WorkflowCatalog.load(PROJECT_ROOT / "config") + ) + definitions = {item.workflow_id: item for item in module.workflow_definitions()} + metrics_douyin = next( + step for step in definitions["shop.metrics.weekly"].steps if step.step_id == "dy" + ) + appeal = definitions["shop.douyin_price_appeal"].steps[0] + + assert appeal.step_id == "dy" + assert appeal.action.command_entry == ( + "collectors/dy_store_competitor_store_scraping.py" + ) + assert appeal.action.command_args == ("--price-appeal", "--execute") + assert appeal.resources == metrics_douyin.resources == ( + f"{SHOP_RESOURCE}:dy", + ) + assert appeal.replay_policy == "idempotent" + + +def test_jd_self_operated_preserves_source_runner_graph_contract() -> None: + module = ShopIntelligenceModule.from_catalog( + WorkflowCatalog.load(PROJECT_ROOT / "config") + ) + workflow = next( + item + for item in module.workflow_definitions() + if item.workflow_id == "shop.jd_self_operated.daily" + ) + brand, product = workflow.steps + + assert (brand.step_id, product.step_id) == ("brand", "product") + assert brand.action.command_entry == ( + "collectors/jd_self_operated_brand_daily.py" + ) + assert product.action.command_entry == ( + "collectors/jd_self_operated_product_daily.py" + ) + assert brand.timeout_seconds == 600 + assert product.timeout_seconds == 1_800 + assert brand.depends_on == () + assert product.depends_on == ("brand",) + assert product.run_after_failure is True diff --git a/tests/test_shop_supply_data_routing.py b/tests/test_shop_supply_data_routing.py index 9685274..ffaa59f 100644 --- a/tests/test_shop_supply_data_routing.py +++ b/tests/test_shop_supply_data_routing.py @@ -21,7 +21,7 @@ def _read(relative: str) -> str: def _load_shop_paths(monkeypatch: pytest.MonkeyPatch, data_root: Path): monkeypatch.setenv("GYXX_DATA_ROOT", str(data_root)) - path = MODULES_ROOT / "shop_intelligence" / "runtime" / "paths.py" + path = MODULES_ROOT / "shop_intelligence" / "paths.py" name = "_gyxx_test_shop_routing_paths" spec = importlib.util.spec_from_file_location(name, path) assert spec and spec.loader @@ -56,10 +56,10 @@ def test_shop_output_resolver_confines_collectors_to_raw_root( def test_shop_debug_and_cli_outputs_use_canonical_roots() -> None: jd_shop = _read( - "src/gyxx_flow/modules/shop_intelligence/runtime/collectors/jd_data_collector.py" + "src/gyxx_flow/modules/shop_intelligence/collectors/jd_data_collector.py" ) jd_peer = _read( - "src/gyxx_flow/modules/shop_intelligence/runtime/collectors/" + "src/gyxx_flow/modules/shop_intelligence/collectors/" "jd_peer_store_data_collector.py" ) @@ -71,28 +71,28 @@ def test_shop_debug_and_cli_outputs_use_canonical_roots() -> None: def test_supply_python_outputs_use_state_raw_tmp_and_export_layers() -> None: feishu = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/feishu_sheets.py" + "src/gyxx_flow/modules/supply_chain/orchestrator/feishu_sheets.py" ) workflow = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/mcp_workflow.py" + "src/gyxx_flow/modules/supply_chain/orchestrator/mcp_workflow.py" ) monitor = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/monitor.py" + "src/gyxx_flow/modules/supply_chain/orchestrator/monitor.py" ) replenishment = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "ProductReplenishment.py" ) confirmation = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "PurchaseConfirmation.py" ) batch = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "batch_process.py" ) bitable = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "insert_replenishment_bitable.py" ) @@ -115,7 +115,7 @@ def test_supply_python_outputs_use_state_raw_tmp_and_export_layers() -> None: def test_supply_raw_outputs_are_run_scoped_and_never_cleaned_in_place() -> None: workflow = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/mcp_workflow.py" + "src/gyxx_flow/modules/supply_chain/orchestrator/mcp_workflow.py" ) for script in ( "collect_confirmation.ps1", @@ -123,7 +123,7 @@ def test_supply_raw_outputs_are_run_scoped_and_never_cleaned_in_place() -> None: "collect_replenishment.ps1", ): source = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + script + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" + script ) assert '"run_id=$workflowId"' in source assert ( @@ -155,11 +155,11 @@ def test_supply_raw_outputs_are_run_scoped_and_never_cleaned_in_place() -> None: def test_purchase_order_trigger_uses_state_task_and_never_cleans_raw_or_source() -> None: trigger = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "trigger_purchase_order_update.py" ) collector = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" "collect_purchase_order_update.ps1" ) @@ -179,7 +179,7 @@ def test_purchase_order_trigger_uses_state_task_and_never_cleans_raw_or_source() ) def test_supply_powershell_prefers_injected_module_roots(script: str) -> None: source = _read( - "src/gyxx_flow/modules/supply_chain/runtime/orchestrator/scripts/" + script + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" + script ) for variable in ( @@ -238,3 +238,37 @@ def test_native_adapter_injects_supply_module_roots(tmp_path: Path) -> None: assert command.env["GYXX_SUPPLY_WORK_ROOT"] == str( (module_root / "tmp" / "supply_chain").resolve() ) + + +def test_replenishment_producer_hands_off_current_xlsx_before_archiving() -> None: + producer = _read( + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" + "ProductReplenishment.py" + ) + consumer = _read( + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/batch_process.py" + ) + + assert "filepath = os.path.join(work_dir, filename)" in producer + assert "archive_path = os.path.join(export_dir, filename)" in producer + assert "shutil.copy2(filepath, archive_path)" in producer + assert "DATA_DIR = WORK_ROOT" in consumer + assert 'DATA_DIR.glob("补货建议_*.xlsx")' in consumer + + +def test_discontinued_sku_fallback_uses_portable_state_reference() -> None: + paths = _read("src/gyxx_flow/modules/supply_chain/paths.py") + producer = _read( + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/" + "ProductReplenishment.py" + ) + consumer = _read( + "src/gyxx_flow/modules/supply_chain/orchestrator/scripts/batch_process.py" + ) + + assert 'REFERENCE_ROOT = STATE_ROOT / "reference"' in paths + assert 'else REFERENCE_ROOT / "停产产品.xlsx"' in paths + assert "GYXX_SUPPLY_DISCONTINUED_SKUS_FILE" in paths + assert "file_path = str(DISCONTINUED_PRODUCTS_FILE)" in producer + assert "file_path = DISCONTINUED_PRODUCTS_FILE" in consumer + assert "Downloads" not in paths diff --git a/tests/test_shop_supply_source_snapshot.py b/tests/test_shop_supply_source_snapshot.py index 9dcee4e..86389c4 100644 --- a/tests/test_shop_supply_source_snapshot.py +++ b/tests/test_shop_supply_source_snapshot.py @@ -31,6 +31,7 @@ EXPECTED = { "adaptive_selectors.py", "collectors/jd_data_collector.py", "collectors/jd_peer_store_data_collector.py", + "collectors/jd_self_operated_brand_daily.py", "collectors/dy_store_competitor_store_scraping.py", "collectors/taobao_sycm.py", "runners/run_shop.py", @@ -39,9 +40,8 @@ EXPECTED = { "writers/peer_store_writer.py", "db/db.py", "db/schema.sql", - "scripts/setup_scheduler.ps1", }, - "classifications": {"scheduled", "internal", "maintenance"}, + "classifications": {"scheduled", "manual", "internal"}, }, "supply_chain": { "minimum_files": 37, @@ -100,13 +100,12 @@ def test_source_manifest_covers_runtime_sources_and_entrypoints(module: str) -> target = PurePosixPath(item["target_relative_path"]) assert not target.is_absolute() assert ".." not in target.parts - assert target.parts[:5] == ( - "src", - "gyxx_flow", - "modules", - module, - "runtime", - ) + if item["category"] == "test": + assert target.parts[:3] == ("tests", "modules", module) + elif target.parts[0] == "src": + assert target.parts[:4] == ("src", "gyxx_flow", "modules", module) + else: + assert target.parts[:2] == ("docs", "history") assert SHA256.fullmatch(item["source_sha256"]) assert SHA256.fullmatch(item["target_sha256"]) assert item["transformed"] is ( @@ -125,9 +124,7 @@ def test_source_manifest_covers_runtime_sources_and_entrypoints(module: str) -> def test_runtime_files_match_manifest_and_are_decoupled(module: str) -> None: manifest_path = MANIFEST_ROOT / f"{module}.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - runtime_root = ( - PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / module / "runtime" - ) + runtime_root = PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / module for item in manifest["files"]: target = PROJECT_ROOT.joinpath( @@ -166,15 +163,18 @@ def test_source_manifest_explicitly_excludes_secrets_and_runtime_state( "**/data/**", "**/logs/**", } <= excluded + if module == "shop_intelligence": + assert { + "scripts/setup_scheduler.ps1", + "scripts/remove_scheduler.ps1", + } <= excluded @pytest.mark.parametrize("module", sorted(EXPECTED)) def test_runtime_has_no_legacy_absolute_paths_or_sys_path_bootstrap( module: str, ) -> None: - runtime_root = ( - PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / module / "runtime" - ) + runtime_root = PROJECT_ROOT / "src" / "gyxx_flow" / "modules" / module for path in runtime_root.rglob("*"): if not path.is_file() or b"\x00" in path.read_bytes(): continue @@ -191,11 +191,10 @@ def test_supply_backfill_reads_only_from_portable_shared_data_root() -> None: / "gyxx_flow" / "modules" / "supply_chain" - / "runtime" / "orchestrator" / "sql" / "backfill_history.py" ).read_text(encoding="utf-8-sig") - assert "from gyxx_flow.modules.supply_chain.runtime.paths import SHARED_ROOT" in backfill + assert "from gyxx_flow.modules.supply_chain.paths import SHARED_ROOT" in backfill assert "ROOT = SHARED_ROOT" in backfill diff --git a/tests/test_source_migration_acceptance.py b/tests/test_source_migration_acceptance.py index f5aa266..e55b341 100644 --- a/tests/test_source_migration_acceptance.py +++ b/tests/test_source_migration_acceptance.py @@ -52,11 +52,11 @@ def test_every_workflow_and_manual_script_resolves_inside_new_project() -> None: / "gyxx_flow" / "modules" / workflow.module - / "runtime" / workflow.entry ).resolve(strict=True) assert target.is_relative_to(PROJECT_ROOT) - scripts = ScriptCatalog.discover_default() - assert len(scripts.scripts) >= 100 - assert all(script.path.is_relative_to(PROJECT_ROOT) for script in scripts.scripts) + commands = ScriptCatalog.load_default() + assert len(commands.scripts) == 33 + assert len(commands.command_ids) == len(set(commands.command_ids)) + assert all(command.path.is_relative_to(PROJECT_ROOT) for command in commands.scripts) diff --git a/tests/test_source_sync.py b/tests/test_source_sync.py new file mode 100644 index 0000000..69fc7f0 --- /dev/null +++ b/tests/test_source_sync.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from gyxx_flow.source_sync import SourceSyncService, SyncState + + +def _digest(content: str) -> str: + return hashlib.sha256(content.encode()).hexdigest() + + +def _write_project( + root: Path, + *, + source_content: str | None = "baseline\n", + target_content: str | None = "baseline\n", + baseline_source: str = "baseline\n", + baseline_target: str = "baseline\n", + transformed: bool = False, + source_relative_path: str = "job.py", + target_relative_path: str = "src/runtime/job.py", +) -> tuple[Path, Path, Path]: + source_root = root / "upstream" + repository_root = root / "repository" + manifest_path = repository_root / "config" / "source-manifests" / "demo.json" + config_path = repository_root / "config" / "source-projects.json" + + source_root.mkdir(parents=True, exist_ok=True) + source_relative = Path(source_relative_path) + source_path_is_safe = ( + not source_relative.is_absolute() and ".." not in source_relative.parts + ) + target_relative = Path(target_relative_path) + target_path_is_safe = ( + not target_relative.is_absolute() and ".." not in target_relative.parts + ) + if source_content is not None and source_path_is_safe: + source_path = source_root / source_relative_path + source_path.parent.mkdir(parents=True, exist_ok=True) + source_path.write_text(source_content, encoding="utf-8", newline="") + if target_content is not None and target_path_is_safe: + target_path = repository_root / target_relative_path + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(target_content, encoding="utf-8", newline="") + + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text( + json.dumps( + { + "schema_version": 1, + "module": "demo", + "files": [ + { + "source_relative_path": source_relative_path, + "target_relative_path": target_relative_path, + "source_sha256": _digest(baseline_source), + "target_sha256": _digest(baseline_target), + "transformed": transformed, + "category": "source", + } + ], + } + ), + encoding="utf-8", + ) + config_path.write_text( + json.dumps( + { + "schema_version": 1, + "projects": [ + { + "module": "demo", + "manifest": "config/source-manifests/demo.json", + "root_env": "GYXX_SOURCE_DEMO_ROOT", + "target_root": "src/runtime", + } + ], + } + ), + encoding="utf-8", + ) + return repository_root, source_root, config_path + + +@pytest.mark.parametrize( + ("source_content", "target_content", "expected"), + [ + ("baseline\n", "baseline\n", SyncState.IN_SYNC), + ("upstream fix\n", "baseline\n", SyncState.SOURCE_CHANGED), + ("baseline\n", "local fix\n", SyncState.TARGET_CHANGED), + ("upstream fix\n", "local fix\n", SyncState.CONFLICT), + (None, "baseline\n", SyncState.SOURCE_MISSING), + ("baseline\n", None, SyncState.TARGET_MISSING), + ], +) +def test_status_uses_manifest_as_three_way_hash_baseline( + tmp_path: Path, + source_content: str | None, + target_content: str | None, + expected: SyncState, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_content=source_content, + target_content=target_content, + ) + + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + + comparison = service.status("demo")[0] + assert comparison.state is expected + assert comparison.safe_to_apply is ( + expected is SyncState.SOURCE_CHANGED + ) + + +def test_status_resolves_source_root_from_environment_without_writing( + tmp_path: Path, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_content="upstream fix\n", + ) + manifest_before = (repository_root / "config/source-manifests/demo.json").read_bytes() + target_before = (repository_root / "src/runtime/job.py").read_bytes() + + service = SourceSyncService( + repository_root, + config_path=config_path, + environ={"GYXX_SOURCE_DEMO_ROOT": str(source_root)}, + ) + + assert service.status("demo")[0].state is SyncState.SOURCE_CHANGED + assert (repository_root / "config/source-manifests/demo.json").read_bytes() == manifest_before + assert (repository_root / "src/runtime/job.py").read_bytes() == target_before + + +def test_apply_safe_copies_source_and_atomically_refreshes_manifest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_content="upstream fix\n", + ) + manifest_path = repository_root / "config/source-manifests/demo.json" + replace_calls: list[tuple[Path, Path]] = [] + + import gyxx_flow.source_sync.service as service_module + + real_replace = service_module.os.replace + + def recording_replace(source: str | Path, target: str | Path) -> None: + replace_calls.append((Path(source), Path(target))) + real_replace(source, target) + + monkeypatch.setattr(service_module.os, "replace", recording_replace) + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + + result = service.apply_safe("demo") + + assert result.applied == ("src/runtime/job.py",) + assert result.skipped == () + assert (repository_root / "src/runtime/job.py").read_text( + encoding="utf-8" + ) == "upstream fix\n" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = manifest["files"][0] + assert entry["source_sha256"] == _digest("upstream fix\n") + assert entry["target_sha256"] == _digest("upstream fix\n") + assert any(target == manifest_path for _, target in replace_calls) + assert not list(manifest_path.parent.glob("*.tmp")) + + +@pytest.mark.parametrize( + ("transformed", "target_content", "expected_state"), + [ + (True, "baseline\n", SyncState.SOURCE_CHANGED), + (False, "local fix\n", SyncState.CONFLICT), + ], +) +def test_apply_safe_does_not_overwrite_transformed_or_conflicting_targets( + tmp_path: Path, + transformed: bool, + target_content: str, + expected_state: SyncState, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_content="upstream fix\n", + target_content=target_content, + transformed=transformed, + ) + target_path = repository_root / "src/runtime/job.py" + manifest_path = repository_root / "config/source-manifests/demo.json" + manifest_before = manifest_path.read_bytes() + + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + result = service.apply_safe("demo") + + assert result.applied == () + assert result.skipped == ("src/runtime/job.py",) + assert service.status("demo")[0].state is expected_state + assert target_path.read_text(encoding="utf-8") == target_content + assert manifest_path.read_bytes() == manifest_before + + +@pytest.mark.parametrize( + ("source_relative_path", "target_relative_path"), + [ + ("../outside.py", "src/runtime/job.py"), + ("job.py", "../outside.py"), + ("C:/outside.py", "src/runtime/job.py"), + ("job.py", "C:/outside.py"), + ], +) +def test_rejects_absolute_paths_and_parent_traversal( + tmp_path: Path, + source_relative_path: str, + target_relative_path: str, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_relative_path=source_relative_path, + target_relative_path=target_relative_path, + ) + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + + with pytest.raises(ValueError, match="unsafe relative path"): + service.status("demo") + + +@pytest.mark.parametrize( + "source_relative_path", + [".env", "credentials.json", "browser/cookies.json", "private.pem"], +) +def test_rejects_sensitive_source_files( + tmp_path: Path, + source_relative_path: str, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_relative_path=source_relative_path, + ) + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + + with pytest.raises(ValueError, match="sensitive"): + service.status("demo") + + +def test_status_reports_unmapped_source_files_but_not_excluded_or_sensitive_files( + tmp_path: Path, +) -> None: + repository_root, source_root, config_path = _write_project(tmp_path) + manifest_path = repository_root / "config/source-manifests/demo.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["intentionally_excluded"] = [ + {"pattern": "tmp_*.py", "reason": "temporary diagnostics"}, + { + "source_relative_path": "inspect_manual.py", + "reason": "manual diagnostic", + }, + ] + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + (source_root / "new_job.py").write_text("print('new')\n", encoding="utf-8") + (source_root / "tmp_probe.py").write_text("temporary\n", encoding="utf-8") + (source_root / "inspect_manual.py").write_text("manual\n", encoding="utf-8") + (source_root / ".env").write_text("TOKEN=secret\n", encoding="utf-8") + cache = source_root / "__pycache__" + cache.mkdir() + (cache / "job.pyc").write_bytes(b"cache") + root_profile = source_root / ".chrome_profile" + root_profile.mkdir() + (root_profile / "session.txt").write_text("browser state\n", encoding="utf-8") + root_logs = source_root / "logs" + root_logs.mkdir() + (root_logs / "run.log").write_text("generated log\n", encoding="utf-8") + + comparisons = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ).status("demo") + + assert [item.source_relative_path for item in comparisons] == [ + "job.py", + "new_job.py", + ] + candidate = comparisons[1] + assert candidate.state is SyncState.NEW_SOURCE + assert candidate.target_relative_path == "src/runtime/new_job.py" + assert candidate.safe_to_apply is False + + +def test_apply_safe_rolls_back_targets_when_manifest_commit_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, source_root, config_path = _write_project( + tmp_path, + source_content="upstream fix\n", + ) + target_path = repository_root / "src/runtime/job.py" + manifest_path = repository_root / "config/source-manifests/demo.json" + original_target = target_path.read_bytes() + original_manifest = manifest_path.read_bytes() + + import gyxx_flow.source_sync.service as service_module + + def fail_manifest_commit(path: Path, payload: object) -> None: + del path, payload + raise OSError("injected manifest failure") + + monkeypatch.setattr( + service_module, + "_atomic_write_json", + fail_manifest_commit, + ) + service = SourceSyncService( + repository_root, + config_path=config_path, + source_roots={"demo": source_root}, + ) + + with pytest.raises(OSError, match="injected manifest failure"): + service.apply_safe("demo") + + assert target_path.read_bytes() == original_target + assert manifest_path.read_bytes() == original_manifest + assert not list(repository_root.rglob("*.tmp")) diff --git a/tests/test_source_sync_cli.py b/tests/test_source_sync_cli.py new file mode 100644 index 0000000..e57e728 --- /dev/null +++ b/tests/test_source_sync_cli.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import hashlib +import io +import json +from pathlib import Path + +from gyxx_flow.cli import EXIT_CONFIGURATION, EXIT_SUCCESS, main +from gyxx_flow.core.config import Settings +from gyxx_flow.source_sync import SourceSyncService + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _project(tmp_path: Path) -> tuple[Settings, Path, Path]: + repository = tmp_path / "repository" + source = tmp_path / "source" + target = repository / "src" / "module" / "job.py" + manifest_path = repository / "config" / "source-manifests" / "module.json" + source_file = source / "job.py" + source_file.parent.mkdir(parents=True) + target.parent.mkdir(parents=True) + manifest_path.parent.mkdir(parents=True) + old = b"old\n" + new = b"new\n" + source_file.write_bytes(new) + target.write_bytes(old) + manifest_path.write_text( + json.dumps( + { + "schema_version": 1, + "module": "module", + "source_project": "legacy", + "files": [ + { + "source_relative_path": "job.py", + "target_relative_path": "src/module/job.py", + "category": "source", + "source_sha256": _sha256(old), + "target_sha256": _sha256(old), + "transformed": False, + } + ], + "intentionally_excluded": [], + "snapshot": "test", + } + ), + encoding="utf-8", + ) + (repository / "config" / "source-projects.json").write_text( + json.dumps( + { + "schema_version": 1, + "projects": [ + { + "module": "module", + "source_project": "legacy", + "manifest": "config/source-manifests/module.json", + "root_env": "MODULE_SOURCE_ROOT", + "target_root": "src/module", + } + ], + } + ), + encoding="utf-8", + ) + return Settings(project_root=repository, data_root=tmp_path / "data"), source, target + + +def test_sources_status_is_read_only_and_does_not_build_workflow_registry( + tmp_path: Path, +) -> None: + settings, source, target = _project(tmp_path) + output = io.StringIO() + + exit_code = main( + [ + "sources", + "status", + "--project", + "module", + "--source-root", + f"module={source}", + "--json", + ], + settings=settings, + stdout=output, + ) + + assert exit_code == EXIT_SUCCESS + assert target.read_bytes() == b"old\n" + payload = json.loads(output.getvalue()) + assert payload["summary"] == {"source_changed": 1} + assert payload["files"][0]["state"] == "source_changed" + + +def test_sources_apply_requires_execute_before_writing(tmp_path: Path) -> None: + settings, source, target = _project(tmp_path) + errors = io.StringIO() + + exit_code = main( + [ + "sources", + "apply", + "--project", + "module", + "--source-root", + f"module={source}", + ], + settings=settings, + stderr=errors, + ) + + assert exit_code == EXIT_CONFIGURATION + assert "--execute" in errors.getvalue() + assert target.read_bytes() == b"old\n" + + +def test_sources_apply_copies_only_after_explicit_execute(tmp_path: Path) -> None: + settings, source, target = _project(tmp_path) + output = io.StringIO() + + exit_code = main( + [ + "sources", + "apply", + "--project", + "module", + "--source-root", + f"module={source}", + "--execute", + "--json", + ], + settings=settings, + stdout=output, + ) + + assert exit_code == EXIT_SUCCESS + assert target.read_bytes() == b"new\n" + payload = json.loads(output.getvalue()) + assert payload["applied"] == ["src/module/job.py"] + + +def test_sources_status_reports_new_unmapped_source_candidate(tmp_path: Path) -> None: + settings, source, target = _project(tmp_path) + del target + (source / "new_job.py").write_text("new candidate\n", encoding="utf-8") + output = io.StringIO() + + exit_code = main( + [ + "sources", + "status", + "--project", + "module", + "--source-root", + f"module={source}", + "--json", + ], + settings=settings, + stdout=output, + ) + + assert exit_code == EXIT_SUCCESS + payload = json.loads(output.getvalue()) + assert payload["summary"] == {"new_source": 1, "source_changed": 1} + candidate = next( + item for item in payload["files"] if item["state"] == "new_source" + ) + assert candidate["source_relative_path"] == "new_job.py" + assert candidate["safe_to_apply"] is False + + +def test_repository_source_project_config_declares_migration_target_roots() -> None: + repository = Path(__file__).parents[1] + service = SourceSyncService( + repository, + source_roots={ + "content_marketing": repository, + "product_commerce": repository, + "shop_intelligence": repository, + "supply_chain": repository, + }, + ) + + assert { + project.module: project.target_root + for project in service._projects + } == { + "content_marketing": "src/gyxx_flow/modules/content_marketing", + "product_commerce": ( + "src/gyxx_flow/modules/product_commerce" + ), + "shop_intelligence": ( + "src/gyxx_flow/modules/shop_intelligence" + ), + "supply_chain": "src/gyxx_flow/modules/supply_chain", + } diff --git a/tests/test_supply_acceptance_isolation.py b/tests/test_supply_acceptance_isolation.py new file mode 100644 index 0000000..3215a85 --- /dev/null +++ b/tests/test_supply_acceptance_isolation.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +import pytest + +from gyxx_flow.adapters import WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID + + +@pytest.fixture +def acceptance_environment( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Path: + evidence = tmp_path / "evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_FEISHU_TABLE_WRITE_DISABLED", "1") + monkeypatch.setenv("GYXX_COOKIE_INVALID_SKIP", "1") + monkeypatch.setenv( + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID", + WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID, + ) + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + return evidence + + +def test_replenishment_bitable_run_is_physically_skipped( + tmp_path: Path, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = importlib.import_module( + "gyxx_flow.modules.supply_chain.orchestrator.scripts." + "insert_replenishment_bitable" + ) + pending = tmp_path / "pending_insert.json" + pending.write_text( + json.dumps( + { + "pending_count": 2, + "records": [ + {"商品编号": "SKU1"}, + {"商品编号": "SKU2"}, + ], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + monkeypatch.setattr( + module, + "_search_record_id_by_sku", + lambda *_args, **_kwargs: pytest.fail("must not access Feishu"), + ) + + result = module.run(pending) + + assert result == { + "inserted": 0, + "updated": 0, + "skipped": 2, + "record_ids": [], + "errors": [], + "skipped_by_policy": True, + } + assert "feishu_write_skipped" in acceptance_environment.read_text( + encoding="utf-8" + ) + + +def test_supply_card_sender_forces_only_wang_yunlong( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = importlib.import_module( + "gyxx_flow.modules.supply_chain.orchestrator.scripts." + "send_card_notification" + ) + sent: list[str] = [] + monkeypatch.setattr(module, "_get_tenant_token", lambda: "test-token") + monkeypatch.setattr( + module, + "_send_card", + lambda _token, open_id, _card: sent.append(open_id) or "mid-1", + ) + + result = module.run(["ou_other", "oc_group"], {"elements": []}) + + assert sent == [WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID] + assert result == {"message_ids": ["mid-1"], "errors": []} + + +def test_supply_config_uses_the_two_local_hermes_api_servers( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "COLLECTOR_API_SERVER_URL", + "http://127.0.0.1:8643/v1/chat/completions", + ) + monkeypatch.setenv( + "ANALYZER_API_SERVER_URL", + "http://127.0.0.1:8642/v1/chat/completions", + ) + module = importlib.import_module( + "gyxx_flow.modules.supply_chain.orchestrator.config" + ) + module = importlib.reload(module) + + assert module.HERMES_GATEWAYS["collector"]["url"] == "http://127.0.0.1:8643/v1" + assert module.HERMES_GATEWAYS["collector"]["model"] == "data-collector" + assert module.HERMES_GATEWAYS["analyzer"]["url"] == "http://127.0.0.1:8642/v1" + assert module.HERMES_GATEWAYS["analyzer"]["model"] == "data-analyzer" + for targets in module.FEISHU_CONFIG["notification_targets"].values(): + assert targets == [WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID] + + +def test_purchase_order_update_is_blocked_before_erp_mutation( + acceptance_environment: Path, +) -> None: + module = importlib.import_module( + "gyxx_flow.modules.supply_chain.orchestrator.mcp_workflow" + ) + module = importlib.reload(module) + + result = module.run_mcp_workflow( + "purchase-order-update", + "acceptance-purchase-order-update", + ) + + assert result["status"] == "blocked_business_mutation" + assert "approved test task/SKU fixture" in result["error"] + assert "business_mutation_blocked" in acceptance_environment.read_text( + encoding="utf-8" + ) diff --git a/tests/test_supply_chain_module.py b/tests/test_supply_chain_module.py index 60e3789..03ac446 100644 --- a/tests/test_supply_chain_module.py +++ b/tests/test_supply_chain_module.py @@ -5,8 +5,6 @@ from pathlib import Path from gyxx_flow.adapters.native import DeferredModuleCommandStep from gyxx_flow.catalog import WorkflowCatalog from gyxx_flow.modules.supply_chain import ( - MANUAL_SUPPLY_WORKFLOW_IDS, - SCHEDULED_SUPPLY_WORKFLOW_IDS, SUPPLY_RESOURCE, SUPPLY_TIMEOUT_SECONDS, SUPPLY_WORKFLOW_IDS, @@ -16,14 +14,13 @@ from gyxx_flow.modules.supply_chain import ( PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_supply_module_registers_scheduled_and_manual_workflows() -> None: +def test_supply_module_registers_only_scheduled_workflows() -> None: module = SupplyChainModule.from_catalog( WorkflowCatalog.load(PROJECT_ROOT / "config") ) assert tuple(item.workflow_id for item in module.workflow_definitions()) == SUPPLY_WORKFLOW_IDS - assert len(SCHEDULED_SUPPLY_WORKFLOW_IDS) == 3 - assert len(MANUAL_SUPPLY_WORKFLOW_IDS) == 1 + assert len(SUPPLY_WORKFLOW_IDS) == 3 def test_supply_workflows_use_project_owned_module_commands() -> None: @@ -32,11 +29,10 @@ def test_supply_workflows_use_project_owned_module_commands() -> None: ) for definition in module.workflow_definitions(): - step = definition.steps[0] - assert step.step_id == "module_supply" - assert isinstance(step.action, DeferredModuleCommandStep) - assert step.timeout_seconds == SUPPLY_TIMEOUT_SECONDS == 2 * 60 * 60 - assert step.max_attempts == 1 - assert step.resources == (SUPPLY_RESOURCE,) == ("module:supply_chain",) - assert step.production_sink is True - + assert definition.steps + for step in definition.steps: + assert isinstance(step.action, DeferredModuleCommandStep) + assert step.timeout_seconds == SUPPLY_TIMEOUT_SECONDS == 2 * 60 * 60 + assert step.max_attempts == 1 + assert step.resources == (SUPPLY_RESOURCE,) == ("module:supply_chain",) + assert step.production_sink is True diff --git a/tests/test_supply_gateway_acceptance_preflight.py b/tests/test_supply_gateway_acceptance_preflight.py new file mode 100644 index 0000000..76f21f4 --- /dev/null +++ b/tests/test_supply_gateway_acceptance_preflight.py @@ -0,0 +1,723 @@ +from __future__ import annotations + +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from gyxx_flow.adapters import COOKIE_SKIP_EXIT_CODE +from gyxx_flow.modules.supply_chain import run as supply_run +from gyxx_flow.modules.supply_chain.orchestrator import config as supply_config +from gyxx_flow.modules.supply_chain.orchestrator import ( + mcp_workflow, + pg_writer, + runner, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SUPPLY_WRAPPERS = ( + PROJECT_ROOT + / "src" + / "gyxx_flow" + / "modules" + / "supply_chain" + / "orchestrator" + / "scripts" +) + + +@pytest.fixture +def acceptance_environment( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Path: + evidence = tmp_path / "evidence.jsonl" + monkeypatch.setenv("GYXX_WORKFLOW_ACCEPTANCE", "1") + monkeypatch.setenv("GYXX_ACCEPTANCE_EVIDENCE_FILE", str(evidence)) + monkeypatch.setenv("GYXX_DATA_ROOT", str(tmp_path / "data")) + return evidence + + +def _forbidden(message: str): + def fail(*_args, **_kwargs): + pytest.fail(message) + + return fail + + +def test_frontend_supply_run_uses_stable_business_day_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-03") + + assert mcp_workflow._resolve_workflow_id("purchase-confirmation", None) == ( + "purchase-confirmation_20260803" + ) + assert mcp_workflow._resolve_workflow_id( + "purchase-confirmation", "explicit-run" + ) == "explicit-run" + + +def test_invalid_business_date_keeps_unique_run_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026/08/03") + + first = mcp_workflow._resolve_workflow_id("purchase-confirmation", None) + second = mcp_workflow._resolve_workflow_id("purchase-confirmation", None) + + assert first.startswith("purchase-confirmation_") + assert first != second + + +def test_local_hermes_profile_api_key_is_used_without_copying_secret( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + profile = tmp_path / "profiles" / "data-analyzer" + profile.mkdir(parents=True) + (profile / ".env").write_text( + "API_SERVER_PORT=8642\nAPI_SERVER_KEY=local-only-key\n", + encoding="utf-8", + ) + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("GYXX_SUPPLY_HERMES_TOKEN", raising=False) + monkeypatch.delenv("GYXX_HERMES_API_KEY", raising=False) + monkeypatch.delenv("HERMES_API_KEY", raising=False) + + assert supply_config._hermes_profile_api_key("data-analyzer") == ( + "local-only-key" + ) + + +def test_stable_run_manifest_accepts_only_current_attempt_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + run_dir = tmp_path / "purchase-confirmation" / "run_id=stable-run" + run_dir.mkdir(parents=True) + old_file = run_dir / "old.csv" + current_file = run_dir / "current.csv" + old_file.write_text("old", encoding="utf-8") + current_file.write_text("current", encoding="utf-8") + (run_dir / "_run_manifest.json").write_text( + json.dumps( + { + "run_id": "stable-run", + "files": [old_file.name, current_file.name], + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(mcp_workflow, "SHARED_DIR", tmp_path) + + assert mcp_workflow._validate_run_manifest( + "purchase-confirmation", "stable-run", [str(current_file)] + ) == (True, "") + assert set( + mcp_workflow._reusable_collected_files( + "purchase-confirmation", "stable-run" + ) + ) == {str(old_file), str(current_file)} + + +def test_replenishment_batch_uses_stable_business_day_ids( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from gyxx_flow.modules.supply_chain.orchestrator.scripts import batch_process + + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-03") + + assert batch_process._stable_run_id("replenishment") == ( + "replenishment_20260803" + ) + assert batch_process._stable_run_id("replenishment-alert") == ( + "replenishment-alert_20260803" + ) + + monkeypatch.setenv("AUTOFLOW_TARGET_WORKFLOW", "replenishment-alert") + assert batch_process._is_alert_only_target() + + monkeypatch.setenv("AUTOFLOW_TARGET_WORKFLOW", "replenishment") + assert not batch_process._is_alert_only_target() + + +def test_alert_batch_stops_before_replenishment_side_effects( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from gyxx_flow.modules.supply_chain.orchestrator.scripts import batch_process + + threshold_file = tmp_path / "threshold_alert_data.json" + monkeypatch.setenv("GYXX_BUSINESS_DATE", "2026-08-03") + monkeypatch.setenv("AUTOFLOW_TARGET_WORKFLOW", "replenishment-alert") + monkeypatch.setattr(batch_process, "DATA_DIR", tmp_path) + monkeypatch.setattr(batch_process, "THRESHOLD_ALERT_FILE", threshold_file) + monkeypatch.setattr(batch_process, "_PG_IMPORT_OK", False) + monkeypatch.setattr(batch_process, "_run_product_replenishment", lambda: None) + monkeypatch.setattr( + batch_process, + "_ensure_threshold_alert_file", + lambda: {"alert_count": 1, "alerts": [{"sku": "sku-1"}]}, + ) + monkeypatch.setattr( + batch_process, + "_pick_latest_xlsx", + _forbidden("alert workflow must not enter replenishment persistence"), + ) + + assert batch_process.main() == 0 + payload = json.loads(threshold_file.read_text(encoding="utf-8")) + assert payload["run_id"] == "replenishment-alert_20260803" + assert payload["pg_ok"] is False + + +def _forbid_gateway_side_effects(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + mcp_workflow, + "_acquire_workflow_lock", + _forbidden("cookie preflight must run before workflow locking"), + ) + monkeypatch.setattr( + mcp_workflow, + "_send_collector_heartbeat", + _forbidden("cookie preflight must run before Feishu heartbeats"), + ) + monkeypatch.setattr( + mcp_workflow, + "call_hermes", + _forbidden("cookie preflight must run before Hermes"), + ) + monkeypatch.setattr( + mcp_workflow, + "_run_local_collection_script", + _forbidden("cookie preflight must not start a browser collector"), + ) + monkeypatch.setattr( + mcp_workflow.requests, + "Session", + _forbidden("cookie preflight must not create an HTTP session"), + ) + monkeypatch.setattr( + mcp_workflow.requests, + "post", + _forbidden("cookie preflight must not send an HTTP request"), + ) + + +@pytest.mark.parametrize( + ("workflow_name", "expected_script_id"), + [ + ( + "purchase-confirmation", + "supply_chain:orchestrator/scripts/PurchaseConfirmation.py", + ), + ( + "replenishment-alert", + "supply_chain:orchestrator/scripts/ProductReplenishment.py", + ), + ( + "replenishment", + "supply_chain:orchestrator/scripts/ProductReplenishment.py", + ), + ], +) +def test_gateway_skips_missing_cookie_before_all_external_effects( + workflow_name: str, + expected_script_id: str, + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_SUPPLY_ERP_PASSWORD", raising=False) + _forbid_gateway_side_effects(monkeypatch) + + result = mcp_workflow.run_mcp_workflow(workflow_name, "acceptance-run") + + assert result["status"] == "skipped_cookie" + assert result["workflow_id"] == "acceptance-run" + assert result["cookie_preflight"]["script_id"] == expected_script_id + assert result["cookie_preflight"]["status"] == "SKIPPED_COOKIE" + assert result["reason"] == "browser state is missing" + + events = [ + json.loads(line) + for line in acceptance_environment.read_text(encoding="utf-8").splitlines() + ] + gateway_event = next( + item for item in events if item["event"] == "supply_gateway_cookie_skipped" + ) + assert gateway_event["operation"] == f"supply.{workflow_name}.cookie_preflight" + assert gateway_event["details"]["script_id"] == expected_script_id + assert gateway_event["details"]["workflow_id"] == "acceptance-run" + + +def test_profile_without_required_platform_cookie_is_still_skipped( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_SUPPLY_ERP_PASSWORD", raising=False) + binding = mcp_workflow._acceptance_binding_for_workflow("purchase-confirmation") + cookie_db = binding.profile_dir / "Default" / "Network" / "Cookies" + cookie_db.parent.mkdir(parents=True) + with sqlite3.connect(cookie_db) as connection: + connection.execute("CREATE TABLE cookies (expires_utc INTEGER)") + connection.execute( + "INSERT INTO cookies (expires_utc) VALUES (?)", + (999_999_999_999_999_999,), + ) + (binding.profile_dir / "Local State").write_text("{}", encoding="utf-8") + _forbid_gateway_side_effects(monkeypatch) + + result = mcp_workflow.run_mcp_workflow( + "purchase-confirmation", + "nonempty-profile-run", + ) + + assert result["status"] == "skipped_cookie" + assert result["cookie_preflight"]["script_id"].endswith( + "/PurchaseConfirmation.py" + ) + assert result["reason"] == "browser state is missing" + events = [ + json.loads(line) + for line in acceptance_environment.read_text(encoding="utf-8").splitlines() + ] + assert events[-1]["event"] == "supply_gateway_cookie_skipped" + assert events[-1]["details"]["reason"] == "browser state is missing" + + +def test_supply_credentials_allow_preflight_without_cookie_or_live_cdp( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + del acceptance_environment + monkeypatch.setenv("GYXX_SUPPLY_ERP_PASSWORD", "configured-for-test") + lock_calls: list[tuple[str, str]] = [] + + def reject_lock(workflow_name: str, workflow_id: str) -> tuple[bool, str]: + lock_calls.append((workflow_name, workflow_id)) + return False, "stop-after-preflight" + + monkeypatch.setattr(mcp_workflow, "_acquire_workflow_lock", reject_lock) + + result = mcp_workflow.run_mcp_workflow( + "purchase-confirmation", + "credential-fallback-run", + ) + + assert result["status"] == "failed" + assert result["error"] == "stop-after-preflight" + assert lock_calls == [("purchase-confirmation", "credential-fallback-run")] + + +def test_analyzer_facts_include_authoritative_absolute_collected_files( + tmp_path: Path, +) -> None: + collected_file = tmp_path / "purchase_confirmation.csv" + + facts = mcp_workflow._build_analyze_facts( + "purchase-confirmation", + [str(collected_file)], + ) + + assert facts == [ + "AUTHORITATIVE COLLECTED FILES (absolute paths): " + f"{collected_file.resolve()}" + ] + + +def test_close_workflow_browser_terminates_only_bound_cdp_tree( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeProcess: + def __init__( + self, + pid: int, + cmdline: list[str], + children: list[FakeProcess] | None = None, + ) -> None: + self.pid = pid + self.info = {"pid": pid, "cmdline": cmdline} + self._children = children or [] + self.terminated = False + self.killed = False + + def children(self, *, recursive: bool) -> list[FakeProcess]: + assert recursive is True + return self._children + + def terminate(self) -> None: + self.terminated = True + + def kill(self) -> None: + self.killed = True + + child = FakeProcess(102, ["chrome.exe", "--type=renderer"]) + bound = FakeProcess( + 101, + ["chrome.exe", "--remote-debugging-port=22116"], + [child], + ) + unrelated = FakeProcess( + 201, + ["chrome.exe", "--remote-debugging-port=22115"], + ) + waited: list[tuple[int, ...]] = [] + + monkeypatch.setattr( + mcp_workflow, + "_acceptance_binding_for_workflow", + lambda _workflow_name: SimpleNamespace(cdp_port=22116), + ) + monkeypatch.setattr( + mcp_workflow.psutil, + "process_iter", + lambda _attrs: [bound, unrelated], + ) + + def fake_wait_procs(processes, *, timeout: int): + assert timeout == 5 + waited.append(tuple(process.pid for process in processes)) + return list(processes), [] + + monkeypatch.setattr(mcp_workflow.psutil, "wait_procs", fake_wait_procs) + + closed = mcp_workflow._close_workflow_browser("purchase-confirmation") + + assert closed == [101, 102] + assert bound.terminated is True + assert child.terminated is True + assert unrelated.terminated is False + assert unrelated.killed is False + assert waited == [(102, 101)] + + +def test_close_workflow_browser_ignores_other_workflows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + mcp_workflow, + "_acceptance_binding_for_workflow", + _forbidden("unmanaged workflow must not resolve a browser binding"), + ) + + assert mcp_workflow._close_workflow_browser("purchase-order-update") == [] + + +def test_purchase_order_update_block_precedes_cookie_preflight( + acceptance_environment: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + mcp_workflow, + "_acceptance_cookie_preflight", + _forbidden("ERP mutation block must take precedence over cookie inspection"), + ) + monkeypatch.setattr( + mcp_workflow, + "_acquire_workflow_lock", + _forbidden("blocked ERP workflow must not acquire a lock"), + ) + monkeypatch.setattr( + mcp_workflow, + "_send_collector_heartbeat", + _forbidden("blocked ERP workflow must not send a heartbeat"), + ) + + result = mcp_workflow.run_mcp_workflow( + "purchase-order-update", + "blocked-erp-run", + ) + + assert result["status"] == "blocked_business_mutation" + assert "business_mutation_blocked" in acceptance_environment.read_text( + encoding="utf-8" + ) + + +def test_analyzer_prompt_does_not_rely_on_two_hop_acceptance_environment( + acceptance_environment: Path, +) -> None: + prompt = mcp_workflow._build_analyze_prompt("replenishment") + + assert "Hermes 网关不会继承编排进程的验收环境变量" in prompt + assert "不得调用 insert_replenishment_bitable.py" in prompt + assert prompt.rfind("【验收最终覆盖规则】") > prompt.rfind( + "多维表写入统一走 Python 脚本" + ) + assert mcp_workflow.ANALYZER_OWNER_OPEN_ID in prompt + assert "ou_8ee224968aa26a74c7d30ba27fed5eeb" not in prompt + + +@pytest.mark.parametrize( + "response_text", + [ + "没有向业务用户发送任何过程或进度消息，仅投递最终业务通知。", + "未向目标用户推送额外进度更新，只发送最终结果。", + ( + "No additional process/progress messages were sent to users; " + "final business notification delivered." + ), + ], +) +def test_notification_policy_allows_explicitly_negated_process_messages( + response_text: str, +) -> None: + assert not mcp_workflow._contains_process_notification_violation(response_text) + + +@pytest.mark.parametrize( + "response_text", + [ + "已向业务用户发送进度通知。", + "Progress notification sent to the target user.", + ], +) +def test_notification_policy_rejects_process_messages_sent_to_users( + response_text: str, +) -> None: + assert mcp_workflow._contains_process_notification_violation(response_text) + + +def test_notification_receipt_rejects_empty_message_ids() -> None: + with pytest.raises(ValueError, match="non-empty message_id"): + pg_writer.mark_notification_sent("replenishment", "run-1", []) + + +def test_notification_receipt_requires_persisted_message_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pg_writer, "notification_already_sent", lambda *_args: False) + assert not mcp_workflow._notification_receipt_exists("replenishment", "run-1") + + monkeypatch.setattr(pg_writer, "notification_already_sent", lambda *_args: True) + assert mcp_workflow._notification_receipt_exists("replenishment", "run-1") + + +def test_notification_receipt_falls_back_to_zero_row_workflow_batch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeCursor: + def __init__(self) -> None: + self.fetches = iter((None, None, (1,))) + + def __enter__(self): + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, _sql: str, _params: object) -> None: + return None + + def fetchone(self): + return next(self.fetches) + + class FakeConnection: + def __init__(self) -> None: + self.cursor_instance = FakeCursor() + + def cursor(self) -> FakeCursor: + return self.cursor_instance + + def close(self) -> None: + return None + + monkeypatch.setattr(pg_writer, "_conn", FakeConnection) + + assert pg_writer.notification_already_sent( + "purchase-confirmation", "zero-row-run" + ) + + +def test_production_mode_preserves_the_existing_lock_first_flow( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("GYXX_WORKFLOW_ACCEPTANCE", raising=False) + monkeypatch.setattr( + mcp_workflow, + "_acceptance_cookie_preflight", + _forbidden("production mode must not run the acceptance preflight"), + ) + lock_calls: list[tuple[str, str]] = [] + + def reject_lock(workflow_name: str, workflow_id: str) -> tuple[bool, str]: + lock_calls.append((workflow_name, workflow_id)) + return False, "already running" + + monkeypatch.setattr(mcp_workflow, "_acquire_workflow_lock", reject_lock) + + result = mcp_workflow.run_mcp_workflow("replenishment", "production-run") + + assert result["status"] == "failed" + assert result["error"] == "already running" + assert lock_calls == [("replenishment", "production-run")] + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + ("completed", 0), + ("skipped_cookie", COOKIE_SKIP_EXIT_CODE), + ("blocked_business_mutation", runner.BLOCKED_EXIT_CODE), + ("failed", 1), + ], +) +def test_both_supply_clis_propagate_structured_result_exit_codes( + status: str, + expected: int, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + result = {"workflow": "replenishment", "status": status} + monkeypatch.setattr(runner, "run_mcp", lambda _workflow: result) + + monkeypatch.setattr( + sys, + "argv", + ["runner.py", "mcp", "replenishment"], + ) + assert runner.main() == expected + + monkeypatch.setattr( + sys, + "argv", + ["run.py", "mcp-run", "replenishment"], + ) + assert supply_run.main() == expected + capsys.readouterr() + + +def test_batch_exit_code_uses_failure_then_block_then_cookie_precedence() -> None: + assert runner.result_exit_code( + { + "results": [ + {"status": "completed"}, + {"status": "skipped_cookie"}, + ] + } + ) == COOKIE_SKIP_EXIT_CODE + assert runner.result_exit_code( + { + "results": [ + {"status": "skipped_cookie"}, + {"status": "blocked_business_mutation"}, + ] + } + ) == runner.BLOCKED_EXIT_CODE + assert runner.result_exit_code( + { + "results": [ + {"status": "blocked_business_mutation"}, + {"status": "failed"}, + ] + } + ) == 1 + + +@pytest.mark.parametrize( + "script_name", + ["collect_confirmation.ps1", "collect_replenishment.ps1"], +) +def test_direct_collection_wrappers_gate_before_browser_start( + script_name: str, +) -> None: + source = (SUPPLY_WRAPPERS / script_name).read_text(encoding="utf-8-sig") + + acceptance_position = source.index('if ($env:GYXX_WORKFLOW_ACCEPTANCE -eq "1")') + gate_position = source.index("[SKIPPED_COOKIE]") + browser_start_position = source.index("Start-Process -FilePath $chromePath") + assert gate_position < browser_start_position + assert "exit 75" in source[gate_position:browser_start_position] + assert '$forceFreshBrowser = $false' in source[gate_position - 200:browser_start_position] + assert "$env:GYXX_SUPPLY_ERP_PASSWORD" in source[ + acceptance_position:browser_start_position + ] + assert "starting the target-owned profile for credential fallback" in source + + +def test_direct_purchase_order_wrapper_blocks_before_task_and_browser_access() -> None: + source = (SUPPLY_WRAPPERS / "collect_purchase_order_update.ps1").read_text( + encoding="utf-8-sig" + ) + + gate_position = source.index("[BLOCKED_BUSINESS_MUTATION]") + assert gate_position < source.index("$taskFile =") + assert gate_position < source.index("Start-Process -FilePath $chromePath") + assert "exit 77" in source[gate_position:source.index("$scriptDir =")] + + +@pytest.mark.parametrize( + ("script_name", "expected_code", "expected_marker"), + [ + ("collect_confirmation.ps1", COOKIE_SKIP_EXIT_CODE, "SKIPPED_COOKIE"), + ("collect_replenishment.ps1", COOKIE_SKIP_EXIT_CODE, "SKIPPED_COOKIE"), + ( + "collect_purchase_order_update.ps1", + runner.BLOCKED_EXIT_CODE, + "BLOCKED_BUSINESS_MUTATION", + ), + ], +) +def test_direct_wrapper_process_never_calls_start_process_when_gated( + script_name: str, + expected_code: int, + expected_marker: str, + tmp_path: Path, +) -> None: + powershell = shutil.which("powershell.exe") + if powershell is None: + pytest.skip("Windows PowerShell is unavailable") + + target = SUPPLY_WRAPPERS / script_name + escaped_target = str(target).replace("'", "''") + command = "\n".join( + [ + "function global:Get-NetTCPConnection {", + " param([int]$LocalPort, [string]$State)", + " return $null", + "}", + "function global:Start-Process { throw 'START_PROCESS_CALLED' }", + f"& '{escaped_target}'", + "$code = $LASTEXITCODE", + "exit $code", + ] + ) + environment = os.environ.copy() + environment.update( + { + "GYXX_WORKFLOW_ACCEPTANCE": "1", + "GYXX_PROJECT_ROOT": str(PROJECT_ROOT), + "GYXX_DATA_ROOT": str(tmp_path / "data"), + } + ) + + completed = subprocess.run( + [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + command, + ], + cwd=PROJECT_ROOT, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=30, + check=False, + ) + + output = completed.stdout + completed.stderr + assert completed.returncode == expected_code, output + assert expected_marker in output + assert "START_PROCESS_CALLED" not in output diff --git a/tests/test_supply_replenishment_batch.py b/tests/test_supply_replenishment_batch.py new file mode 100644 index 0000000..174506f --- /dev/null +++ b/tests/test_supply_replenishment_batch.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from gyxx_flow.modules.supply_chain.orchestrator.scripts import batch_process + + +def test_batch_only_selects_an_xlsx_from_the_current_collection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(batch_process, "DATA_DIR", tmp_path) + old = tmp_path / "补货建议_旧.xlsx" + current = tmp_path / "补货建议_本次.xlsx" + old.write_bytes(b"old") + current.write_bytes(b"current") + os.utime(old, (100.0, 100.0)) + os.utime(current, (201.0, 201.0)) + + assert batch_process._pick_latest_xlsx(since_epoch=200.0) == current + assert batch_process._pick_latest_xlsx(since_epoch=202.0) is None + + +def test_no_replenishment_is_encoded_as_a_successful_business_result() -> None: + source = ( + Path(batch_process.__file__).with_name("ProductReplenishment.py") + ).read_text(encoding="utf-8-sig") + + assert "completed_without_replenishment = True" in source + assert "return filepath or completed_without_replenishment" in source diff --git a/tests/test_unified_module_data_paths.py b/tests/test_unified_module_data_paths.py index 62e36b2..2c650c1 100644 --- a/tests/test_unified_module_data_paths.py +++ b/tests/test_unified_module_data_paths.py @@ -14,9 +14,9 @@ RUNTIME_ROOT = PROJECT_ROOT / "src" / "gyxx_flow" / "modules" def _load_path_module(module: str, filename: str, monkeypatch, data_root: Path): monkeypatch.setenv("GYXX_DATA_ROOT", str(data_root)) - runtime_root = RUNTIME_ROOT / module / "runtime" - monkeypatch.setenv("GYXX_MODULE_ROOT", str(runtime_root)) - path = runtime_root / filename + module_root = RUNTIME_ROOT / module + monkeypatch.setenv("GYXX_MODULE_ROOT", str(module_root)) + path = module_root / filename name = f"_gyxx_test_paths_{module}" spec = importlib.util.spec_from_file_location(name, path) assert spec and spec.loader @@ -128,9 +128,9 @@ def test_blank_data_root_uses_the_project_var_default( root_name: str, ) -> None: monkeypatch.setenv("GYXX_DATA_ROOT", configured) - runtime_root = RUNTIME_ROOT / module / "runtime" - monkeypatch.setenv("GYXX_MODULE_ROOT", str(runtime_root)) - path = runtime_root / filename + module_root = RUNTIME_ROOT / module + monkeypatch.setenv("GYXX_MODULE_ROOT", str(module_root)) + path = module_root / filename name = f"_gyxx_test_blank_paths_{module}" spec = importlib.util.spec_from_file_location(name, path) assert spec and spec.loader diff --git a/tests/test_workflow_acceptance_policy.py b/tests/test_workflow_acceptance_policy.py new file mode 100644 index 0000000..7c36463 --- /dev/null +++ b/tests/test_workflow_acceptance_policy.py @@ -0,0 +1,416 @@ +from __future__ import annotations + +import json +import multiprocessing +import os +import sqlite3 +import threading +from dataclasses import replace +from pathlib import Path + +import pytest + +import gyxx_flow.adapters.acceptance_policy as acceptance_policy_module +from gyxx_flow.adapters import ( + WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID, + RuntimeIntegrationBinding, + WorkflowAcceptancePolicy, + WorkflowAcceptancePolicyError, +) + + +def _write_acceptance_evidence(path: str, worker: int, count: int) -> None: + environment = { + "GYXX_WORKFLOW_ACCEPTANCE": "1", + "GYXX_ACCEPTANCE_EVIDENCE_FILE": path, + "GYXX_WORKFLOW_ID": "content.metrics.daily", + "GYXX_RUN_ID": f"worker-{worker}", + } + os.environ.update(environment) + policy = WorkflowAcceptancePolicy.from_environment(environment) + for index in range(count): + policy.record( + "feishu_write_skipped", + operation="concurrency-test", + details={"worker": worker, "index": index}, + ) + + +def _binding(tmp_path: Path) -> RuntimeIntegrationBinding: + return RuntimeIntegrationBinding( + script_id="content_marketing:collector.py", + module="content_marketing", + entry="collector.py", + cdp_port=22998, + cdp_url="http://127.0.0.1:22998", + profile_dir=tmp_path / "profile", + cookie_file=tmp_path / "cookies.json", + storage_state_file=tmp_path / "storage_state.json", + ) + + +def _environment(tmp_path: Path) -> dict[str, str]: + return { + "GYXX_WORKFLOW_ACCEPTANCE": "1", + "GYXX_DATA_ROOT": str(tmp_path), + "GYXX_WORKFLOW_ID": "content.metrics.daily", + "GYXX_RUN_ID": "acceptance-run-1", + } + + +def test_disabled_policy_does_not_change_recipients_or_writes(tmp_path: Path) -> None: + policy = WorkflowAcceptancePolicy.from_environment({}) + + assert policy.notification_recipients(("ou_a", "ou_a", "ou_b")) == ( + "ou_a", + "ou_b", + ) + assert policy.skip_feishu_write("base.upsert") is False + assert policy.evidence_file is None + + +def test_acceptance_policy_is_fail_closed_and_forces_wang_yunlong( + tmp_path: Path, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + + assert policy.notification_recipients(("ou_other", "oc_group")) == ( + WORKFLOW_ACCEPTANCE_RECIPIENT_OPEN_ID, + ) + assert policy.environment()["GYXX_FEISHU_TABLE_WRITE_DISABLED"] == "1" + assert policy.environment()["GYXX_COOKIE_INVALID_SKIP"] == "1" + + with pytest.raises(WorkflowAcceptancePolicyError, match="Wang Yunlong"): + WorkflowAcceptancePolicy.from_environment( + { + **_environment(tmp_path), + "GYXX_NOTIFICATION_RECIPIENT_OPEN_ID": "ou_other", + } + ) + with pytest.raises(WorkflowAcceptancePolicyError, match="writes"): + WorkflowAcceptancePolicy.from_environment( + { + **_environment(tmp_path), + "GYXX_FEISHU_TABLE_WRITE_DISABLED": "0", + } + ) + + +def test_feishu_write_skip_appends_run_scoped_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + environment = _environment(tmp_path) + policy = WorkflowAcceptancePolicy.from_environment(environment) + monkeypatch.setenv("GYXX_WORKFLOW_ID", environment["GYXX_WORKFLOW_ID"]) + monkeypatch.setenv("GYXX_RUN_ID", environment["GYXX_RUN_ID"]) + + assert policy.skip_feishu_write( + "bitable.record-upsert", + details={"table": "tbl1", "api_token": "must-not-leak"}, + ) + + assert policy.evidence_file is not None + payload = json.loads(policy.evidence_file.read_text(encoding="utf-8")) + assert payload["event"] == "feishu_write_skipped" + assert payload["workflow_id"] == "content.metrics.daily" + assert payload["run_id"] == "acceptance-run-1" + assert payload["details"]["api_token"] == "" + + +def test_acceptance_evidence_recursively_redacts_nested_secrets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + environment = _environment(tmp_path) + policy = WorkflowAcceptancePolicy.from_environment(environment) + monkeypatch.setenv("GYXX_WORKFLOW_ID", environment["GYXX_WORKFLOW_ID"]) + monkeypatch.setenv("GYXX_RUN_ID", environment["GYXX_RUN_ID"]) + credential_values = {"value": "not-a-real-" + "nested-secret-value"} + details = { + "request": { + "headers": {"access_" + "token": credential_values["value"]}, + "items": [ + {"pass" + "word": credential_values["value"]}, + ("safe", {"secret_" + "key": credential_values["value"]}), + ], + }, + "safe_value": "visible", + } + + safe_details = acceptance_policy_module._safe_details(details) + assert isinstance(safe_details["request"]["items"][1], tuple) + + policy.record( + "nested-redaction-test", + details=details, + ) + + assert policy.evidence_file is not None + text = policy.evidence_file.read_text(encoding="utf-8") + assert credential_values["value"] not in text + payload = json.loads(text) + assert payload["details"] == { + "request": { + "headers": {"access_token": ""}, + "items": [ + {"password": ""}, + ["safe", {"secret_key": ""}], + ], + }, + "safe_value": "visible", + } + + +def test_acceptance_evidence_is_valid_under_multiprocess_writers( + tmp_path: Path, +) -> None: + evidence_file = tmp_path / "concurrent-evidence.jsonl" + context = multiprocessing.get_context("spawn") + workers = [ + context.Process( + target=_write_acceptance_evidence, + args=(str(evidence_file), worker, 25), + ) + for worker in range(4) + ] + + for process in workers: + process.start() + for process in workers: + process.join(timeout=30) + + assert [process.exitcode for process in workers] == [0, 0, 0, 0] + lines = evidence_file.read_text(encoding="utf-8").splitlines() + payloads = [json.loads(line) for line in lines] + assert len(payloads) == 100 + assert { + (payload["details"]["worker"], payload["details"]["index"]) + for payload in payloads + } == {(worker, index) for worker in range(4) for index in range(25)} + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires POSIX fork") +def test_acceptance_evidence_resets_inherited_thread_lock_after_fork( + tmp_path: Path, +) -> None: + evidence_file = tmp_path / "fork-evidence.jsonl" + locked = threading.Event() + release = threading.Event() + + def hold_parent_lock() -> None: + with acceptance_policy_module._EVIDENCE_THREAD_LOCK: + locked.set() + release.wait(timeout=10) + + thread = threading.Thread(target=hold_parent_lock) + thread.start() + assert locked.wait(timeout=2) + context = multiprocessing.get_context("fork") + process = context.Process( + target=_write_acceptance_evidence, + args=(str(evidence_file), 1, 1), + ) + try: + process.start() + process.join(timeout=5) + finally: + release.set() + thread.join(timeout=2) + if process.is_alive(): + process.terminate() + process.join(timeout=2) + + assert process.exitcode == 0 + payload = json.loads(evidence_file.read_text(encoding="utf-8")) + assert payload["run_id"] == "worker-1" + + +def test_acceptance_evidence_thread_lock_has_a_timeout( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + monkeypatch.setattr( + acceptance_policy_module, + "_EVIDENCE_LOCK_TIMEOUT_SECONDS", + 0.05, + ) + lock = acceptance_policy_module._EVIDENCE_THREAD_LOCK + assert lock.acquire(timeout=1) + try: + with pytest.raises(WorkflowAcceptancePolicyError, match="this process"): + policy.record("lock-timeout-test") + finally: + lock.release() + + +def test_cookie_preflight_skips_missing_invalid_and_expired_state( + tmp_path: Path, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + binding = _binding(tmp_path) + + missing = policy.preflight_cookie(binding, now_epoch=100.0) + assert missing.status == "SKIPPED_COOKIE" + assert missing.reason == "browser state is missing" + + binding.cookie_file.write_text("not-json", encoding="utf-8") + invalid = policy.preflight_cookie(binding, now_epoch=100.0) + assert invalid.status == "SKIPPED_COOKIE" + assert invalid.reason == "cookie file is unreadable" + + binding.cookie_file.write_text( + json.dumps([{"name": "session", "value": "x", "expires": 99.0}]), + encoding="utf-8", + ) + expired = policy.preflight_cookie(binding, now_epoch=100.0) + assert expired.status == "SKIPPED_COOKIE" + assert expired.reason == "all cookies are expired" + + +def test_cookie_preflight_accepts_unexpired_or_profile_cookie_state( + tmp_path: Path, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + binding = _binding(tmp_path) + binding.cookie_file.write_text( + json.dumps([{"name": "session", "value": "x", "expires": 101.0}]), + encoding="utf-8", + ) + + assert policy.preflight_cookie(binding, now_epoch=100.0).status == "READY" + + binding.cookie_file.unlink() + binding.profile_dir.mkdir() + (binding.profile_dir / "Local State").write_text("{}", encoding="utf-8") + assert policy.preflight_cookie(binding, now_epoch=100.0).status == "SKIPPED_COOKIE" + + cookie_db = binding.profile_dir / "Default" / "Network" / "Cookies" + cookie_db.parent.mkdir(parents=True) + with sqlite3.connect(cookie_db) as connection: + connection.execute("CREATE TABLE cookies (expires_utc INTEGER NOT NULL)") + connection.execute("INSERT INTO cookies VALUES (?)", (11_644_473_701_000_000,)) + assert policy.preflight_cookie(binding, now_epoch=100.0).status == "READY" + + +@pytest.mark.parametrize( + "payload", + [ + [{"name": "session", "value": "x", "domain": ".example.test"}], + { + "cookies": [ + {"name": "session", "value": "x", "domain": ".example.test"} + ] + }, + { + "cookies": [ + {"name": "session", "value": "x", "domain": ".example.test"} + ], + "origins": [{"origin": "https://example.test", "localStorage": []}], + }, + ], +) +def test_cookie_preflight_supports_cookie_bundles_and_storage_state( + tmp_path: Path, + payload: object, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + binding = replace( + _binding(tmp_path), + required_cookie_domains=("example.test",), + required_cookie_names=("session",), + ) + binding.cookie_file.write_text(json.dumps(payload), encoding="utf-8") + + assert policy.preflight_cookie(binding, now_epoch=100.0).status == "READY" + + +def test_cookie_preflight_rejects_an_unrelated_domain_or_cookie_name( + tmp_path: Path, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + binding = replace( + _binding(tmp_path), + required_cookie_domains=("target.example",), + required_cookie_names=("target_session",), + ) + binding.cookie_file.write_text( + json.dumps( + [ + { + "name": "unrelated_session", + "value": "x", + "domain": ".other.example", + } + ] + ), + encoding="utf-8", + ) + + wrong_domain = policy.preflight_cookie(binding, now_epoch=100.0) + assert wrong_domain.status == "SKIPPED_COOKIE" + assert "platform domain" in wrong_domain.reason + + binding.cookie_file.write_text( + json.dumps( + [ + { + "name": "unrelated_session", + "value": "x", + "domain": ".target.example", + } + ] + ), + encoding="utf-8", + ) + wrong_name = policy.preflight_cookie(binding, now_epoch=100.0) + assert wrong_name.status == "SKIPPED_COOKIE" + assert "login cookie" in wrong_name.reason + + +@pytest.mark.parametrize("login_mode", ["B", "C"]) +def test_credential_capable_modes_do_not_require_cookie_when_credentials_exist( + tmp_path: Path, + login_mode: str, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + account_name = "TARGET_" + "ACCOUNT" + password_name = "TARGET_" + "PASSWORD" + configured_value = "config" + "ured" + binding = replace( + _binding(tmp_path), + login_mode=login_mode, + required_cookie_domains=("target.example",), + credential_env_names=(account_name, password_name), + ) + + missing = policy.preflight_cookie(binding, now_epoch=100.0, environment={}) + assert missing.status == "SKIPPED_COOKIE" + + available = policy.preflight_cookie( + binding, + now_epoch=100.0, + environment={ + account_name: configured_value, + password_name: configured_value, + }, + ) + assert available.status == "READY" + assert available.reason == "credential login fallback is available" + + +def test_interactive_login_mode_is_always_skipped_during_acceptance( + tmp_path: Path, +) -> None: + policy = WorkflowAcceptancePolicy.from_environment(_environment(tmp_path)) + binding = replace(_binding(tmp_path), login_mode="D") + binding.cookie_file.write_text( + json.dumps([{"name": "session", "value": "x"}]), + encoding="utf-8", + ) + + result = policy.preflight_cookie(binding, now_epoch=100.0) + + assert result.status == "SKIPPED_COOKIE" + assert "interactive login" in result.reason diff --git a/tests/test_workflow_engine.py b/tests/test_workflow_engine.py index 82a86e8..5aaf119 100644 --- a/tests/test_workflow_engine.py +++ b/tests/test_workflow_engine.py @@ -1,14 +1,19 @@ from __future__ import annotations import json +import os import subprocess +import sys +import time from collections import deque from datetime import datetime, timezone from pathlib import Path from typing import Any +import psutil import pytest +import gyxx_flow.workflow.steps as steps_module from gyxx_flow.core.context import RunContext from gyxx_flow.core.layout import DataLayout from gyxx_flow.core.locks import LockManager @@ -99,6 +104,8 @@ def test_workflow_validates_relations_and_uses_stable_topological_order() -> Non [ ({"max_attempts": 0}, "max_attempts"), ({"timeout_seconds": 0}, "timeout_seconds"), + ({"timeout_seconds": True}, "timeout_seconds"), + ({"timeout_seconds": "60"}, "timeout_seconds"), ({"retry_delay_seconds": -1}, "retry_delay_seconds"), ({"depends_on": ("one", "one")}, "duplicate dependency"), ], @@ -115,12 +122,20 @@ def test_command_step_uses_argv_explicit_cwd_env_and_timeout( ) -> None: captured: dict[str, object] = {} - def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + class FakeProcess: + pid = 424242 + + def wait(self, timeout: float | None = None) -> int: + captured["timeout"] = timeout + return 0 + + def fake_start(argv: list[str], **kwargs: object) -> tuple[FakeProcess, None]: captured["argv"] = argv captured.update(kwargs) - return subprocess.CompletedProcess(argv, 0, stdout="ok", stderr="") + kwargs["stdout"].write("ok") # type: ignore[union-attr] + return FakeProcess(), None - monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(steps_module, "_start_process", fake_start) step = CommandStep( argv=("C:/Python312/python.exe", "worker.py", "--date", "2026-07-27"), cwd=tmp_path, @@ -140,7 +155,8 @@ def test_command_step_uses_argv_explicit_cwd_env_and_timeout( assert captured["cwd"] == tmp_path assert captured["env"] == {"MODE": "test"} assert captured["timeout"] == 9.5 - assert captured["shell"] is False + assert captured["stdout"].closed is True # type: ignore[union-attr] + assert captured["stderr"].closed is True # type: ignore[union-attr] def test_command_step_dry_run_never_starts_a_process( @@ -149,7 +165,7 @@ def test_command_step_dry_run_never_starts_a_process( def fail_if_called(*args: object, **kwargs: object) -> None: raise AssertionError("subprocess must not be started") - monkeypatch.setattr(subprocess, "run", fail_if_called) + monkeypatch.setattr(steps_module, "_start_process", fail_if_called) step = CommandStep(argv=("tool.exe", "--write"), cwd=tmp_path, env={}) outcome = step.execute(context=_context(), timeout_seconds=None, dry_run=True) @@ -159,19 +175,188 @@ def test_command_step_dry_run_never_starts_a_process( assert outcome.exit_code == 0 +def test_command_step_uses_stdout_as_failure_detail_when_stderr_is_empty( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + class FakeProcess: + pid = 424243 + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 1 + + def fake_start(argv: list[str], **kwargs: object) -> tuple[FakeProcess, None]: + del argv + kwargs["stdout"].write("business workflow failed") # type: ignore[union-attr] + return FakeProcess(), None + + monkeypatch.setattr(steps_module, "_start_process", fake_start) + step = CommandStep(argv=("worker.exe",), cwd=tmp_path, env={}) + + outcome = step.execute(context=_context(), timeout_seconds=None, dry_run=False) + + assert outcome.exit_code == 1 + assert outcome.error == "business workflow failed" + + +def test_command_step_failure_detail_preserves_log_head_and_root_cause_tail( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + class FakeProcess: + pid = 424244 + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 1 + + def fake_start(argv: list[str], **kwargs: object) -> tuple[FakeProcess, None]: + del argv + kwargs["stdout"].write( # type: ignore[union-attr] + "collector startup context\n" + "progress line\n" * 2_000 + ) + kwargs["stderr"].write( # type: ignore[union-attr] + "Traceback: ROOT CAUSE AT LOG TAIL" + ) + return FakeProcess(), None + + monkeypatch.setattr(steps_module, "_start_process", fake_start) + step = CommandStep(argv=("worker.exe",), cwd=tmp_path, env={}) + + outcome = step.execute(context=_context(), timeout_seconds=None, dry_run=False) + + assert outcome.exit_code == 1 + assert outcome.error is not None + assert outcome.error.startswith("collector startup context") + assert "[middle output omitted]" in outcome.error + assert outcome.error.endswith("Traceback: ROOT CAUSE AT LOG TAIL") + assert len(outcome.error) <= 16_000 + + + def test_command_step_reports_timeout_as_a_failed_exit( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - def timeout(*args: object, **kwargs: object) -> None: - raise subprocess.TimeoutExpired(cmd=["tool.exe"], timeout=2) + terminated: list[object] = [] - monkeypatch.setattr(subprocess, "run", timeout) + class TimeoutProcess: + pid = 424242 + + def wait(self, timeout: float | None = None) -> int: + raise subprocess.TimeoutExpired(cmd=["tool.exe"], timeout=timeout) + + def fake_start(*_args: object, **_kwargs: object) -> tuple[TimeoutProcess, None]: + return TimeoutProcess(), None + + monkeypatch.setattr(steps_module, "_start_process", fake_start) + monkeypatch.setattr( + steps_module, + "_terminate_process_tree", + lambda process, scope: terminated.append((process, scope)), + ) step = CommandStep(argv=("tool.exe",), cwd=tmp_path, env={}) outcome = step.execute(context=_context(), timeout_seconds=2, dry_run=False) assert outcome.exit_code == 124 assert "timed out" in (outcome.error or "") + assert len(terminated) == 1 + + +def test_command_step_timeout_terminates_grandchild_before_return( + tmp_path: Path, +) -> None: + identity_file = tmp_path / "grandchild.json" + marker_file = tmp_path / "grandchild-finished.txt" + grandchild_code = ( + "import json,os,pathlib,psutil,time; " + f"pathlib.Path({str(identity_file)!r}).write_text(" + "json.dumps({'pid': os.getpid(), " + "'created': psutil.Process().create_time()}), encoding='utf-8'); " + "time.sleep(1.2); " + f"pathlib.Path({str(marker_file)!r}).write_text('alive', encoding='utf-8')" + ) + parent_code = ( + "import pathlib,subprocess,sys,time; " + f"subprocess.Popen([sys.executable, '-c', {grandchild_code!r}], " + "stdout=sys.stdout, stderr=sys.stderr, close_fds=False); " + f"identity = pathlib.Path({str(identity_file)!r}); " + "deadline = time.monotonic() + 2; " + "\nwhile not identity.exists() and time.monotonic() < deadline: " + "time.sleep(0.01)" + "\ntime.sleep(10)" + ) + step = CommandStep( + argv=(sys.executable, "-c", parent_code), + cwd=tmp_path, + env=dict(os.environ), + ) + + outcome = step.execute(context=_context(), timeout_seconds=0.8, dry_run=False) + + assert outcome.exit_code == 124 + assert identity_file.is_file() + identity = json.loads(identity_file.read_text(encoding="utf-8")) + try: + descendant = psutil.Process(identity["pid"]) + same_process_is_live = ( + descendant.create_time() == pytest.approx(identity["created"], abs=0.01) + and descendant.is_running() + and descendant.status() != psutil.STATUS_ZOMBIE + ) + except psutil.Error: + same_process_is_live = False + assert same_process_is_live is False + time.sleep(1.3) + assert marker_file.exists() is False + + +def test_command_step_does_not_wait_for_grandchild_output_handle( + tmp_path: Path, +) -> None: + child_code = ( + "import subprocess,sys; " + "subprocess.Popen([sys.executable, '-c', " + "'import time; time.sleep(3)'], stdout=sys.stdout, stderr=sys.stderr, " + "close_fds=False); " + "print('parent done')" + ) + step = CommandStep( + argv=(sys.executable, "-c", child_code), + cwd=tmp_path, + env=dict(os.environ), + ) + + outcome = step.execute(context=_context(), timeout_seconds=1, dry_run=False) + + assert outcome.exit_code == 0 + assert "parent done" in outcome.stdout + + +def test_command_step_maps_reserved_cookie_exit_to_a_skip( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + class SkippedProcess: + pid = 424242 + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 75 + + def skipped(*_args: object, **kwargs: object) -> tuple[SkippedProcess, None]: + kwargs["stdout"].write( # type: ignore[union-attr] + "[SKIPPED_COOKIE] state missing" + ) + return SkippedProcess(), None + + monkeypatch.setattr(steps_module, "_start_process", skipped) + step = CommandStep(argv=("collector.exe",), cwd=tmp_path, env={}) + + outcome = step.execute(context=_context(), timeout_seconds=2, dry_run=False) + + assert outcome.exit_code == 75 + assert outcome.skipped is True + assert outcome.reason == "cookie-preflight" def test_engine_retries_records_every_attempt_and_aggregates_failures(tmp_path: Path) -> None: @@ -343,6 +528,71 @@ def test_effect_ledger_skips_a_successful_production_sink_replay(tmp_path: Path) assert len(action.calls) == 1 +def test_idempotent_sink_runs_again_for_same_business_date(tmp_path: Path) -> None: + action = FakeStep(StepExecution(exit_code=0), StepExecution(exit_code=0)) + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition( + "publish", + action, + production_sink=True, + replay_policy="idempotent", + ), + ), + ) + ledger = EffectLedger(tmp_path) + first_context = _context() + second_context = RunContext.create( + "shop.weekly", "2026-07-27", random_suffix="replay3" + ) + engine = WorkflowEngine( + LockManager(tmp_path / "locks"), effect_ledger=ledger + ) + + first = engine.execute( + workflow, + context=first_context, + journal=RunJournal.create(DataLayout(tmp_path), first_context), + ) + second = engine.execute( + workflow, + context=second_context, + journal=RunJournal.create(DataLayout(tmp_path), second_context), + ) + + assert first.status == "success" + assert second.status == "success" + assert second.steps["publish"].reason is None + assert len(action.calls) == 2 + assert list((tmp_path / "state" / "ops" / "effects").glob("*.json")) == [] + + +def test_skipped_production_sink_cancels_effect_claim_for_future_runs( + tmp_path: Path, +) -> None: + action = FakeStep(StepExecution(exit_code=75, skipped=True, reason="cookie-preflight")) + workflow = WorkflowDefinition( + "shop.weekly", + (StepDefinition("collect", action, production_sink=True),), + ) + ledger = EffectLedger(tmp_path) + context = _context() + + result = WorkflowEngine( + LockManager(tmp_path / "locks"), + effect_ledger=ledger, + ).execute( + workflow, + context=context, + journal=RunJournal.create(DataLayout(tmp_path), context), + ) + + assert result.status == "success" + assert result.steps["collect"].status == "skipped" + assert list((tmp_path / "state" / "ops" / "effects").glob("*.json")) == [] + + def test_failed_dependency_skips_downstream_critical_step(tmp_path: Path) -> None: downstream = FakeStep() workflow = WorkflowDefinition( @@ -363,3 +613,37 @@ def test_failed_dependency_skips_downstream_critical_step(tmp_path: Path) -> Non assert result.steps["publish"].status == "skipped" assert result.steps["publish"].reason == "dependency-failed" assert downstream.calls == [] + + +def test_run_after_failure_executes_downstream_but_preserves_workflow_failure( + tmp_path: Path, +) -> None: + product = FakeStep() + workflow = WorkflowDefinition( + "shop.weekly", + ( + StepDefinition( + "brand", + FakeStep(StepExecution(exit_code=3, error="brand failed")), + ), + StepDefinition( + "product", + product, + depends_on=("brand",), + run_after_failure=True, + ), + ), + ) + context = _context() + + result = WorkflowEngine(LockManager(tmp_path / "locks")).execute( + workflow, + context=context, + journal=RunJournal.create(DataLayout(tmp_path), context), + ) + + assert result.status == "failed" + assert result.exit_code == 1 + assert result.steps["brand"].status == "failed" + assert result.steps["product"].status == "success" + assert len(product.calls) == 1 diff --git a/uv.lock b/uv.lock index 3ff853b..67aadce 100644 --- a/uv.lock +++ b/uv.lock @@ -104,6 +104,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/35/ce962f738ae28ffce6293e7607b129075633e6bb185a5ab87e49246eedc2/browserforge-1.2.4-py3-none-any.whl", hash = "sha256:fb1c14e62ac09de221dcfc73074200269f697596c642cb200ceaab1127a17542", size = 37890, upload-time = "2026-02-03T02:52:08.745Z" }, ] +[[package]] +name = "build" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/e0/df5e171f685f82f37b12e1f208064e24244911079d7b767447d1af7e0d70/build-1.5.0.tar.gz", hash = "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647", size = 89796, upload-time = "2026-04-30T03:18:25.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/fe/6bea5c9162869c5beba5d9c8abbed835ec85bf1ec1fba05a3822325c45f3/build-1.5.0-py3-none-any.whl", hash = "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f", size = 26018, upload-time = "2026-04-30T03:18:23.644Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -522,6 +536,13 @@ test = [ { name = "pytest" }, ] +[package.dev-dependencies] +dev = [ + { name = "build" }, + { name = "pytest" }, + { name = "ruff" }, +] + [package.metadata] requires-dist = [ { name = "browserforge", specifier = ">=1.2.4" }, @@ -554,6 +575,13 @@ requires-dist = [ ] provides-extras = ["test"] +[package.metadata.requires-dev] +dev = [ + { name = "build", specifier = ">=1.2" }, + { name = "pytest", specifier = ">=8.0" }, + { name = "ruff", specifier = ">=0.12" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1898,6 +1926,15 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + [[package]] name = "pysocks" version = "1.7.1" @@ -2256,6 +2293,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, ] +[[package]] +name = "ruff" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, + { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, + { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, +] + [[package]] name = "scrapling" version = "0.4.12"