wangyunlong/gyxx-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete production workflow migration

Browse Source
This commit is contained in:
wangyunlong
2026-08-06 14:29:57 +08:00
parent 7f215e79c4
commit 8df5266abb
448 changed files with 56937 additions and 14619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_product_source_migration.py
+36 -26
View File
@@ -10,14 +10,15 @@ from pathlib import Path
from gyxx_flow.security import scan_repository
PROJECT_ROOT = Path(__file__).resolve().parents[1]
RUNTIME_ROOT = (
MODULE_ROOT = (
PROJECT_ROOT
/ "src"
/ "gyxx_flow"
/ "modules"
/ "product_commerce"
/ "runtime"
)
MIGRATED_TEST_ROOT = PROJECT_ROOT / "tests" / "modules" / "product_commerce"
HISTORY_ROOT = PROJECT_ROOT / "docs" / "history" / "product-commerce"
MANIFEST_PATH = (
PROJECT_ROOT / "config" / "source-manifests" / "product_commerce.json"
)
@@ -58,6 +59,8 @@ PRODUCTION_PYTHON = {
"main_image_db.py",
"main_image_paths.py",
"market_rank_hermes_notification.py",
"market_rank_limits.py",
"market_rank_product_import.py",
"market_rank_report_archive.py",
"orchestrate_daily_collection.py",
"orchestrate_market_rank_collection.py",
@@ -113,6 +116,8 @@ LEGACY_TESTS = {
"test_jd_market_rank.py",
"test_main_image_concurrency.py",
"test_market_rank_hermes_notification.py",
"test_market_rank_limits.py",
"test_market_rank_product_import.py",
"test_market_rank_report_archive.py",
"test_market_rank_workflow.py",
"test_persona_launcher.py",
@@ -142,6 +147,7 @@ INTENTIONALLY_EXCLUDED = {
"debug_jd_main_image_snapshot.py",
"inspect_dy_comment_filters.py",
"jd_collect_test.py",
"scripts/inspect_fields.py",
}
SUBPROCESS_TARGETS = {
@@ -182,7 +188,7 @@ def _sha256(path: Path) -> str:
def _load_runtime_paths():
path = RUNTIME_ROOT / "runtime_paths.py"
path = MODULE_ROOT / "runtime_paths.py"
spec = importlib.util.spec_from_file_location("product_runtime_paths_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
@@ -191,16 +197,15 @@ def _load_runtime_paths():
def test_product_runtime_contains_the_complete_reviewed_source_snapshot() -> None:
assert len(PRODUCTION_PYTHON) == 63
assert all((RUNTIME_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON)
assert all((RUNTIME_ROOT / relative).is_file() for relative in RESOURCES)
assert len(PRODUCTION_PYTHON) == 65
assert all((MODULE_ROOT / relative).is_file() for relative in PRODUCTION_PYTHON)
assert all((MODULE_ROOT / relative).is_file() for relative in RESOURCES)
assert all((MIGRATED_TEST_ROOT / name).is_file() for name in LEGACY_TESTS)
assert all(
(RUNTIME_ROOT / "runtime_tests" / name).is_file() for name in LEGACY_TESTS
)
assert all(
(RUNTIME_ROOT / "launchers_reference" / name).is_file()
(HISTORY_ROOT / "launchers_reference" / name).is_file()
for name in LAUNCHER_REFERENCES
)
assert (HISTORY_ROOT / "MIGRATION_STATUS.md").is_file()
def test_product_source_manifest_is_complete_and_verifiable() -> None:
@@ -232,19 +237,24 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None:
assert {
item["source_relative_path"]
for item in manifest["intentionally_excluded"]
if "source_relative_path" in item
} == INTENTIONALLY_EXCLUDED
assert all(
item.get("source_relative_path") or item.get("pattern")
for item in manifest["intentionally_excluded"]
)
expected_targets = {
*(
f"src/gyxx_flow/modules/product_commerce/runtime/{path}"
f"src/gyxx_flow/modules/product_commerce/{path}"
for path in PRODUCTION_PYTHON | RESOURCES
),
*(
f"src/gyxx_flow/modules/product_commerce/runtime/runtime_tests/{name}"
f"tests/modules/product_commerce/{name}"
for name in LEGACY_TESTS
),
*(
f"src/gyxx_flow/modules/product_commerce/runtime/launchers_reference/{name}"
f"docs/history/product-commerce/launchers_reference/{name}"
for name in LAUNCHER_REFERENCES
),
}
@@ -270,7 +280,7 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None:
generated = audit["generated_files"]
assert {item["target_relative_path"] for item in generated} == {
"src/gyxx_flow/modules/product_commerce/runtime/runtime_paths.py"
"src/gyxx_flow/modules/product_commerce/runtime_paths.py"
}
for item in generated:
target = PROJECT_ROOT / item["target_relative_path"]
@@ -279,10 +289,10 @@ def test_product_source_manifest_is_complete_and_verifiable() -> None:
def test_migrated_product_snapshot_has_no_plaintext_credentials_or_old_roots() -> None:
assert scan_repository(RUNTIME_ROOT) == []
assert scan_repository(MODULE_ROOT) == []
forbidden = ("E:\\auto-flow", "D:\\product-collector-analyze-flow")
for path in RUNTIME_ROOT.rglob("*"):
for path in MODULE_ROOT.rglob("*"):
if not path.is_file() or path.suffix.casefold() in {".pyc", ".pyo"}:
continue
try:
@@ -293,11 +303,11 @@ def test_migrated_product_snapshot_has_no_plaintext_credentials_or_old_roots() -
def test_product_runtime_paths_are_portable_and_layered(tmp_path, monkeypatch) -> None:
monkeypatch.setenv("GYXX_MODULE_ROOT", str(RUNTIME_ROOT))
monkeypatch.setenv("GYXX_MODULE_ROOT", str(MODULE_ROOT))
monkeypatch.setenv("GYXX_DATA_ROOT", str(tmp_path))
paths = _load_runtime_paths()
assert paths.MODULE_ROOT == RUNTIME_ROOT.resolve()
assert paths.MODULE_ROOT == MODULE_ROOT.resolve()
assert paths.DATA_HOME == tmp_path.resolve()
assert paths.RAW_DATA_ROOT == tmp_path / "data" / "raw" / "product_commerce"
assert paths.NORMALIZED_DATA_ROOT == (
@@ -317,10 +327,10 @@ def test_product_runtime_paths_are_portable_and_layered(tmp_path, monkeypatch) -
assert paths.TMP_ROOT == tmp_path / "tmp" / "product_commerce"
assert paths.vendor_source_root("jd") == (
RUNTIME_ROOT / "vendors" / "jd-data-flow"
MODULE_ROOT / "vendors" / "jd-data-flow"
).resolve()
assert paths.vendor_source_root("dy") == (
RUNTIME_ROOT / "vendors" / "dy-data-flow"
MODULE_ROOT / "vendors" / "dy-data-flow"
).resolve()
for relative in SUBPROCESS_TARGETS:
assert paths.runtime_script(relative).is_file()
@@ -345,7 +355,7 @@ def test_product_runtime_has_no_machine_or_legacy_path_escape_hatches() -> None:
'ROOT / "data"',
'os.path.join(PROJECT_ROOT, "data"',
)
for path in RUNTIME_ROOT.rglob("*"):
for path in MODULE_ROOT.rglob("*"):
if not path.is_file() or path.suffix.casefold() in {".pyc", ".pyo"}:
continue
try:
@@ -359,11 +369,11 @@ def test_product_runtime_has_no_hardcoded_long_hex_keys() -> None:
hardcoded_key = re.compile(
r"(?is)(?:api[_-]?key|secret|token).{0,160}?[\"'][0-9a-f]{32,}[\"']"
)
for path in RUNTIME_ROOT.rglob("*.py"):
for path in MODULE_ROOT.rglob("*.py"):
text = path.read_text(encoding="utf-8-sig")
assert hardcoded_key.search(text) is None, path
analyzer = (RUNTIME_ROOT / "analyze_style_with_hermes.py").read_text(
analyzer = (MODULE_ROOT / "analyze_style_with_hermes.py").read_text(
encoding="utf-8-sig"
)
assert 'os.getenv("HERMES_API_KEY")' in analyzer
@@ -372,7 +382,7 @@ def test_product_runtime_has_no_hardcoded_long_hex_keys() -> None:
def test_product_core_runtime_has_no_known_f821_regressions() -> None:
erp_tree = ast.parse(
(RUNTIME_ROOT / "collect_erp_yesterday_metrics.py").read_text(
(MODULE_ROOT / "collect_erp_yesterday_metrics.py").read_text(
encoding="utf-8-sig"
)
)
@@ -398,8 +408,8 @@ def test_product_core_runtime_has_no_known_f821_regressions() -> None:
self.stored.add(node.id)
jd_paths = (
RUNTIME_ROOT / "jd_product_data_collector.py",
RUNTIME_ROOT / "vendors" / "jd-data-flow" / "jd_product_data_collector.py",
MODULE_ROOT / "jd_product_data_collector.py",
MODULE_ROOT / "vendors" / "jd-data-flow" / "jd_product_data_collector.py",
)
for path in jd_paths:
tree = ast.parse(path.read_text(encoding="utf-8-sig"))