feat: complete production workflow migration
This commit is contained in:
@@ -1,17 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import gyxx_flow.core.locks as locks_module
|
||||
from gyxx_flow.core.context import RunContext
|
||||
from gyxx_flow.core.layout import DataLayout
|
||||
from gyxx_flow.core.locks import LockManager, ResourceBusyError
|
||||
from gyxx_flow.core.records import RunJournal
|
||||
|
||||
|
||||
def _crash_while_holding_reclaim_guard(path: str) -> None:
|
||||
with locks_module._exclusive_reclaim_guard(
|
||||
Path(path),
|
||||
resource="module:product",
|
||||
deadline=time.monotonic() + 5,
|
||||
poll_seconds=0.01,
|
||||
):
|
||||
os._exit(0)
|
||||
|
||||
|
||||
def _acquire_shared_resource(root: str, worker: int, queue) -> None: # type: ignore[no-untyped-def]
|
||||
try:
|
||||
with LockManager(Path(root)).acquire(
|
||||
"module:product",
|
||||
owner=f"worker-{worker}",
|
||||
timeout_seconds=10,
|
||||
poll_seconds=0.01,
|
||||
):
|
||||
time.sleep(0.03)
|
||||
except Exception as exc:
|
||||
queue.put((worker, type(exc).__name__))
|
||||
else:
|
||||
queue.put((worker, "ok"))
|
||||
|
||||
|
||||
def _context() -> RunContext:
|
||||
return RunContext.create(
|
||||
"shop.weekly",
|
||||
@@ -89,6 +118,233 @@ def test_named_resource_lock_blocks_concurrent_owner_and_releases(tmp_path: Path
|
||||
assert metadata["owner"] == "run-002"
|
||||
|
||||
|
||||
def test_named_resource_lock_quarantines_provably_dead_owner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
stale = manager.acquire("module:product", owner="run-dead")
|
||||
stale.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"resource": "module:product",
|
||||
"owner": "run-dead",
|
||||
"acquired_at": "2026-08-01T10:00:00+00:00",
|
||||
"pid": 424242,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: False)
|
||||
|
||||
with manager.acquire(
|
||||
"module:product", owner="run-new", timeout_seconds=0
|
||||
) as acquired:
|
||||
metadata = json.loads(acquired.path.read_text(encoding="utf-8"))
|
||||
assert metadata["owner"] == "run-new"
|
||||
|
||||
quarantined = list((tmp_path / "stale").glob("*.lock"))
|
||||
assert len(quarantined) == 1
|
||||
assert json.loads(quarantined[0].read_text(encoding="utf-8"))["owner"] == "run-dead"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("liveness", [True, None])
|
||||
def test_named_resource_lock_keeps_active_or_unknown_owner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
liveness: bool | None,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
occupied = manager.acquire("module:product", owner="run-existing")
|
||||
occupied.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
occupied.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"resource": "module:product",
|
||||
"owner": "run-existing",
|
||||
"acquired_at": "2026-08-01T10:00:00+00:00",
|
||||
"pid": 424242,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: liveness)
|
||||
|
||||
with pytest.raises(ResourceBusyError, match="module:product"):
|
||||
with manager.acquire("module:product", owner="run-new", timeout_seconds=0):
|
||||
pass
|
||||
|
||||
assert occupied.path.exists()
|
||||
|
||||
|
||||
def test_named_resource_lock_does_not_reclaim_recent_malformed_lock(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
occupied = manager.acquire("module:product", owner="run-existing")
|
||||
occupied.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
occupied.path.write_text('{"resource": "module:product"', encoding="utf-8")
|
||||
|
||||
with pytest.raises(ResourceBusyError, match="module:product"):
|
||||
with manager.acquire("module:product", owner="run-new", timeout_seconds=0):
|
||||
pass
|
||||
|
||||
assert occupied.path.exists()
|
||||
|
||||
|
||||
def test_named_resource_lock_quarantines_old_malformed_lock_without_pid(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
occupied = manager.acquire("module:product", owner="run-existing")
|
||||
occupied.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
occupied.path.write_text('{"resource": "module:product"', encoding="utf-8")
|
||||
old = time.time() - locks_module._MALFORMED_LOCK_GRACE_SECONDS - 5
|
||||
os.utime(occupied.path, (old, old))
|
||||
|
||||
with manager.acquire(
|
||||
"module:product", owner="run-new", timeout_seconds=0
|
||||
) as acquired:
|
||||
metadata = json.loads(acquired.path.read_text(encoding="utf-8"))
|
||||
assert metadata["owner"] == "run-new"
|
||||
|
||||
quarantined = list((tmp_path / "stale").glob("*.lock"))
|
||||
assert len(quarantined) == 1
|
||||
|
||||
|
||||
def test_named_resource_lock_keeps_old_malformed_lock_with_active_pid(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
occupied = manager.acquire("module:product", owner="run-existing")
|
||||
occupied.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
occupied.path.write_text(f'{{"pid": {os.getpid()},', encoding="utf-8")
|
||||
old = time.time() - locks_module._MALFORMED_LOCK_GRACE_SECONDS - 5
|
||||
os.utime(occupied.path, (old, old))
|
||||
|
||||
with pytest.raises(ResourceBusyError, match="module:product"):
|
||||
with manager.acquire("module:product", owner="run-new", timeout_seconds=0):
|
||||
pass
|
||||
|
||||
assert occupied.path.exists()
|
||||
|
||||
|
||||
def test_named_resource_lock_uses_process_start_to_detect_pid_reuse(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
occupied = manager.acquire("module:product", owner="run-existing")
|
||||
occupied.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
occupied.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"resource": "module:product",
|
||||
"owner": "run-existing",
|
||||
"acquired_at": "2026-08-01T10:00:00+00:00",
|
||||
"pid": 424242,
|
||||
"process_started_at": 100.0,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(locks_module, "_process_is_running", lambda _pid: True)
|
||||
monkeypatch.setattr(locks_module, "_process_started_at", lambda _pid: 100.0)
|
||||
|
||||
with pytest.raises(ResourceBusyError, match="module:product"):
|
||||
with manager.acquire("module:product", owner="run-new", timeout_seconds=0):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(locks_module, "_process_started_at", lambda _pid: 200.0)
|
||||
with manager.acquire(
|
||||
"module:product", owner="run-new", timeout_seconds=0
|
||||
) as acquired:
|
||||
assert acquired.path.exists()
|
||||
|
||||
|
||||
def test_named_resource_lock_guard_is_released_when_reclaimer_crashes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
lock = manager.acquire("module:product", owner="run-new")
|
||||
guard = lock.path.with_suffix(lock.path.suffix + ".reclaim")
|
||||
context = multiprocessing.get_context("spawn")
|
||||
process = context.Process(
|
||||
target=_crash_while_holding_reclaim_guard,
|
||||
args=(str(guard),),
|
||||
)
|
||||
|
||||
process.start()
|
||||
process.join(timeout=15)
|
||||
|
||||
assert process.exitcode == 0
|
||||
assert guard.exists()
|
||||
with manager.acquire(
|
||||
"module:product", owner="run-new", timeout_seconds=1
|
||||
) as acquired:
|
||||
assert acquired.path.exists()
|
||||
|
||||
|
||||
def test_named_resource_lock_serializes_concurrent_stale_recovery(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
stale = manager.acquire("module:product", owner="run-dead")
|
||||
stale.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale.path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"resource": "module:product",
|
||||
"owner": "run-dead",
|
||||
"acquired_at": "2026-08-01T10:00:00+00:00",
|
||||
"pid": 2_147_483_647,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
context = multiprocessing.get_context("spawn")
|
||||
queue = context.Queue()
|
||||
workers = [
|
||||
context.Process(
|
||||
target=_acquire_shared_resource,
|
||||
args=(str(tmp_path), worker, queue),
|
||||
)
|
||||
for worker in range(4)
|
||||
]
|
||||
|
||||
for process in workers:
|
||||
process.start()
|
||||
for process in workers:
|
||||
process.join(timeout=20)
|
||||
|
||||
assert [process.exitcode for process in workers] == [0, 0, 0, 0]
|
||||
results = sorted(queue.get(timeout=2) for _ in workers)
|
||||
assert results == [(worker, "ok") for worker in range(4)]
|
||||
assert not stale.path.exists()
|
||||
assert len(list((tmp_path / "stale").glob("*.lock"))) == 1
|
||||
|
||||
|
||||
def test_named_resource_lock_never_publishes_partial_metadata(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
lock = manager.acquire("module:product", owner="run-new")
|
||||
|
||||
def fail_link(_source: Path, _target: Path) -> None:
|
||||
raise OSError("injected link failure")
|
||||
|
||||
monkeypatch.setattr(locks_module.os, "link", fail_link)
|
||||
|
||||
with pytest.raises(OSError, match="injected link failure"):
|
||||
with lock:
|
||||
pass
|
||||
|
||||
assert not lock.path.exists()
|
||||
assert list(tmp_path.glob(".*.tmp")) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resource", ["", "../state", "resource\nname"])
|
||||
def test_named_resource_lock_rejects_unsafe_name(tmp_path: Path, resource: str) -> None:
|
||||
manager = LockManager(tmp_path)
|
||||
|
||||
Reference in New Issue
Block a user