feat: consolidate legacy workflows into gyxx-flow
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
from .inventory import collect_code_inventory, collect_data_summary
|
||||
from .manifest import build_baseline_manifest, write_manifest_atomic
|
||||
from .models import (
|
||||
DEFAULT_CODE_EXTENSIONS,
|
||||
DataRootSpec,
|
||||
ProjectSpec,
|
||||
ScheduledTask,
|
||||
project_specs_from_env,
|
||||
)
|
||||
from .tasks import (
|
||||
PowerShellScheduledTaskProvider,
|
||||
ScheduledTaskProvider,
|
||||
TaskCountMismatch,
|
||||
collect_scheduled_task_inventory,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_CODE_EXTENSIONS",
|
||||
"DataRootSpec",
|
||||
"ProjectSpec",
|
||||
"ScheduledTask",
|
||||
"PowerShellScheduledTaskProvider",
|
||||
"ScheduledTaskProvider",
|
||||
"TaskCountMismatch",
|
||||
"build_baseline_manifest",
|
||||
"collect_code_inventory",
|
||||
"collect_data_summary",
|
||||
"collect_scheduled_task_inventory",
|
||||
"write_manifest_atomic",
|
||||
"project_specs_from_env",
|
||||
]
|
||||
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
|
||||
from .models import DEFAULT_CODE_EXTENSIONS
|
||||
|
||||
|
||||
class Aggregate(TypedDict):
|
||||
file_count: int
|
||||
total_bytes: int
|
||||
|
||||
|
||||
class DataSummary(Aggregate):
|
||||
by_extension: dict[str, Aggregate]
|
||||
by_top_level: dict[str, Aggregate]
|
||||
|
||||
|
||||
_EXCLUDED_DIRECTORY_NAMES = frozenset(
|
||||
{
|
||||
".cache",
|
||||
".git",
|
||||
".hermes_tmp",
|
||||
".mypy_cache",
|
||||
".pytest_cache",
|
||||
".ruff_cache",
|
||||
".tox",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"cache",
|
||||
"data",
|
||||
"debug",
|
||||
"evidence",
|
||||
"log",
|
||||
"logs",
|
||||
"node_modules",
|
||||
"reports",
|
||||
"runtime",
|
||||
"state",
|
||||
"tmp",
|
||||
"var",
|
||||
}
|
||||
)
|
||||
_SENSITIVE_DIRECTORY_MARKERS = ("cookie", "profile")
|
||||
_SENSITIVE_FILE_NAMES = frozenset(
|
||||
{
|
||||
".env",
|
||||
"credentials.json",
|
||||
"secrets.json",
|
||||
"service-account.json",
|
||||
"service_account.json",
|
||||
}
|
||||
)
|
||||
_SENSITIVE_FILE_MARKERS = ("cookie", "credential", "secret", "token")
|
||||
|
||||
|
||||
def _is_excluded_directory(name: str) -> bool:
|
||||
normalized = name.casefold()
|
||||
return normalized in _EXCLUDED_DIRECTORY_NAMES or any(
|
||||
marker in normalized for marker in _SENSITIVE_DIRECTORY_MARKERS
|
||||
)
|
||||
|
||||
|
||||
def _is_sensitive_file(name: str) -> bool:
|
||||
normalized = name.casefold()
|
||||
if normalized in _SENSITIVE_FILE_NAMES or normalized.startswith(".env."):
|
||||
return True
|
||||
stem = Path(normalized).stem
|
||||
return any(marker in stem for marker in _SENSITIVE_FILE_MARKERS)
|
||||
|
||||
|
||||
def _walk_files(
|
||||
root: Path,
|
||||
*,
|
||||
exclude_runtime: bool,
|
||||
excluded_roots: frozenset[Path] = frozenset(),
|
||||
) -> Iterable[Path]:
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"inventory root is not a directory: {root}")
|
||||
for directory, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
if exclude_runtime:
|
||||
dirnames[:] = sorted(
|
||||
name
|
||||
for name in dirnames
|
||||
if not _is_excluded_directory(name)
|
||||
and not (Path(directory) / name).is_symlink()
|
||||
and (Path(directory) / name).resolve(strict=False) not in excluded_roots
|
||||
)
|
||||
else:
|
||||
dirnames[:] = sorted(
|
||||
name for name in dirnames if not (Path(directory) / name).is_symlink()
|
||||
)
|
||||
for filename in sorted(filenames):
|
||||
path = Path(directory) / filename
|
||||
if not path.is_symlink():
|
||||
yield path
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def collect_code_inventory(
|
||||
root: Path,
|
||||
*,
|
||||
allowed_extensions: frozenset[str] = DEFAULT_CODE_EXTENSIONS,
|
||||
excluded_roots: Iterable[Path] = (),
|
||||
) -> list[dict[str, str | int]]:
|
||||
"""Hash code-like files without retaining file contents or config values."""
|
||||
|
||||
root = Path(root)
|
||||
normalized_extensions = frozenset(extension.casefold() for extension in allowed_extensions)
|
||||
normalized_excluded_roots = frozenset(
|
||||
Path(excluded_root).resolve(strict=False) for excluded_root in excluded_roots
|
||||
)
|
||||
inventory: list[dict[str, str | int]] = []
|
||||
for path in _walk_files(
|
||||
root, exclude_runtime=True, excluded_roots=normalized_excluded_roots
|
||||
):
|
||||
if path.suffix.casefold() not in normalized_extensions or _is_sensitive_file(path.name):
|
||||
continue
|
||||
stat = path.stat()
|
||||
inventory.append(
|
||||
{
|
||||
"relative_path": path.relative_to(root).as_posix(),
|
||||
"size_bytes": stat.st_size,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
)
|
||||
return sorted(inventory, key=lambda item: str(item["relative_path"]).casefold())
|
||||
|
||||
|
||||
def _increment(target: dict[str, Aggregate], key: str, size: int) -> None:
|
||||
aggregate = target.setdefault(key, {"file_count": 0, "total_bytes": 0})
|
||||
aggregate["file_count"] += 1
|
||||
aggregate["total_bytes"] += size
|
||||
|
||||
|
||||
def collect_data_summary(root: Path) -> DataSummary:
|
||||
"""Summarize a data tree without exposing names, paths, times, hashes, or values."""
|
||||
|
||||
root = Path(root)
|
||||
summary: DataSummary = {
|
||||
"file_count": 0,
|
||||
"total_bytes": 0,
|
||||
"by_extension": {},
|
||||
"by_top_level": {},
|
||||
}
|
||||
for path in _walk_files(root, exclude_runtime=False):
|
||||
size = path.stat().st_size
|
||||
relative = path.relative_to(root)
|
||||
top_level = relative.parts[0] if len(relative.parts) > 1 else "."
|
||||
extension = path.suffix.casefold() or "[no_extension]"
|
||||
summary["file_count"] += 1
|
||||
summary["total_bytes"] += size
|
||||
_increment(summary["by_extension"], extension, size)
|
||||
_increment(summary["by_top_level"], top_level, size)
|
||||
summary["by_extension"] = dict(sorted(summary["by_extension"].items()))
|
||||
summary["by_top_level"] = dict(sorted(summary["by_top_level"].items()))
|
||||
return summary
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .inventory import collect_code_inventory, collect_data_summary
|
||||
from .models import ProjectSpec
|
||||
from .tasks import ScheduledTaskProvider, collect_scheduled_task_inventory
|
||||
|
||||
|
||||
def build_baseline_manifest(
|
||||
specs: Sequence[ProjectSpec],
|
||||
task_provider: ScheduledTaskProvider,
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
expected_task_count: int = 21,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a read-only baseline. It never serializes task actions or file contents."""
|
||||
|
||||
timestamp = generated_at or datetime.now(timezone.utc)
|
||||
if timestamp.tzinfo is None or timestamp.utcoffset() is None:
|
||||
raise ValueError("generated_at must be timezone-aware")
|
||||
|
||||
projects: list[dict[str, Any]] = []
|
||||
for spec in specs:
|
||||
projects.append(
|
||||
{
|
||||
"project_id": spec.project_id,
|
||||
"source_root": str(spec.root),
|
||||
"code_inventory": collect_code_inventory(
|
||||
spec.root,
|
||||
allowed_extensions=spec.code_extensions,
|
||||
excluded_roots=(data_root.path for data_root in spec.data_roots),
|
||||
),
|
||||
"data_summaries": [
|
||||
{"label": data_root.label, "summary": collect_data_summary(data_root.path)}
|
||||
for data_root in spec.data_roots
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"generated_at": timestamp.isoformat(),
|
||||
"projects": projects,
|
||||
"scheduled_tasks": collect_scheduled_task_inventory(
|
||||
specs, task_provider, expected_count=expected_task_count
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def write_manifest_atomic(path: Path, manifest: object) -> None:
|
||||
"""Durably replace a JSON manifest without exposing a partial destination file."""
|
||||
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
dir=path.parent,
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
delete=False,
|
||||
) as stream:
|
||||
temporary_path = Path(stream.name)
|
||||
json.dump(manifest, stream, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
temporary_path = None
|
||||
finally:
|
||||
if temporary_path is not None:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Mapping
|
||||
|
||||
DEFAULT_CODE_EXTENSIONS = frozenset(
|
||||
{
|
||||
".bat",
|
||||
".cmd",
|
||||
".json",
|
||||
".md",
|
||||
".ps1",
|
||||
".py",
|
||||
".pyi",
|
||||
".rst",
|
||||
".sh",
|
||||
".sql",
|
||||
".toml",
|
||||
".txt",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DataRootSpec:
|
||||
"""A named data tree whose file-level details must remain private."""
|
||||
|
||||
label: str
|
||||
path: Path
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.label or self.label in {".", ".."}:
|
||||
raise ValueError("data root label must be a non-empty logical name")
|
||||
object.__setattr__(self, "path", Path(self.path))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProjectSpec:
|
||||
"""Read-only discovery boundaries for one legacy project."""
|
||||
|
||||
project_id: str
|
||||
root: Path
|
||||
data_roots: tuple[DataRootSpec, ...] = ()
|
||||
code_extensions: frozenset[str] = field(default=DEFAULT_CODE_EXTENSIONS)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.project_id:
|
||||
raise ValueError("project_id is required")
|
||||
object.__setattr__(self, "root", Path(self.root))
|
||||
object.__setattr__(self, "data_roots", tuple(self.data_roots))
|
||||
normalized_extensions = frozenset(
|
||||
extension.casefold() if extension.startswith(".") else f".{extension.casefold()}"
|
||||
for extension in self.code_extensions
|
||||
)
|
||||
object.__setattr__(self, "code_extensions", normalized_extensions)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScheduledTask:
|
||||
"""Provider-neutral scheduled-task action used only for source matching."""
|
||||
|
||||
task_id: str
|
||||
command: str
|
||||
arguments: str = ""
|
||||
working_directory: str = ""
|
||||
|
||||
|
||||
def project_specs_from_env(
|
||||
env: Mapping[str, str] | None = None,
|
||||
) -> tuple[ProjectSpec, ...]:
|
||||
"""Build legacy discovery boundaries without embedding machine-specific roots."""
|
||||
|
||||
values = os.environ if env is None else env
|
||||
keys = (
|
||||
"GYXX_LEGACY_CONTENT_ROOT",
|
||||
"GYXX_LEGACY_SHOP_ROOT",
|
||||
"GYXX_LEGACY_PRODUCT_ROOT",
|
||||
"GYXX_LEGACY_SUPPLY_ROOT",
|
||||
)
|
||||
missing = [key for key in keys if not values.get(key, "").strip()]
|
||||
if missing:
|
||||
raise ValueError("missing legacy root environment variables: " + ", ".join(missing))
|
||||
|
||||
content = Path(values[keys[0]]).expanduser()
|
||||
shop = Path(values[keys[1]]).expanduser()
|
||||
product = Path(values[keys[2]]).expanduser()
|
||||
supply = Path(values[keys[3]]).expanduser()
|
||||
return (
|
||||
ProjectSpec(
|
||||
"content_marketing",
|
||||
content,
|
||||
(DataRootSpec("data", content / "data"), DataRootSpec("reports", content / "reports")),
|
||||
),
|
||||
ProjectSpec(
|
||||
"shop_intelligence",
|
||||
shop,
|
||||
(DataRootSpec("data", shop / "data"),),
|
||||
),
|
||||
ProjectSpec(
|
||||
"product_commerce",
|
||||
product,
|
||||
(DataRootSpec("data", product / "data"),),
|
||||
),
|
||||
ProjectSpec(
|
||||
"supply_chain",
|
||||
supply,
|
||||
(DataRootSpec("shared-data", supply / "shared-data"),),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Protocol
|
||||
|
||||
from .models import ProjectSpec, ScheduledTask
|
||||
|
||||
|
||||
class ScheduledTaskProvider(Protocol):
|
||||
def scheduled_tasks(self) -> Iterable[ScheduledTask]: ...
|
||||
|
||||
|
||||
class PowerShellScheduledTaskProvider:
|
||||
"""Read Windows Task Scheduler actions without modifying scheduler state."""
|
||||
|
||||
_SCRIPT = r"""
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$rows = foreach ($task in Get-ScheduledTask) {
|
||||
[pscustomobject]@{
|
||||
task_id = "$($task.TaskPath)$($task.TaskName)"
|
||||
command = [string](($task.Actions | ForEach-Object { $_.Execute }) -join ' ')
|
||||
arguments = [string](($task.Actions | ForEach-Object { $_.Arguments }) -join ' ')
|
||||
working_directory = [string](($task.Actions | ForEach-Object { $_.WorkingDirectory }) -join ' ')
|
||||
}
|
||||
}
|
||||
@($rows) | ConvertTo-Json -Depth 3 -Compress
|
||||
"""
|
||||
|
||||
def __init__(self, executable: str = "powershell.exe") -> None:
|
||||
self.executable = executable
|
||||
|
||||
def scheduled_tasks(self) -> list[ScheduledTask]:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
self.executable,
|
||||
"-NoLogo",
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
self._SCRIPT,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="strict",
|
||||
shell=False,
|
||||
)
|
||||
payload = json.loads(completed.stdout or "[]")
|
||||
rows = payload if isinstance(payload, list) else [payload]
|
||||
tasks: list[ScheduledTask] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict) or not isinstance(row.get("task_id"), str):
|
||||
raise ValueError("invalid scheduled-task provider response")
|
||||
tasks.append(
|
||||
ScheduledTask(
|
||||
task_id=row["task_id"],
|
||||
command=str(row.get("command") or ""),
|
||||
arguments=str(row.get("arguments") or ""),
|
||||
working_directory=str(row.get("working_directory") or ""),
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
|
||||
|
||||
class TaskCountMismatch(ValueError):
|
||||
"""The discovered legacy task inventory is not the accepted baseline."""
|
||||
|
||||
|
||||
def _match_project(task: ScheduledTask, specs: Sequence[ProjectSpec]) -> str | None:
|
||||
action = " ".join((task.command, task.arguments, task.working_directory))
|
||||
normalized_action = os.path.normcase(action).replace("/", "\\")
|
||||
for spec in specs:
|
||||
normalized_root = os.path.normcase(str(spec.root)).replace("/", "\\").rstrip("\\")
|
||||
root_pattern = re.escape(normalized_root) + r"(?=$|[\\\s\"'])"
|
||||
if normalized_root and re.search(root_pattern, normalized_action):
|
||||
return spec.project_id
|
||||
return None
|
||||
|
||||
|
||||
def collect_scheduled_task_inventory(
|
||||
specs: Sequence[ProjectSpec],
|
||||
provider: ScheduledTaskProvider,
|
||||
*,
|
||||
expected_count: int = 21,
|
||||
) -> dict[str, object]:
|
||||
"""Keep only legacy-project tasks and deliberately discard action details."""
|
||||
|
||||
tasks: list[dict[str, str]] = []
|
||||
seen_ids: set[str] = set()
|
||||
for task in provider.scheduled_tasks():
|
||||
project_id = _match_project(task, specs)
|
||||
if project_id is None:
|
||||
continue
|
||||
if task.task_id in seen_ids:
|
||||
raise ValueError(f"duplicate scheduled task id: {task.task_id}")
|
||||
seen_ids.add(task.task_id)
|
||||
tasks.append({"task_id": task.task_id, "project_id": project_id})
|
||||
|
||||
tasks.sort(key=lambda item: item["task_id"].casefold())
|
||||
actual_count = len(tasks)
|
||||
if actual_count != expected_count:
|
||||
raise TaskCountMismatch(
|
||||
f"scheduled task baseline expected {expected_count} tasks but found {actual_count}"
|
||||
)
|
||||
return {
|
||||
"expected_count": expected_count,
|
||||
"actual_count": actual_count,
|
||||
"is_complete": True,
|
||||
"tasks": tasks,
|
||||
}
|
||||
Reference in New Issue
Block a user