from __future__ import annotations import hashlib import os from collections.abc import Iterable from pathlib import Path from typing import TypedDict from .models import DEFAULT_CODE_EXTENSIONS class Aggregate(TypedDict): file_count: int total_bytes: int class DataSummary(Aggregate): by_extension: dict[str, Aggregate] by_top_level: dict[str, Aggregate] _EXCLUDED_DIRECTORY_NAMES = frozenset( { ".cache", ".git", ".hermes_tmp", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".tox", ".venv", "__pycache__", "cache", "data", "debug", "evidence", "log", "logs", "node_modules", "reports", "runtime", "state", "tmp", "var", } ) _SENSITIVE_DIRECTORY_MARKERS = ("cookie", "profile") _SENSITIVE_FILE_NAMES = frozenset( { ".env", "credentials.json", "secrets.json", "service-account.json", "service_account.json", } ) _SENSITIVE_FILE_MARKERS = ("cookie", "credential", "secret", "token") def _is_excluded_directory(name: str) -> bool: normalized = name.casefold() return normalized in _EXCLUDED_DIRECTORY_NAMES or any( marker in normalized for marker in _SENSITIVE_DIRECTORY_MARKERS ) def _is_sensitive_file(name: str) -> bool: normalized = name.casefold() if normalized in _SENSITIVE_FILE_NAMES or normalized.startswith(".env."): return True stem = Path(normalized).stem return any(marker in stem for marker in _SENSITIVE_FILE_MARKERS) def _walk_files( root: Path, *, exclude_runtime: bool, excluded_roots: frozenset[Path] = frozenset(), ) -> Iterable[Path]: if not root.is_dir(): raise FileNotFoundError(f"inventory root is not a directory: {root}") for directory, dirnames, filenames in os.walk(root, followlinks=False): if exclude_runtime: dirnames[:] = sorted( name for name in dirnames if not _is_excluded_directory(name) and not (Path(directory) / name).is_symlink() and (Path(directory) / name).resolve(strict=False) not in excluded_roots ) else: dirnames[:] = sorted( name for name in dirnames if not (Path(directory) / name).is_symlink() ) for filename in sorted(filenames): path = Path(directory) / filename if not path.is_symlink(): yield path def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def collect_code_inventory( root: Path, *, allowed_extensions: frozenset[str] = DEFAULT_CODE_EXTENSIONS, excluded_roots: Iterable[Path] = (), ) -> list[dict[str, str | int]]: """Hash code-like files without retaining file contents or config values.""" root = Path(root) normalized_extensions = frozenset(extension.casefold() for extension in allowed_extensions) normalized_excluded_roots = frozenset( Path(excluded_root).resolve(strict=False) for excluded_root in excluded_roots ) inventory: list[dict[str, str | int]] = [] for path in _walk_files( root, exclude_runtime=True, excluded_roots=normalized_excluded_roots ): if path.suffix.casefold() not in normalized_extensions or _is_sensitive_file(path.name): continue stat = path.stat() inventory.append( { "relative_path": path.relative_to(root).as_posix(), "size_bytes": stat.st_size, "mtime_ns": stat.st_mtime_ns, "sha256": _sha256(path), } ) return sorted(inventory, key=lambda item: str(item["relative_path"]).casefold()) def _increment(target: dict[str, Aggregate], key: str, size: int) -> None: aggregate = target.setdefault(key, {"file_count": 0, "total_bytes": 0}) aggregate["file_count"] += 1 aggregate["total_bytes"] += size def collect_data_summary(root: Path) -> DataSummary: """Summarize a data tree without exposing names, paths, times, hashes, or values.""" root = Path(root) summary: DataSummary = { "file_count": 0, "total_bytes": 0, "by_extension": {}, "by_top_level": {}, } for path in _walk_files(root, exclude_runtime=False): size = path.stat().st_size relative = path.relative_to(root) top_level = relative.parts[0] if len(relative.parts) > 1 else "." extension = path.suffix.casefold() or "[no_extension]" summary["file_count"] += 1 summary["total_bytes"] += size _increment(summary["by_extension"], extension, size) _increment(summary["by_top_level"], top_level, size) summary["by_extension"] = dict(sorted(summary["by_extension"].items())) summary["by_top_level"] = dict(sorted(summary["by_top_level"].items())) return summary