llm_context_builder / scanner.py
pourimoto's picture
Create scanner.py
60a5d0b verified
Raw
History Blame Contribute Delete
1.85 kB
"""
scanner.py
Project scanning
"""
from __future__ import annotations
from pathlib import Path
from filters import (
is_probably_binary,
should_ignore_dir,
should_ignore_file,
)
def scan_files(
root: Path,
extra_ignore_dirs: set[str],
max_size_mb: float,
allowed_extensions: set[str] | None,
) -> list[Path]:
"""
Scan project recursively.
"""
files = []
max_bytes = int(max_size_mb * 1024 * 1024)
for path in root.rglob("*"):
if path.is_dir():
continue
rel_parts = path.relative_to(root).parts[:-1]
if any(
should_ignore_dir(
part,
extra_ignore_dirs,
)
for part in rel_parts
):
continue
if should_ignore_file(
path,
allowed_extensions,
):
continue
try:
if path.stat().st_size > max_bytes:
continue
except OSError:
continue
if is_probably_binary(path):
continue
files.append(path)
files.sort(
key=lambda p: p.relative_to(root).as_posix()
)
return files
def build_tree_text(
root: Path,
files: list[Path],
) -> str:
"""
Render project tree.
"""
if not files:
return "(No files found)"
lines = []
for file in files:
rel = file.relative_to(root)
indent = " " * (len(rel.parts) - 1)
lines.append(
f"{indent}β”œβ”€β”€ {rel.name}"
)
return "\n".join(lines)
def total_project_size(
files: list[Path],
) -> int:
"""
Sum file sizes.
"""
total = 0
for file in files:
try:
total += file.stat().st_size
except OSError:
pass
return total