File size: 1,641 Bytes
248d017 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | """
utils.py
Common helper functions
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from typing import Iterable
from config import (
CHARS_PER_TOKEN,
LANGUAGE_MAP,
)
def estimate_tokens(text: str) -> int:
"""
Rough token estimation.
"""
return max(1, int(len(text) / CHARS_PER_TOKEN))
def short_sha256(text: str) -> str:
"""
First 12 chars of sha256.
"""
return hashlib.sha256(
text.encode("utf-8", errors="ignore")
).hexdigest()[:12]
def xml_safe(text: str) -> str:
"""
Prevent CDATA termination.
"""
return text.replace("]]>", "]]]]><![CDATA[>")
def language_from_extension(ext: str) -> str:
return LANGUAGE_MAP.get(
ext.lower(),
ext.lower() if ext else "Unknown",
)
def human_size(num_bytes: int) -> str:
value = float(num_bytes)
for unit in ("B", "KB", "MB", "GB", "TB"):
if value < 1024:
return f"{value:.2f} {unit}"
value /= 1024
return f"{value:.2f} PB"
def count_lines(text: str) -> int:
if not text:
return 0
return text.count("\n") + 1
def unique_languages(entries: Iterable[dict]) -> list[str]:
langs = {
language_from_extension(
e.get("ext", "")
)
for e in entries
}
return sorted(langs)
def safe_relative(path: Path, root: Path) -> str:
return path.relative_to(root).as_posix()
def average_file_size(total_size: int, count: int) -> int:
if count == 0:
return 0
return total_size // count
def project_name(root: Path) -> str:
return root.resolve().name |