llm_context_builder / statistics.py
pourimoto's picture
Create statistics.py
9c105b7 verified
Raw
History Blame Contribute Delete
1.78 kB
"""
statistics.py
Project statistics
"""
from __future__ import annotations
from pathlib import Path
from utils import (
estimate_tokens,
count_lines,
human_size,
language_from_extension,
)
def build_statistics(files: list[Path], contents: list[tuple[Path, str]]) -> dict:
total_size = 0
total_lines = 0
total_tokens = 0
languages = {}
largest = None
smallest = None
for path, text in contents:
try:
size = path.stat().st_size
except Exception:
size = len(text.encode())
total_size += size
total_lines += count_lines(text)
total_tokens += estimate_tokens(text)
lang = language_from_extension(path.suffix)
languages[lang] = languages.get(lang, 0) + 1
if largest is None or size > largest[1]:
largest = (path.name, size)
if smallest is None or size < smallest[1]:
smallest = (path.name, size)
return {
"files": len(files),
"lines": total_lines,
"tokens": total_tokens,
"size": total_size,
"size_human": human_size(total_size),
"languages": languages,
"largest": largest,
"smallest": smallest,
"average_size": total_size // max(len(files), 1),
}
def statistics_markdown(stats: dict) -> str:
langs = "\n".join(
f"- {k}: {v}"
for k, v in sorted(stats["languages"].items())
)
return f"""
## Project Statistics
Files: {stats["files"]}
Lines: {stats["lines"]}
Approx Tokens: {stats["tokens"]}
Project Size: {stats["size_human"]}
Average File Size: {human_size(stats["average_size"])}
Largest File:
{stats["largest"]}
Smallest File:
{stats["smallest"]}
### Languages
{langs}
""".strip()