tools/text_stats.py

# filename: text_stats.py

import string
import unicodedata
import tiktoken
import os

# --------------------
# CONFIG
# --------------------
# --------------------
# CONFIG
# --------------------
import sys
if len(sys.argv) > 1:
    FILE_PATH = sys.argv[1]
else:
    FILE_PATH = r"C:\Users\dcanb\.gemini\antigravity\scratch\wcs3\sessions\capture_20260123_1503_www_vap_org_tr"

N_TOP_BIGGEST = 3
M_TOP_SMALLEST = 3

# Filter settings
INCLUDE_TYPES = [".py", ".json", ".md", ".txt", ".js", ".jsonl"] # List of extensions like [".py", ".json"] or ["all"]
EXCLUDE_TYPES = [".pyc", ".png", ".jpg", ".bin"]      # List of extensions to skip like [".json"]
MIN_FILE_SIZE = "10b"   # e.g. "500b", "1kb", "0b" for no limit
MAX_FILE_SIZE = "100mb"   # Skip files larger than this

# Exclude directories
EXCLUDE_DIRS = {".git", "__pycache__", ".idea", "node_modules", ".vscode", "capture", "captures", "logs", "data", "venv", "env", "global_knowledge"}

# --------------------
# HELPERS
# --------------------
def parse_size(size_str: str) -> int:
    """Parses strings like '1kb', '2mb', '500b' into bytes."""
    size_str = size_str.lower().strip()
    if not size_str or size_str == "0b":
        return 0
    
    units = {"kb": 1024, "mb": 1024**2, "gb": 1024**3, "b": 1}
    for unit, factor in units.items():
        if size_str.endswith(unit):
            try:
                number = float(size_str.replace(unit, "").strip())
                return int(number * factor)
            except ValueError:
                pass
    return 0

def format_bytes(b):
    if b < 0: return "0.00 B"
    for u in ["B", "KB", "MB", "GB"]:
        if b < 1024: return f"{b:.2f} {u}"
        b /= 1024
    return f"{b:.2f} TB"
def fmt(n: float | int) -> str:
    return f"{n:,.2f}".replace(",", ".") if isinstance(n, float) else f"{n:,}".replace(",", ".")

# --------------------
# READ FILE / DIRECTORY
# --------------------

text = ""
file_stats = [] # Store (name, size, tokens)
min_bytes = parse_size(MIN_FILE_SIZE)

if os.path.isfile(FILE_PATH):
    print(f"Reading file: {FILE_PATH}")
    ext = os.path.splitext(FILE_PATH)[1].lower()
    size = os.path.getsize(FILE_PATH)
    
    # Check filters
    is_included = "all" in INCLUDE_TYPES or ext in [t.lower() for t in INCLUDE_TYPES]
    is_excluded = ext in [t.lower() for t in EXCLUDE_TYPES]
    is_big_enough = size >= min_bytes
    
    if is_included and not is_excluded and is_big_enough:
        with open(FILE_PATH, "r", encoding="utf-8", errors="ignore") as f:
            content = f.read()
            text = content
            enc = tiktoken.get_encoding("cl100k_base")
            tokens = len(enc.encode(content))
            file_stats.append((os.path.basename(FILE_PATH), size, tokens))

elif os.path.isdir(FILE_PATH):
    max_bytes = parse_size(MAX_FILE_SIZE)
    
    print(f"Reading directory: {FILE_PATH}")
    print(f"Filters: Include={INCLUDE_TYPES}, Exclude={EXCLUDE_TYPES}")
    print(f"Size Constraints: Min={MIN_FILE_SIZE}, Max={MAX_FILE_SIZE}")
    print(f"Ignoring Dirs: {EXCLUDE_DIRS}")

    file_count = 0
    enc = tiktoken.get_encoding("cl100k_base")
    
    # Walk the directory
    for root, dirs, files in os.walk(FILE_PATH):
        # Skip common non-code/garbage directories
        dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS]
        
        for file in files:
            ext = os.path.splitext(file)[1].lower()
            full_path = os.path.join(root, file)
            try:
                size = os.path.getsize(full_path)
                
                # Filter Logic
                is_included = "all" in INCLUDE_TYPES or ext in [t.lower() for t in INCLUDE_TYPES]
                is_excluded = ext in [t.lower() for t in EXCLUDE_TYPES]
                is_big_enough = size >= min_bytes
                is_small_enough = (max_bytes == 0) or (size <= max_bytes)

                if is_included and not is_excluded and is_big_enough and is_small_enough:
                    # Print progress for potentially slow files (e.g. > 100KB)
                    if size > 100 * 1024:
                        print(f"Processing large file: {file} ({format_bytes(size)})")
                        
                    with open(full_path, "r", encoding="utf-8", errors="ignore") as f:
                        content = f.read()
                        text += content + "\n" # Add newline between files
                        tokens = len(enc.encode(content))
                        file_stats.append((file, size, tokens))
                        file_count += 1
                elif not is_small_enough:
                     print(f"Skipping too large: {file} ({format_bytes(size)})")

            except Exception as e:
                print(f"Skipping {file}: {e}")
    print(f"Processed {file_count} files.")

else:
    print(f"Error: Path not found - {FILE_PATH}")
    exit()

lines = text.splitlines()

# --------------------
# BASIC STATS
# --------------------
char_count = len(text)
line_count = len(lines)
word_count = len(text.split())

# --------------------
# UNICODE STATS
# --------------------
letters = uppercase = lowercase = accented = 0
digits = punctuation = symbols = math_symbols = currency_symbols = emojis = whitespace = 0

punct_set = set(string.punctuation)

for ch in text:
    cat = unicodedata.category(ch)

    if ch.isalpha():
        letters += 1
        if ch.isupper():
            uppercase += 1
        elif ch.islower():
            lowercase += 1
        if "WITH" in unicodedata.name(ch, ""):
            accented += 1

    elif ch.isdigit():
        digits += 1
    elif ch in punct_set:
        punctuation += 1
    elif cat.startswith("S"):
        symbols += 1
        if cat == "Sm":
            math_symbols += 1
        elif cat == "Sc":
            currency_symbols += 1
        elif cat == "So":
            emojis += 1
    elif ch.isspace():
        whitespace += 1

# --------------------
# LINE STATS
# --------------------
line_lengths = [len(line) for line in lines]
empty_lines = sum(1 for l in lines if l == "")
whitespace_only_lines = sum(1 for l in lines if l.strip() == "" and l != "")
longest_line = max(line_lengths) if lines else 0
avg_line_len = int(sum(line_lengths) / line_count) if line_count else 0

# --------------------
# TOKENIZATION
# --------------------
enc = tiktoken.get_encoding("cl100k_base")
input_tokens = len(enc.encode(text))

# --------------------
# OUTPUT
# --------------------
print("Stats")
print("-" * 60)
print("Lines:", fmt(line_count))
print("  Empty:", fmt(empty_lines))
print("  Whitespace-only:", fmt(whitespace_only_lines))
print("  Longest line:", fmt(longest_line))
print("  Avg line length:", fmt(avg_line_len))
print("-" * 60)
print("Words:", fmt(word_count))
print("Characters:", fmt(char_count))
print("Letters:", fmt(letters))
print("  Uppercase:", fmt(uppercase))
print("  Lowercase:", fmt(lowercase))
print("  Accented:", fmt(accented))
print("Digits:", fmt(digits))
print("Punctuation:", fmt(punctuation))
print("Symbols:", fmt(symbols))
print("  Math:", fmt(math_symbols))
print("  Currency:", fmt(currency_symbols))
print("Emojis:", fmt(emojis))
print("Whitespace chars:", fmt(whitespace))
print("-" * 60)
print("TikToken tokens (cl100k_base):", fmt(input_tokens))

# --------------------
# RANKED FILE STATS
# --------------------
def format_bytes(b):
    for u in ["B", "KB", "MB"]:
        if b < 1024: return f"{b:.2f} {u}"
        b /= 1024

if file_stats:
    # Sort by size
    sorted_by_size = sorted(file_stats, key=lambda x: x[1], reverse=True)
    
    print("\nTop Biggest Files (Size)")
    print("-" * 40)
    for name, size, tokens in sorted_by_size[:N_TOP_BIGGEST]:
        print(f"{name:<25} | {format_bytes(size):>10} | {fmt(tokens):>8} tokens")

    print(f"\nTop Smallest Files (Size)")
    print("-" * 40)
    for name, size, tokens in sorted_by_size[-M_TOP_SMALLEST:]:
        # Reverse the smallest list if you want it ascending
        pass
    
    # Let's do it properly:
    smallest = sorted(file_stats, key=lambda x: x[1])
    for name, size, tokens in smallest[:M_TOP_SMALLEST]:
        print(f"{name:<25} | {format_bytes(size):>10} | {fmt(tokens):>8} tokens")