data_loader.py

import pandas as pd
from pathlib import Path
from typing import List, Dict, Tuple
import random

def load_hindi_dataset(base_path: str = "data", split: str = "train", num_files: int = None) -> str:
    """
    Load Hindi text from dataset with train/validation split structure.
    
    Args:
        base_path: Base directory containing train and validation folders
        split: Either 'train' or 'valid'
        num_files: Number of files to load (None for all files)
    """
    base_dir = Path(base_path)
    split_dir = base_dir / split / split
    
    if not split_dir.exists():
        raise FileNotFoundError(f"Directory not found: {split_dir}")
    
    print(f"\nLoading Hindi dataset from {split_dir}")
    
    # Get all txt files in the directory
    txt_files = list(split_dir.glob("*.txt"))
    
    if not txt_files:
        raise FileNotFoundError(f"No txt files found in {split_dir}")
    
    # Sort files by word count (assuming filenames contain word counts)
    txt_files.sort(key=lambda x: int(x.stem))
    
    # Sample files if num_files is specified
    if num_files is not None:
        if num_files < len(txt_files):
            txt_files = random.sample(txt_files, num_files)
    
    print(f"Found {len(txt_files)} files")
    
    # Load and combine text from files
    texts = []
    total_chars = 0
    total_words = 0
    
    for idx, file_path in enumerate(txt_files, 1):
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                text = f.read().strip()
                word_count = int(file_path.stem)  # Filename is word count
                texts.append(text)
                total_chars += len(text)
                total_words += word_count
                
                if idx % 10 == 0:
                    print(f"Processed {idx}/{len(txt_files)} files. "
                          f"Total characters: {total_chars:,}, "
                          f"Total words: {total_words:,}")
                
        except Exception as e:
            print(f"Error reading file {file_path}: {e}")
            continue
    
    combined_text = "\n\n".join(texts)
    
    print(f"\nDataset loading completed:")
    print(f"Total files: {len(texts)}")
    print(f"Total characters: {len(combined_text):,}")
    print(f"Total words: {total_words:,}")
    print(f"Average words per file: {total_words/len(texts):,.1f}")
    
    return combined_text

def get_dataset_stats(base_path: str = "data") -> Dict:
    """Get statistics about the dataset."""
    stats = {}
    for split in ['train', 'valid']:
        split_dir = Path(base_path) / split
        if split_dir.exists():
            txt_files = list(split_dir.glob("*.txt"))
            word_counts = [int(f.stem) for f in txt_files]
            stats[split] = {
                'num_files': len(txt_files),
                'total_words': sum(word_counts),
                'min_words': min(word_counts) if word_counts else 0,
                'max_words': max(word_counts) if word_counts else 0,
                'avg_words': sum(word_counts)/len(word_counts) if word_counts else 0
            }
    return stats

def load_train_valid_split(base_path: str = "data", 
                          train_files: int = None,
                          valid_files: int = None) -> Tuple[str, str]:
    """Load both train and validation splits."""
    train_text = load_hindi_dataset(base_path, "train", train_files)
    valid_text = load_hindi_dataset(base_path, "valid", valid_files)
    return train_text, valid_text

if __name__ == "__main__":
    # Print dataset statistics
    stats = get_dataset_stats()
    print("\nDataset Statistics:")
    print("-" * 50)
    for split, split_stats in stats.items():
        print(f"\n{split.upper()} Split:")
        for key, value in split_stats.items():
            if isinstance(value, (int, float)):
                print(f"{key}: {value:,}")
            else:
                print(f"{key}: {value}")
    
    # Load sample data
    print("\nLoading sample data...")
    train_text, valid_text = load_train_valid_split(train_files=5, valid_files=2)
    print(f"\nSample train text (first 200 chars):\n{train_text[:200]}")
    print(f"\nSample valid text (first 200 chars):\n{valid_text[:200]}")