scripts/load_data.py

import os
import json
from datasets import load_dataset

def load_data():
    os.makedirs('data/raw', exist_ok=True)
    out_path = 'data/raw/samanantar_kn_50k.json'

    print("Downloading ai4bharat/samanantar kn subset...")
    ds = load_dataset('ai4bharat/samanantar', 'kn', split='train', streaming=True)
    
    samples = []
    print("Fetching 50k samples...")
    for i, item in enumerate(ds):
        if i >= 50000:
            break
        en = item.get('src', item.get('en', item.get('english', '')))
        kn = item.get('tgt', item.get('kn', item.get('kannada', '')))
        samples.append({'en': en, 'kn': kn})
    
    with open(out_path, 'w', encoding='utf-8') as f:
        json.dump(samples, f, ensure_ascii=False, indent=2)
    print(f"Saved 50k samples to {out_path}")

if __name__ == '__main__':
    load_data()