prepare_corpus.py

import os

DATA_PATH = "data/politics"
OUTPUT_FILE = "data/tamil_corpus.txt"


def collect_text():
    full_text = ""

    for root, dirs, files in os.walk(DATA_PATH):
        for file in files:
            if file.endswith(".txt"):
                path = os.path.join(root, file)

                try:
                    with open(path, encoding="utf-8", errors="ignore") as f:
                        full_text += f.read() + "\n"
                except:
                    continue

    return full_text


def save(text):
    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
        f.write(text)

    print("Corpus saved:", OUTPUT_FILE)


if __name__ == "__main__":
    text = collect_text()
    save(text)