preprocessing/tokenizer.py

# Sentence tokenizer
import re

import re

def tokenize_sentences(text):
    # Step 1: normalize spacing
    text = re.sub(r'\s+', ' ', text)

    # Step 2: split using Tamil patterns + fallback chunking
    sentences = re.split(r'[.?!।]|(?<=\u0B95)\s|(?<=\u0BB2)\s', text)

    # Step 3: fallback → chunk long paragraphs
    final_sentences = []

    for sent in sentences:
        sent = sent.strip()

        # If too long → split manually every ~15 words
        words = sent.split()
        if len(words) > 20:
            for i in range(0, len(words), 15):
                chunk = " ".join(words[i:i+15])
                if len(chunk) > 30:
                    final_sentences.append(chunk)
        else:
            if len(sent) > 30:
                final_sentences.append(sent)

    return final_sentences


# Word tokenizer
def tokenize_text(sentence):
    return sentence.strip().split()


# Stopwords loader
def load_stopwords(path):
    with open(path, "r", encoding="utf-8") as f:
        return set(f.read().splitlines())


# Stopwords remover
def remove_stopwords(tokens, stopwords):
    return [word for word in tokens if word not in stopwords]