| # Sentence tokenizer | |
| import re | |
| import re | |
| def tokenize_sentences(text): | |
| # Step 1: normalize spacing | |
| text = re.sub(r'\s+', ' ', text) | |
| # Step 2: split using Tamil patterns + fallback chunking | |
| sentences = re.split(r'[.?!।]|(?<=\u0B95)\s|(?<=\u0BB2)\s', text) | |
| # Step 3: fallback → chunk long paragraphs | |
| final_sentences = [] | |
| for sent in sentences: | |
| sent = sent.strip() | |
| # If too long → split manually every ~15 words | |
| words = sent.split() | |
| if len(words) > 20: | |
| for i in range(0, len(words), 15): | |
| chunk = " ".join(words[i:i+15]) | |
| if len(chunk) > 30: | |
| final_sentences.append(chunk) | |
| else: | |
| if len(sent) > 30: | |
| final_sentences.append(sent) | |
| return final_sentences | |
| # Word tokenizer | |
| def tokenize_text(sentence): | |
| return sentence.strip().split() | |
| # Stopwords loader | |
| def load_stopwords(path): | |
| with open(path, "r", encoding="utf-8") as f: | |
| return set(f.read().splitlines()) | |
| # Stopwords remover | |
| def remove_stopwords(tokens, stopwords): | |
| return [word for word in tokens if word not in stopwords] |