tokenizer.py

import re
import json

class MCQTokenizer:
    def __init__(self, vocab=None, max_len=128):
        self.vocab = vocab or {"<PAD>": 0, "<UNK>": 1}
        self.max_len = max_len

    @staticmethod
    def clean_text(text):
        text = str(text).lower()
        text = re.sub(r'[^a-z0-9 ]', '', text)
        return text

    def tokenize(self, text):
        words = self.clean_text(text).split()
        tokens = [self.vocab.get(w, self.vocab.get("<UNK>", 1)) for w in words]
        if len(tokens) < self.max_len:
            tokens = tokens + [self.vocab.get("<PAD>", 0)] * (self.max_len - len(tokens))
        else:
            tokens = tokens[:self.max_len]
        return tokens

    @classmethod
    def load_vocab(cls, vocab_path, max_len=128):
        with open(vocab_path, 'r', encoding='utf-8') as f:
            vocab = json.load(f)
        return cls(vocab=vocab, max_len=max_len)