File size: 3,884 Bytes
b8eb008 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | from transformers import AutoTokenizer, AutoModel
import numpy as np
class HindiEnglishEncodeDecode:
def __init__(self, model_name):
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.model = AutoModel.from_pretrained(model_name)
def test_languages(self):
test_texts = {
'Hindi': [
'नमस्ते, मैं भारत से हूँ। दिल्ली बहुत बड़ा शहर है।',
'हिंदी भाषा बहुत सुंदर है।',
'मुझे किताबें पढ़ना पसंद है।',
'यह एक उदाहरण वाक्य है।'
],
'English': [
'Hello, I am from India. Delhi is a big city.',
'The English language is widely spoken.',
'I enjoy reading books.',
'This is an example sentence.'
]
}
results = {}
for language, texts in test_texts.items():
results[language] = []
for text in texts:
try:
token_ids = self.tokenizer.encode(text, add_special_tokens=True)
token_strings = self.tokenizer.tokenize(text)
decoded_text = self.tokenizer.decode(token_ids, skip_special_tokens=True)
token_stats = {
'min': min(token_ids),
'max': max(token_ids),
'mean': np.mean(token_ids)
}
# Append results for this text
results[language].append({
'original_text': text,
'token_ids_count': len(token_ids),
'token_strings_count': len(token_strings),
'decoded_text': decoded_text,
'text_match': text == decoded_text,
'token_id_stats': token_stats
})
print(f"\n{language} Analysis:")
print(f"Original Text: {text}")
print(f"Token IDs Count: {len(token_ids)}")
print(f"Token Strings: {token_strings}")
print(f"Decoded Text: {decoded_text}")
print(f"Text Reconstruction: {text == decoded_text}")
except Exception as e:
results[language].append({'error': str(e)})
print(f"{language} Error: {e}")
return results
def detailed_token_analysis(self, text):
token_ids = self.tokenizer.encode(text, add_special_tokens=True)
token_strings = self.tokenizer.tokenize(text)
analysis = {
'original_text': text,
'original_length': len(text),
'tokens': {
'ids': token_ids,
'strings': token_strings
},
'token_stats': {
'total_tokens': len(token_ids),
'unique_tokens': len(set(token_ids)),
'avg_token_length': np.mean([len(token) for token in token_strings])
}
}
return analysis
def main():
MODEL_NAME = 'tinycompany/ShawtyIsBad-bgem3'
tokenizer_model = HindiEnglishEncodeDecode(MODEL_NAME)
results = tokenizer_model.test_languages()
sample_text = 'नमस्ते, मैं भारत से हूँ। दिल्ली बहुत बड़ा शहर है।'
detailed_result = tokenizer_model.detailed_token_analysis(sample_text)
import json
with open('hindi_english_tokenization_results.json', 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=4)
if __name__ == "__main__":
main() |