Streamlit app working

BPE.py

@@ -3,7 +3,7 @@ import regex as re
 from tqdm import tqdm
 
 # Read text from a file
-with open('text_file_eng_long.txt', 'r', encoding='utf-8') as file:
+with open('text_file.txt', 'r', encoding='utf-8') as file:
     text = file.read()
 
 # Hindi-focused pattern
@@ -88,9 +88,11 @@ def perform_bpe():
 
 if __name__ == "__main__":
     print('---')
-    print("length of text:", len(text))
+    print("length of text (characters):", len(text))
+    print("length of text (words):", len(text.split()))
     print('---')
     print("length of tokens:", len(tokens))
+    #print("sample tokens:", tokens[:5])  # Show first 5 tokens
     
     # Run BPE and save results
     merges, ids, num_merges = perform_bpe()

README.md

@@ -0,0 +1,28 @@
+# Hindi Tokenizer 
+
+## Dataset
+
+The final dataset used for the tokenizer training is found in text_file.txt in the repo. 
+
+There were 2 prime sources which were combined in the .txt file - 
+
+- https://www.kaggle.com/datasets/disisbig/hindi-text-short-summarization-corpus (test dataset)
+- https://hindi.newslaundry.com/report 
+
+length of text (characters): 5933269
+
+length of text (words):      1150937
+
+
+## Results 
+
+length of text (words):      1150937
+
+length of tokens (regex):    1354962
+
+---
+Total bytes before: 14659421
+
+Total bytes after: 1889786
+
+Compression ratio: 7.76X

app.py

@@ -0,0 +1,87 @@
+import streamlit as st
+from tokenizer import HindiTokenizer
+
+# Initialize tokenizer
+@st.cache_resource
+def load_tokenizer():
+    return HindiTokenizer()
+
+def format_token_ids(token_ids):
+    # Format token IDs in a readable way, 10 per line
+    lines = []
+    for i in range(0, len(token_ids), 10):
+        line = token_ids[i:i+10]
+        lines.append(' '.join(str(id) for id in line))
+    return '\n'.join(lines)
+
+def format_hindi_tokens(tokens):
+    # Join tokens with double spaces
+    return '  '.join(tokens)
+
+def main():
+    st.title("Hindi Text Tokenizer")
+    
+    tokenizer = load_tokenizer()
+    
+    # Create columns for metrics
+    col1, col2, col3 = st.columns(3)
+    
+    with col1:
+        st.subheader("Word Count")
+    with col2:
+        st.subheader("Compression Ratio")
+    with col3:
+        st.subheader("BPE Tokens")  # Renamed to clarify these are post-BPE tokens
+    
+    # Text input
+    st.subheader("Input Text:")
+    text_input = st.text_area(
+        label="Input Hindi text", 
+        height=150, 
+        key="input",
+        label_visibility="collapsed"
+    )
+    
+    if st.button("Tokenize"):
+        if text_input:
+            # Get tokens and IDs
+            token_ids, original_tokens, decoded_tokens = tokenizer.tokenize(text_input)
+            
+            # Calculate metrics
+            word_count = len(text_input.split())
+            original_bytes = sum(len(token.encode('utf-8')) for token in original_tokens)
+            compression_ratio = original_bytes / len(token_ids)
+            
+            # Update metrics
+            col1.write(f"{word_count}")
+            col2.write(f"{compression_ratio:.2f}X")
+            col3.write(f"{len(token_ids)}")  # This is post-BPE token count
+            
+            # Optional: Display both token counts for comparison
+            st.caption(f"Initial tokens (after regex): {len(original_tokens)}")
+            st.caption(f"Final tokens (after BPE): {len(token_ids)}")
+            
+            # Display token IDs in a formatted way
+            st.subheader("Token IDs:")
+            st.text_area(
+                label="Generated token IDs",
+                value=format_token_ids(token_ids), 
+                height=150, 
+                key="ids",
+                label_visibility="collapsed"
+            )
+            
+            # Display decoded tokens with tab separation
+            st.subheader("Tokenized Text:")
+            st.text_area(
+                label="Tokenized output",
+                value='\t'.join(decoded_tokens), 
+                height=150, 
+                key="tokens",
+                label_visibility="collapsed"
+            )
+        else:
+            st.warning("Please enter some text to tokenize.")
+
+if __name__ == "__main__":
+    main() 

data_analysis.py

@@ -1,30 +0,0 @@
-# Read text from a file
-with open('text_file.txt', 'r', encoding='utf-8') as file:
-    text = file.read()
-
-tokens = text.encode("utf-8")  # raw bytes
-tokens = list(map(int, tokens))  # convert to a list of integers in range 0..255 for convenience
-
-print('---')
-print("length of text:", len(text))
-print('---')
-#print(tokens)
-print('---')
-print("length of tokens:", len(tokens))
-
-def get_stats(ids):
-    counts = {}
-    for pair in zip(ids, ids[1:]): # Pythonic way to iterate consecutive elements
-        counts[pair] = counts.get(pair, 0) + 1
-    return counts
-
-stats = get_stats(tokens)
-print('---')
-# print(stats)
-#print(sorted(((v,k) for k,v in stats.items()), reverse=True))
-
-print('---')
-top_pair = max(stats, key=stats.get)
-print(top_pair)
-
-#print(chr(224), chr(164))

decoded_output.txt

@@ -1,3 +1 @@
-अम	ज	द	 के	 परिवार	 की	 तीन	 पी	ढ़	ियां	 चांद	नी	 चौ	क	 निर्�	�	ाचन	 क्षेत्र	 में	 ह	वे	ली	 आज	़	म	 ख	ां	 के	 नाम	 से	 पहच	ाने	 जाने	 वाले	 एक	दम	 स	ट	कर	 बने	 घ	रों	 के	 झ	ुण	्ड	 में	 रहती	 हैं	.	 यह	 इलाक	ा	 दिल्ली	 की	 ऐ	त	िहास	िक	 ज	ामा	 मस्जिद	 से	 पै	दल	 की	 द	ूरी	 पर	 है	,	 और	 इस	 परिवार	 के	 23	 सदस्य	 मतदान	 केंद्र	 10	 पर	 प	ंज	ीक	ृत	 मत	द	ाता	 हैं	.	 लेकिन	 पिछले	 साल	 लोकसभा	 चुनावों	 के	 दौरान	 अम	ज	द	 को	 पता	 चला	 कि	 वह	 अपने	 परिवार	 के	 उन	 20	 लोगों	 में	 से	 एक	 हैं	,	 जिन	का	 नाम	 मत	द	ाता	 सू	ची	 से	 इस	 वजह	 से	 काट	 दिया	 गया	 कि	 उन्होंने	 अपना	 घर	 बदल	 लिया	 है	.	
-	
-	5	5	 वर्षीय	 अम	ज	द	 ने	 न्यू	ज़	ल	ॉन	्ड	्री	 को	 बताया	,	 	"	हम	ारे	 सामने	 ये	 पहली	 बार	 हुआ	 है	.	 लेकिन	 नाम	 कट	ने	 के	 बारे	 में	 सबसे	 ज्यादा	 निर	ाश	ाज	न	क	 बात	 ये	 थी	 कि	 इसका	 पता	 मतदान	 के	 दिन	 ही	 चला	.	 जब	 हम	 पहली	 बार	 बू	थ	 10	 पर	 गए	 तो	 उन्होंने	 हमें	 बताया	 कि	 उन्हें	 मत	द	ाता	 सू	ची	 में	 हमारा	 नाम	 नहीं	 मिला	.	 इसलिए	 हमें	 ज	ामा	 मस्जिद	 में	 किसी	 दूसरे	 बू	थ	 पर	 जाकर	 देखना	 चाहिए	.	 वहां	 से	 हमें	 दूसरे	 बू	थ	 पर	 भेज	 दिया	 गया	.	 इस	 तरह	 हमने	 पांच	 से	 छह	 बू	थ	ों	 का	 दौर	ा	 किया	.	 और	 फिर	 अंत	 में	 हमें	 जो	 कारण	 बताया	 गया	,	 वो	 यह	 था	 कि	 शायद	 घर	-	घ	र	 जाकर	 सर्वे	क्षण	 के	 दौरान	 बी	एल	ओ	 (	ब	ू	थ	 ले	वल	 ऑफिस	र	)	 को	 हम	 घर	 पर	 नहीं	 मिले	 इसलिए	 उन्होंने	 हमारे	 नाम	 काट	 दिए	.”
+च	ां	द	नी	 चौ	क	 विधानसभा	 क्षेत्र	 में	 सबसे	 ज्यादा	 मत	दाता	 सूची	 से	 हट	ाए	 गए	 तीन	 बू	थ	ों	 में	 से	 एक	 बू	थ	 वो	 था	,	 जहां	 अम	ज	द	 का	 घर	 था	 और	 दूसरा	 बू	थ	 कांग्रेस	 उम्मीदवार	 जे	पी	 अग्र	वाल	 के	 पड़	ोस	 में	 था	े	.	 अपने	 पक्ष	 में	 मतदान	 में	 14	 प्रतिशत	 की	 बढ़	ोत	री	 के	 बावजूद	 अग्र	वाल	 भाजपा	 से	 9	0	,	000	 वोटों	 के	 अंतर	 से	 हार	 गए	.	 न्यू	ज़	ल	ॉन	्ड	्री	 ने	 घर	-	घ	र	 जाकर	 सर्वे	क्षण	 किया	 और	 प	ुष्ट	ि	 की	,	 कि	 इन	 तीन	 बू	थ	ों	 पर	 हट	ाए	 गए	 4	9	7	 मत	दाताओं	 में	 से	 कम	 से	 कम	 14	7	 (	लग	भग	 29	.	6	 प्रतिशत	)	 गलत	 तरीके	 से	 हट	ाए	 गए	 थे	.

encode_decode.py

@@ -25,7 +25,21 @@ def decode(ids):
     return text
 
 # Example: Decode a list of IDs
-set_of_ids = [2342, 307, 295, 286, 1413, 302, 839, 644, 574, 982, 3877, 405, 1086, 272, 978, 181, 3927, 1171, 294, 274, 964, 438, 767, 337, 284, 361, 332, 286, 776, 315, 2331, 429, 841, 631, 385, 1694, 273, 310, 418, 1607, 445, 935, 286, 962, 1244, 698, 294, 3069, 347, 46, 450, 1462, 259, 646, 302, 554, 276, 2252, 334, 292, 2835, 2500, 315, 1006, 3367, 302, 296, 1299, 330, 289, 44, 327, 345, 1413, 286, 2911, 1906, 2592, 1322, 888, 330, 279, 711, 1474, 997, 1068, 295, 1236, 347, 46, 513, 1067, 579, 1194, 2596, 286, 847, 732, 307, 295, 309, 1423, 1953, 340, 555, 563, 1413, 286, 376, 466, 596, 294, 315, 385, 347, 44, 1001, 478, 776, 1068, 295, 1236, 919, 1216, 315, 345, 1115, 315, 3189, 481, 437, 340, 557, 1125, 1135, 1501, 857, 289, 46, 10, 10, 53, 53, 2794, 732, 307, 295, 317, 2705, 2246, 280, 1308, 698, 486, 309, 739, 44, 32, 34, 808, 830, 1015, 516, 1315, 544, 667, 289, 46, 513, 776, 1914, 311, 286, 948, 294, 856, 915, 2438, 658, 367, 271, 272, 564, 516, 472, 340, 1571, 1423, 2592, 286, 638, 416, 1953, 46, 586, 462, 1315, 544, 3075, 583, 888, 330, 588, 444, 557, 1448, 739, 340, 737, 1068, 295, 1236, 919, 1216, 294, 3253, 776, 391, 1410, 46, 1496, 1448, 292, 2835, 2500, 294, 738, 1374, 3075, 583, 330, 2660, 3252, 904, 46, 1441, 315, 1448, 1374, 3075, 583, 330, 1473, 481, 437, 46, 345, 778, 1758, 1307, 315, 2210, 3075, 583, 299, 333, 751, 259, 420, 46, 327, 766, 1200, 294, 1448, 499, 1394, 739, 437, 44, 707, 450, 413, 340, 3602, 1135, 45, 864, 261, 2660, 2749, 1930, 286, 847, 447, 1782, 1633, 510, 308, 306, 583, 399, 1508, 2632, 261, 41, 309, 462, 1135, 330, 391, 1193, 1496, 557, 1574, 776, 3189, 1340, 3435]
+set_of_ids = [335, 332, 295, 401, 1050, 273, 1153, 1094, 294, 843,
+859, 1092, 3583, 3327, 315, 2457, 437, 585, 867, 3747,
+587, 299, 294, 315, 388, 3747, 587, 785, 414, 44,
+1110, 712, 307, 295, 334, 984, 414, 329, 2892, 3747,
+587, 583, 1160, 1593, 427, 3934, 621, 285, 1583, 1936,
+294, 414, 260, 46, 548, 2007, 294, 2733, 294, 1467,
+1553, 300, 763, 2045, 381, 285, 2093, 3934, 621, 1882,
+315, 1077, 48, 44, 1581, 3991, 285, 1909, 315, 1595,
+585, 46, 2161, 2714, 280, 1016, 698, 475, 316, 984,
+45, 861, 261, 2836, 2999, 1947, 418, 329, 279, 3331,
+266, 300, 44, 343, 591, 867, 3747, 587, 299, 330,
+2457, 437, 585, 715, 57, 55, 1092, 3017, 294, 315,
+565, 315, 565, 1467, 55, 489, 2139, 2057, 2927, 46,
+54, 1553, 41, 2217, 2695, 315, 2457, 437, 585, 533,
+46]
 decoded_text = decode(set_of_ids)  # Pass the list of IDs
 print(decoded_text)
 

requirements.txt

@@ -0,0 +1,2 @@
+streamlit
+regex 

text_file_eng.txt

@@ -1,1100 +0,0 @@
-First Citizen:
-Before we proceed any further, hear me speak.
-
-All:
-Speak, speak.
-
-First Citizen:
-You are all resolved rather to die than to famish?
-
-All:
-Resolved. resolved.
-
-First Citizen:
-First, you know Caius Marcius is chief enemy to the people.
-
-All:
-We know't, we know't.
-
-First Citizen:
-Let us kill him, and we'll have corn at our own price.
-Is't a verdict?
-
-All:
-No more talking on't; let it be done: away, away!
-
-Second Citizen:
-One word, good citizens.
-
-First Citizen:
-We are accounted poor citizens, the patricians good.
-What authority surfeits on would relieve us: if they
-would yield us but the superfluity, while it were
-wholesome, we might guess they relieved us humanely;
-but they think we are too dear: the leanness that
-afflicts us, the object of our misery, is as an
-inventory to particularise their abundance; our
-sufferance is a gain to them Let us revenge this with
-our pikes, ere we become rakes: for the gods know I
-speak this in hunger for bread, not in thirst for revenge.
-
-Second Citizen:
-Would you proceed especially against Caius Marcius?
-
-All:
-Against him first: he's a very dog to the commonalty.
-
-Second Citizen:
-Consider you what services he has done for his country?
-
-First Citizen:
-Very well; and could be content to give him good
-report fort, but that he pays himself with being proud.
-
-Second Citizen:
-Nay, but speak not maliciously.
-
-First Citizen:
-I say unto you, what he hath done famously, he did
-it to that end: though soft-conscienced men can be
-content to say it was for his country he did it to
-please his mother and to be partly proud; which he
-is, even till the altitude of his virtue.
-
-Second Citizen:
-What he cannot help in his nature, you account a
-vice in him. You must in no way say he is covetous.
-
-First Citizen:
-If I must not, I need not be barren of accusations;
-he hath faults, with surplus, to tire in repetition.
-What shouts are these? The other side o' the city
-is risen: why stay we prating here? to the Capitol!
-
-All:
-Come, come.
-
-First Citizen:
-Soft! who comes here?
-
-Second Citizen:
-Worthy Menenius Agrippa; one that hath always loved
-the people.
-
-First Citizen:
-He's one honest enough: would all the rest were so!
-
-MENENIUS:
-What work's, my countrymen, in hand? where go you
-With bats and clubs? The matter? speak, I pray you.
-
-First Citizen:
-Our business is not unknown to the senate; they have
-had inkling this fortnight what we intend to do,
-which now we'll show 'em in deeds. They say poor
-suitors have strong breaths: they shall know we
-have strong arms too.
-
-MENENIUS:
-Why, masters, my good friends, mine honest neighbours,
-Will you undo yourselves?
-
-First Citizen:
-We cannot, sir, we are undone already.
-
-MENENIUS:
-I tell you, friends, most charitable care
-Have the patricians of you. For your wants,
-Your suffering in this dearth, you may as well
-Strike at the heaven with your staves as lift them
-Against the Roman state, whose course will on
-The way it takes, cracking ten thousand curbs
-Of more strong link asunder than can ever
-Appear in your impediment. For the dearth,
-The gods, not the patricians, make it, and
-Your knees to them, not arms, must help. Alack,
-You are transported by calamity
-Thither where more attends you, and you slander
-The helms o' the state, who care for you like fathers,
-When you curse them as enemies.
-
-First Citizen:
-Care for us! True, indeed! They ne'er cared for us
-yet: suffer us to famish, and their store-houses
-crammed with grain; make edicts for usury, to
-support usurers; repeal daily any wholesome act
-established against the rich, and provide more
-piercing statutes daily, to chain up and restrain
-the poor. If the wars eat us not up, they will; and
-there's all the love they bear us.
-
-MENENIUS:
-Either you must
-Confess yourselves wondrous malicious,
-Or be accused of folly. I shall tell you
-A pretty tale: it may be you have heard it;
-But, since it serves my purpose, I will venture
-To stale 't a little more.
-
-First Citizen:
-Well, I'll hear it, sir: yet you must not think to
-fob off our disgrace with a tale: but, an 't please
-you, deliver.
-
-MENENIUS:
-There was a time when all the body's members
-Rebell'd against the belly, thus accused it:
-That only like a gulf it did remain
-I' the midst o' the body, idle and unactive,
-Still cupboarding the viand, never bearing
-Like labour with the rest, where the other instruments
-Did see and hear, devise, instruct, walk, feel,
-And, mutually participate, did minister
-Unto the appetite and affection common
-Of the whole body. The belly answer'd--
-
-First Citizen:
-Well, sir, what answer made the belly?
-
-MENENIUS:
-Sir, I shall tell you. With a kind of smile,
-Which ne'er came from the lungs, but even thus--
-For, look you, I may make the belly smile
-As well as speak--it tauntingly replied
-To the discontented members, the mutinous parts
-That envied his receipt; even so most fitly
-As you malign our senators for that
-They are not such as you.
-
-First Citizen:
-Your belly's answer? What!
-The kingly-crowned head, the vigilant eye,
-The counsellor heart, the arm our soldier,
-Our steed the leg, the tongue our trumpeter.
-With other muniments and petty helps
-In this our fabric, if that they--
-
-MENENIUS:
-What then?
-'Fore me, this fellow speaks! What then? what then?
-
-First Citizen:
-Should by the cormorant belly be restrain'd,
-Who is the sink o' the body,--
-
-MENENIUS:
-Well, what then?
-
-First Citizen:
-The former agents, if they did complain,
-What could the belly answer?
-
-MENENIUS:
-I will tell you
-If you'll bestow a small--of what you have little--
-Patience awhile, you'll hear the belly's answer.
-
-First Citizen:
-Ye're long about it.
-
-MENENIUS:
-Note me this, good friend;
-Your most grave belly was deliberate,
-Not rash like his accusers, and thus answer'd:
-'True is it, my incorporate friends,' quoth he,
-'That I receive the general food at first,
-Which you do live upon; and fit it is,
-Because I am the store-house and the shop
-Of the whole body: but, if you do remember,
-I send it through the rivers of your blood,
-Even to the court, the heart, to the seat o' the brain;
-And, through the cranks and offices of man,
-The strongest nerves and small inferior veins
-From me receive that natural competency
-Whereby they live: and though that all at once,
-You, my good friends,'--this says the belly, mark me,--
-
-First Citizen:
-Ay, sir; well, well.
-
-MENENIUS:
-'Though all at once cannot
-See what I do deliver out to each,
-Yet I can make my audit up, that all
-From me do back receive the flour of all,
-And leave me but the bran.' What say you to't?
-
-First Citizen:
-It was an answer: how apply you this?
-
-MENENIUS:
-The senators of Rome are this good belly,
-And you the mutinous members; for examine
-Their counsels and their cares, digest things rightly
-Touching the weal o' the common, you shall find
-No public benefit which you receive
-But it proceeds or comes from them to you
-And no way from yourselves. What do you think,
-You, the great toe of this assembly?
-
-First Citizen:
-I the great toe! why the great toe?
-
-MENENIUS:
-For that, being one o' the lowest, basest, poorest,
-Of this most wise rebellion, thou go'st foremost:
-Thou rascal, that art worst in blood to run,
-Lead'st first to win some vantage.
-But make you ready your stiff bats and clubs:
-Rome and her rats are at the point of battle;
-The one side must have bale.
-Hail, noble Marcius!
-
-MARCIUS:
-Thanks. What's the matter, you dissentious rogues,
-That, rubbing the poor itch of your opinion,
-Make yourselves scabs?
-
-First Citizen:
-We have ever your good word.
-
-MARCIUS:
-He that will give good words to thee will flatter
-Beneath abhorring. What would you have, you curs,
-That like nor peace nor war? the one affrights you,
-The other makes you proud. He that trusts to you,
-Where he should find you lions, finds you hares;
-Where foxes, geese: you are no surer, no,
-Than is the coal of fire upon the ice,
-Or hailstone in the sun. Your virtue is
-To make him worthy whose offence subdues him
-And curse that justice did it.
-Who deserves greatness
-Deserves your hate; and your affections are
-A sick man's appetite, who desires most that
-Which would increase his evil. He that depends
-Upon your favours swims with fins of lead
-And hews down oaks with rushes. Hang ye! Trust Ye?
-With every minute you do change a mind,
-And call him noble that was now your hate,
-Him vile that was your garland. What's the matter,
-That in these several places of the city
-You cry against the noble senate, who,
-Under the gods, keep you in awe, which else
-Would feed on one another? What's their seeking?
-
-MENENIUS:
-For corn at their own rates; whereof, they say,
-The city is well stored.
-
-MARCIUS:
-Hang 'em! They say!
-They'll sit by the fire, and presume to know
-What's done i' the Capitol; who's like to rise,
-Who thrives and who declines; side factions
-and give out
-Conjectural marriages; making parties strong
-And feebling such as stand not in their liking
-Below their cobbled shoes. They say there's
-grain enough!
-Would the nobility lay aside their ruth,
-And let me use my sword, I'll make a quarry
-With thousands of these quarter'd slaves, as high
-As I could pick my lance.
-
-MENENIUS:
-Nay, these are almost thoroughly persuaded;
-For though abundantly they lack discretion,
-Yet are they passing cowardly. But, I beseech you,
-What says the other troop?
-
-MARCIUS:
-They are dissolved: hang 'em!
-They said they were an-hungry; sigh'd forth proverbs,
-That hunger broke stone walls, that dogs must eat,
-That meat was made for mouths, that the gods sent not
-Corn for the rich men only: with these shreds
-They vented their complainings; which being answer'd,
-And a petition granted them, a strange one--
-To break the heart of generosity,
-And make bold power look pale--they threw their caps
-As they would hang them on the horns o' the moon,
-Shouting their emulation.
-
-MENENIUS:
-What is granted them?
-
-MARCIUS:
-Five tribunes to defend their vulgar wisdoms,
-Of their own choice: one's Junius Brutus,
-Sicinius Velutus, and I know not--'Sdeath!
-The rabble should have first unroof'd the city,
-Ere so prevail'd with me: it will in time
-Win upon power and throw forth greater themes
-For insurrection's arguing.
-
-MENENIUS:
-This is strange.
-
-MARCIUS:
-Go, get you home, you fragments!
-
-Messenger:
-Where's Caius Marcius?
-
-MARCIUS:
-Here: what's the matter?
-
-Messenger:
-The news is, sir, the Volsces are in arms.
-
-MARCIUS:
-I am glad on 't: then we shall ha' means to vent
-Our musty superfluity. See, our best elders.
-
-First Senator:
-Marcius, 'tis true that you have lately told us;
-The Volsces are in arms.
-
-MARCIUS:
-They have a leader,
-Tullus Aufidius, that will put you to 't.
-I sin in envying his nobility,
-And were I any thing but what I am,
-I would wish me only he.
-
-COMINIUS:
-You have fought together.
-
-MARCIUS:
-Were half to half the world by the ears and he.
-Upon my party, I'ld revolt to make
-Only my wars with him: he is a lion
-That I am proud to hunt.
-
-First Senator:
-Then, worthy Marcius,
-Attend upon Cominius to these wars.
-
-COMINIUS:
-It is your former promise.
-
-MARCIUS:
-Sir, it is;
-And I am constant. Titus Lartius, thou
-Shalt see me once more strike at Tullus' face.
-What, art thou stiff? stand'st out?
-
-TITUS:
-No, Caius Marcius;
-I'll lean upon one crutch and fight with t'other,
-Ere stay behind this business.
-
-MENENIUS:
-O, true-bred!
-
-First Senator:
-Your company to the Capitol; where, I know,
-Our greatest friends attend us.
-
-TITUS:
-
-COMINIUS:
-Noble Marcius!
-
-First Senator:
-
-MARCIUS:
-Nay, let them follow:
-The Volsces have much corn; take these rats thither
-To gnaw their garners. Worshipful mutiners,
-Your valour puts well forth: pray, follow.
-
-SICINIUS:
-Was ever man so proud as is this Marcius?
-
-BRUTUS:
-He has no equal.
-
-SICINIUS:
-When we were chosen tribunes for the people,--
-
-BRUTUS:
-Mark'd you his lip and eyes?
-
-SICINIUS:
-Nay. but his taunts.
-
-BRUTUS:
-Being moved, he will not spare to gird the gods.
-
-SICINIUS:
-Be-mock the modest moon.
-
-BRUTUS:
-The present wars devour him: he is grown
-Too proud to be so valiant.
-
-SICINIUS:
-Such a nature,
-Tickled with good success, disdains the shadow
-Which he treads on at noon: but I do wonder
-His insolence can brook to be commanded
-Under Cominius.
-
-BRUTUS:
-Fame, at the which he aims,
-In whom already he's well graced, can not
-Better be held nor more attain'd than by
-A place below the first: for what miscarries
-Shall be the general's fault, though he perform
-To the utmost of a man, and giddy censure
-Will then cry out of Marcius 'O if he
-Had borne the business!'
-
-SICINIUS:
-Besides, if things go well,
-Opinion that so sticks on Marcius shall
-Of his demerits rob Cominius.
-
-BRUTUS:
-Come:
-Half all Cominius' honours are to Marcius.
-Though Marcius earned them not, and all his faults
-To Marcius shall be honours, though indeed
-In aught he merit not.
-
-SICINIUS:
-Let's hence, and hear
-How the dispatch is made, and in what fashion,
-More than his singularity, he goes
-Upon this present action.
-
-BRUTUS:
-Lets along.
-
-First Senator:
-So, your opinion is, Aufidius,
-That they of Rome are entered in our counsels
-And know how we proceed.
-
-AUFIDIUS:
-Is it not yours?
-What ever have been thought on in this state,
-That could be brought to bodily act ere Rome
-Had circumvention? 'Tis not four days gone
-Since I heard thence; these are the words: I think
-I have the letter here; yes, here it is.
-'They have press'd a power, but it is not known
-Whether for east or west: the dearth is great;
-The people mutinous; and it is rumour'd,
-Cominius, Marcius your old enemy,
-Who is of Rome worse hated than of you,
-And Titus Lartius, a most valiant Roman,
-These three lead on this preparation
-Whither 'tis bent: most likely 'tis for you:
-Consider of it.'
-
-First Senator:
-Our army's in the field
-We never yet made doubt but Rome was ready
-To answer us.
-
-AUFIDIUS:
-Nor did you think it folly
-To keep your great pretences veil'd till when
-They needs must show themselves; which
-in the hatching,
-It seem'd, appear'd to Rome. By the discovery.
-We shall be shorten'd in our aim, which was
-To take in many towns ere almost Rome
-Should know we were afoot.
-
-Second Senator:
-Noble Aufidius,
-Take your commission; hie you to your bands:
-Let us alone to guard Corioli:
-If they set down before 's, for the remove
-Bring your army; but, I think, you'll find
-They've not prepared for us.
-
-AUFIDIUS:
-O, doubt not that;
-I speak from certainties. Nay, more,
-Some parcels of their power are forth already,
-And only hitherward. I leave your honours.
-If we and Caius Marcius chance to meet,
-'Tis sworn between us we shall ever strike
-Till one can do no more.
-
-All:
-The gods assist you!
-
-AUFIDIUS:
-And keep your honours safe!
-
-First Senator:
-Farewell.
-
-Second Senator:
-Farewell.
-
-All:
-Farewell.
-
-VOLUMNIA:
-I pray you, daughter, sing; or express yourself in a
-more comfortable sort: if my son were my husband, I
-should freelier rejoice in that absence wherein he
-won honour than in the embracements of his bed where
-he would show most love. When yet he was but
-tender-bodied and the only son of my womb, when
-youth with comeliness plucked all gaze his way, when
-for a day of kings' entreaties a mother should not
-sell him an hour from her beholding, I, considering
-how honour would become such a person. that it was
-no better than picture-like to hang by the wall, if
-renown made it not stir, was pleased to let him seek
-danger where he was like to find fame. To a cruel
-war I sent him; from whence he returned, his brows
-bound with oak. I tell thee, daughter, I sprang not
-more in joy at first hearing he was a man-child
-than now in first seeing he had proved himself a
-man.
-
-VIRGILIA:
-But had he died in the business, madam; how then?
-
-VOLUMNIA:
-Then his good report should have been my son; I
-therein would have found issue. Hear me profess
-sincerely: had I a dozen sons, each in my love
-alike and none less dear than thine and my good
-Marcius, I had rather had eleven die nobly for their
-country than one voluptuously surfeit out of action.
-
-Gentlewoman:
-Madam, the Lady Valeria is come to visit you.
-
-VIRGILIA:
-Beseech you, give me leave to retire myself.
-
-VOLUMNIA:
-Indeed, you shall not.
-Methinks I hear hither your husband's drum,
-See him pluck Aufidius down by the hair,
-As children from a bear, the Volsces shunning him:
-Methinks I see him stamp thus, and call thus:
-'Come on, you cowards! you were got in fear,
-Though you were born in Rome:' his bloody brow
-With his mail'd hand then wiping, forth he goes,
-Like to a harvest-man that's task'd to mow
-Or all or lose his hire.
-
-VIRGILIA:
-His bloody brow! O Jupiter, no blood!
-
-VOLUMNIA:
-Away, you fool! it more becomes a man
-Than gilt his trophy: the breasts of Hecuba,
-When she did suckle Hector, look'd not lovelier
-Than Hector's forehead when it spit forth blood
-At Grecian sword, contemning. Tell Valeria,
-We are fit to bid her welcome.
-
-VIRGILIA:
-Heavens bless my lord from fell Aufidius!
-
-VOLUMNIA:
-He'll beat Aufidius 'head below his knee
-And tread upon his neck.
-
-VALERIA:
-My ladies both, good day to you.
-
-VOLUMNIA:
-Sweet madam.
-
-VIRGILIA:
-I am glad to see your ladyship.
-
-VALERIA:
-How do you both? you are manifest house-keepers.
-What are you sewing here? A fine spot, in good
-faith. How does your little son?
-
-VIRGILIA:
-I thank your ladyship; well, good madam.
-
-VOLUMNIA:
-He had rather see the swords, and hear a drum, than
-look upon his school-master.
-
-VALERIA:
-O' my word, the father's son: I'll swear,'tis a
-very pretty boy. O' my troth, I looked upon him o'
-Wednesday half an hour together: has such a
-confirmed countenance. I saw him run after a gilded
-butterfly: and when he caught it, he let it go
-again; and after it again; and over and over he
-comes, and again; catched it again; or whether his
-fall enraged him, or how 'twas, he did so set his
-teeth and tear it; O, I warrant it, how he mammocked
-it!
-
-VOLUMNIA:
-One on 's father's moods.
-
-VALERIA:
-Indeed, la, 'tis a noble child.
-
-VIRGILIA:
-A crack, madam.
-
-VALERIA:
-Come, lay aside your stitchery; I must have you play
-the idle husewife with me this afternoon.
-
-VIRGILIA:
-No, good madam; I will not out of doors.
-
-VALERIA:
-Not out of doors!
-
-VOLUMNIA:
-She shall, she shall.
-
-VIRGILIA:
-Indeed, no, by your patience; I'll not over the
-threshold till my lord return from the wars.
-
-VALERIA:
-Fie, you confine yourself most unreasonably: come,
-you must go visit the good lady that lies in.
-
-VIRGILIA:
-I will wish her speedy strength, and visit her with
-my prayers; but I cannot go thither.
-
-VOLUMNIA:
-Why, I pray you?
-
-VIRGILIA:
-'Tis not to save labour, nor that I want love.
-
-VALERIA:
-You would be another Penelope: yet, they say, all
-the yarn she spun in Ulysses' absence did but fill
-Ithaca full of moths. Come; I would your cambric
-were sensible as your finger, that you might leave
-pricking it for pity. Come, you shall go with us.
-
-VIRGILIA:
-No, good madam, pardon me; indeed, I will not forth.
-
-VALERIA:
-In truth, la, go with me; and I'll tell you
-excellent news of your husband.
-
-VIRGILIA:
-O, good madam, there can be none yet.
-
-VALERIA:
-Verily, I do not jest with you; there came news from
-him last night.
-
-VIRGILIA:
-Indeed, madam?
-
-VALERIA:
-In earnest, it's true; I heard a senator speak it.
-Thus it is: the Volsces have an army forth; against
-whom Cominius the general is gone, with one part of
-our Roman power: your lord and Titus Lartius are set
-down before their city Corioli; they nothing doubt
-prevailing and to make it brief wars. This is true,
-on mine honour; and so, I pray, go with us.
-
-VIRGILIA:
-Give me excuse, good madam; I will obey you in every
-thing hereafter.
-
-VOLUMNIA:
-Let her alone, lady: as she is now, she will but
-disease our better mirth.
-
-VALERIA:
-In troth, I think she would. Fare you well, then.
-Come, good sweet lady. Prithee, Virgilia, turn thy
-solemness out o' door. and go along with us.
-
-VIRGILIA:
-No, at a word, madam; indeed, I must not. I wish
-you much mirth.
-
-VALERIA:
-Well, then, farewell.
-
-MARCIUS:
-Yonder comes news. A wager they have met.
-
-LARTIUS:
-My horse to yours, no.
-
-MARCIUS:
-'Tis done.
-
-LARTIUS:
-Agreed.
-
-MARCIUS:
-Say, has our general met the enemy?
-
-Messenger:
-They lie in view; but have not spoke as yet.
-
-LARTIUS:
-So, the good horse is mine.
-
-MARCIUS:
-I'll buy him of you.
-
-LARTIUS:
-No, I'll nor sell nor give him: lend you him I will
-For half a hundred years. Summon the town.
-
-MARCIUS:
-How far off lie these armies?
-
-Messenger:
-Within this mile and half.
-
-MARCIUS:
-Then shall we hear their 'larum, and they ours.
-Now, Mars, I prithee, make us quick in work,
-That we with smoking swords may march from hence,
-To help our fielded friends! Come, blow thy blast.
-Tutus Aufidius, is he within your walls?
-
-First Senator:
-No, nor a man that fears you less than he,
-That's lesser than a little.
-Hark! our drums
-Are bringing forth our youth. We'll break our walls,
-Rather than they shall pound us up: our gates,
-Which yet seem shut, we, have but pinn'd with rushes;
-They'll open of themselves.
-Hark you. far off!
-There is Aufidius; list, what work he makes
-Amongst your cloven army.
-
-MARCIUS:
-O, they are at it!
-
-LARTIUS:
-Their noise be our instruction. Ladders, ho!
-
-MARCIUS:
-They fear us not, but issue forth their city.
-Now put your shields before your hearts, and fight
-With hearts more proof than shields. Advance,
-brave Titus:
-They do disdain us much beyond our thoughts,
-Which makes me sweat with wrath. Come on, my fellows:
-He that retires I'll take him for a Volsce,
-And he shall feel mine edge.
-
-MARCIUS:
-All the contagion of the south light on you,
-You shames of Rome! you herd of--Boils and plagues
-Plaster you o'er, that you may be abhorr'd
-Further than seen and one infect another
-Against the wind a mile! You souls of geese,
-That bear the shapes of men, how have you run
-From slaves that apes would beat! Pluto and hell!
-All hurt behind; backs red, and faces pale
-With flight and agued fear! Mend and charge home,
-Or, by the fires of heaven, I'll leave the foe
-And make my wars on you: look to't: come on;
-If you'll stand fast, we'll beat them to their wives,
-As they us to our trenches followed.
-So, now the gates are ope: now prove good seconds:
-'Tis for the followers fortune widens them,
-Not for the fliers: mark me, and do the like.
-
-First Soldier:
-Fool-hardiness; not I.
-
-Second Soldier:
-Nor I.
-
-First Soldier:
-See, they have shut him in.
-
-All:
-To the pot, I warrant him.
-
-LARTIUS:
-What is become of Marcius?
-
-All:
-Slain, sir, doubtless.
-
-First Soldier:
-Following the fliers at the very heels,
-With them he enters; who, upon the sudden,
-Clapp'd to their gates: he is himself alone,
-To answer all the city.
-
-LARTIUS:
-O noble fellow!
-Who sensibly outdares his senseless sword,
-And, when it bows, stands up. Thou art left, Marcius:
-A carbuncle entire, as big as thou art,
-Were not so rich a jewel. Thou wast a soldier
-Even to Cato's wish, not fierce and terrible
-Only in strokes; but, with thy grim looks and
-The thunder-like percussion of thy sounds,
-Thou madst thine enemies shake, as if the world
-Were feverous and did tremble.
-
-First Soldier:
-Look, sir.
-
-LARTIUS:
-O,'tis Marcius!
-Let's fetch him off, or make remain alike.
-
-First Roman:
-This will I carry to Rome.
-
-Second Roman:
-And I this.
-
-Third Roman:
-A murrain on't! I took this for silver.
-
-MARCIUS:
-See here these movers that do prize their hours
-At a crack'd drachm! Cushions, leaden spoons,
-Irons of a doit, doublets that hangmen would
-Bury with those that wore them, these base slaves,
-Ere yet the fight be done, pack up: down with them!
-And hark, what noise the general makes! To him!
-There is the man of my soul's hate, Aufidius,
-Piercing our Romans: then, valiant Titus, take
-Convenient numbers to make good the city;
-Whilst I, with those that have the spirit, will haste
-To help Cominius.
-
-LARTIUS:
-Worthy sir, thou bleed'st;
-Thy exercise hath been too violent for
-A second course of fight.
-
-MARCIUS:
-Sir, praise me not;
-My work hath yet not warm'd me: fare you well:
-The blood I drop is rather physical
-Than dangerous to me: to Aufidius thus
-I will appear, and fight.
-
-LARTIUS:
-Now the fair goddess, Fortune,
-Fall deep in love with thee; and her great charms
-Misguide thy opposers' swords! Bold gentleman,
-Prosperity be thy page!
-
-MARCIUS:
-Thy friend no less
-Than those she placeth highest! So, farewell.
-
-LARTIUS:
-Thou worthiest Marcius!
-Go, sound thy trumpet in the market-place;
-Call thither all the officers o' the town,
-Where they shall know our mind: away!
-
-COMINIUS:
-Breathe you, my friends: well fought;
-we are come off
-Like Romans, neither foolish in our stands,
-Nor cowardly in retire: believe me, sirs,
-We shall be charged again. Whiles we have struck,
-By interims and conveying gusts we have heard
-The charges of our friends. Ye Roman gods!
-Lead their successes as we wish our own,
-That both our powers, with smiling
-fronts encountering,
-May give you thankful sacrifice.
-Thy news?
-
-Messenger:
-The citizens of Corioli have issued,
-And given to Lartius and to Marcius battle:
-I saw our party to their trenches driven,
-And then I came away.
-
-COMINIUS:
-Though thou speak'st truth,
-Methinks thou speak'st not well.
-How long is't since?
-
-Messenger:
-Above an hour, my lord.
-
-COMINIUS:
-'Tis not a mile; briefly we heard their drums:
-How couldst thou in a mile confound an hour,
-And bring thy news so late?
-
-Messenger:
-Spies of the Volsces
-Held me in chase, that I was forced to wheel
-Three or four miles about, else had I, sir,
-Half an hour since brought my report.
-
-COMINIUS:
-Who's yonder,
-That does appear as he were flay'd? O gods
-He has the stamp of Marcius; and I have
-Before-time seen him thus.
-
-MARCIUS:
-
-COMINIUS:
-The shepherd knows not thunder from a tabour
-More than I know the sound of Marcius' tongue
-From every meaner man.
-
-MARCIUS:
-Come I too late?
-
-COMINIUS:
-Ay, if you come not in the blood of others,
-But mantled in your own.
-
-MARCIUS:
-O, let me clip ye
-In arms as sound as when I woo'd, in heart
-As merry as when our nuptial day was done,
-And tapers burn'd to bedward!
-
-COMINIUS:
-Flower of warriors,
-How is it with Titus Lartius?
-
-MARCIUS:
-As with a man busied about decrees:
-Condemning some to death, and some to exile;
-Ransoming him, or pitying, threatening the other;
-Holding Corioli in the name of Rome,
-Even like a fawning greyhound in the leash,
-To let him slip at will.
-
-COMINIUS:
-Where is that slave
-Which told me they had beat you to your trenches?
-Where is he? call him hither.
-
-MARCIUS:
-Let him alone;
-He did inform the truth: but for our gentlemen,
-The common file--a plague! tribunes for them!--
-The mouse ne'er shunn'd the cat as they did budge
-From rascals worse than they.
-
-COMINIUS:
-But how prevail'd you?
-
-MARCIUS:
-Will the time serve to tell? I do not think.
-Where is the enemy? are you lords o' the field?
-If not, why cease you till you are so?
-
-COMINIUS:
-Marcius,
-We have at disadvantage fought and did
-Retire to win our purpose.
-
-MARCIUS:
-How lies their battle? know you on which side
-They have placed their men of trust?
-
-COMINIUS:
-As I guess, Marcius,
-Their bands i' the vaward are the Antiates,
-Of their best trust; o'er them Aufidius,
-Their very heart of hope.
-
-MARCIUS:
-I do beseech you,
-By all the battles wherein we have fought,
-By the blood we have shed together, by the vows
-We have made to endure friends, that you directly
-Set me against Aufidius and his Antiates;
-And that you not delay the present, but,
-Filling the air with swords advanced and darts,
-We prove this very hour.
-
-COMINIUS:
-Though I could wish
-You were conducted to a gentle bath
-And balms applied to, you, yet dare I never
-Deny your asking: take your choice of those
-That best can aid your action.
-
-MARCIUS:
-Those are they
-That most are willing. If any such be here--
-As it were sin to doubt--that love this painting
-Wherein you see me smear'd; if any fear
-Lesser his person than an ill report;
-If any think brave death outweighs bad life
-And that his country's dearer than himself;
-Let him alone, or so many so minded,
-Wave thus, to express his disposition,
-And follow Marcius.
-O, me alone! make you a sword of me?
-If these shows be not outward, which of you
-But is four Volsces? none of you but is
-Able to bear against the great Aufidius
-A shield as hard as his. A certain number,
-Though thanks to all, must I select
-from all: the rest
-Shall bear the business in some other fight,
-As cause will be obey'd. Please you to march;
-And four shall quickly draw out my command,
-Which men are best inclined.
-
-COMINIUS:
-March on, my fellows:
-Make good this ostentation, and you shall
-Divide in all with us.
-
-LARTIUS:
-So, let the ports be guarded: keep your duties,
-As I have set them down. If I do send, dispatch
-Those centuries to our aid: the rest will serve
-For a short holding: if we lose the field,
-We cannot keep the town.
-
-Lieutenant:
-Fear not our care, sir.
-
-LARTIUS:
-Hence, and shut your gates upon's.
-Our guider, come; to the Roman camp conduct us.
-
-MARCIUS:
-I'll fight with none but thee; for I do hate thee
-Worse than a promise-breaker.
-
-AUFIDIUS:
-We hate alike:
-Not Afric owns a serpent I abhor
-More than thy fame and envy. Fix thy foot.
-
-MARCIUS:
-Let the first budger die the other's slave,
-And the gods doom him after!

tokenizer.py

@@ -0,0 +1,75 @@
+import pickle
+import regex as re
+from typing import List, Tuple
+
+class HindiTokenizer:
+    def __init__(self, model_path: str = 'bpe_results.pkl'):
+        # Load the BPE model
+        with open(model_path, 'rb') as f:
+            self.merges, self.ids, self.num_merges = pickle.load(f)
+            
+        # Initialize vocabulary
+        self.vocab = {idx: bytes([idx]) for idx in range(256)}
+        for (p0, p1), idx in self.merges.items():
+            self.vocab[idx] = self.vocab[p0] + self.vocab[p1]
+            
+        # Hindi-focused pattern
+        self.pattern = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{N}+| ?(?:[\u0904-\u0939\u093d-\u093d\u0950-\u0950\u0958-\u0961\u0970-\u097f\ua8f2-\ua8fe\U00011b00-\U00011b09\u1cd3-\u1cd3\u1ce9-\u1cec\u1cee-\u1cf3\u1cf5-\u1cf6\u1cfa-\u1cfa][\u0900-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962-\u0963\ua8e0-\ua8f1\ua8ff-\ua8ff\u1cd0-\u1cd2\u1cd4-\u1ce8\u1ced-\u1ced\u1cf4-\u1cf4\u1cf7-\u1cf9]*)+| ?\p{L}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
+    
+    def tokenize(self, text: str) -> Tuple[List[int], List[str], List[str]]:
+        # Get initial tokens using regex
+        tokens = re.findall(self.pattern, text)
+        
+        # Convert tokens to byte sequences and maintain grouping
+        byte_tokens = [token.encode('utf-8') for token in tokens]
+        token_list = [list(token) for token in byte_tokens]
+        
+        # Process each token
+        final_tokens = []
+        for token in token_list:
+            current_token = list(token)
+            while len(current_token) >= 2:
+                stats = self._get_stats([current_token])
+                if not stats:
+                    break
+                pair = min(stats, key=lambda p: self.merges.get(p, float("inf")))
+                if pair not in self.merges:
+                    break
+                idx = self.merges[pair]
+                current_token = self._merge([current_token], pair, idx)[0]
+            final_tokens.extend(current_token)
+        
+        # Decode the tokens
+        decoded_tokens = [self.vocab[idx].decode("utf-8", errors="replace") for idx in final_tokens]
+        
+        return final_tokens, tokens, decoded_tokens
+    
+    def _get_stats(self, token_list):
+        """Count frequency of pairs across all tokens"""
+        counts = {}
+        for token in token_list:
+            if len(token) < 2:
+                continue
+            for pair in zip(token, token[1:]):
+                counts[pair] = counts.get(pair, 0) + 1
+        return counts
+    
+    def _merge(self, token_list, pair, idx):
+        """Merge all occurrences of pair within each token"""
+        newids = []
+        for token in token_list:
+            if len(token) < 2:
+                newids.append(token)
+                continue
+                
+            new_token = []
+            i = 0
+            while i < len(token):
+                if i < len(token) - 1 and (token[i], token[i+1]) == pair:
+                    new_token.append(idx)
+                    i += 2
+                else:
+                    new_token.append(token[i])
+                    i += 1
+            newids.append(new_token)
+        return newids