pragya-e2 / tokenizer.vocab

Push model artifacts + checkpoints (whitelisted)

bdf0b03 verified 5 days ago

5.04 kB

<unk>	0
<pad>	0
<s>	0
</s>	0
<0x00>	0
<0x01>	0
<0x02>	0
<0x03>	0
<0x04>	0
<0x05>	0
<0x06>	0
<0x07>	0
<0x08>	0
<0x09>	0
<0x0A>	0
<0x0B>	0
<0x0C>	0
<0x0D>	0
<0x0E>	0
<0x0F>	0
<0x10>	0
<0x11>	0
<0x12>	0
<0x13>	0
<0x14>	0
<0x15>	0
<0x16>	0
<0x17>	0
<0x18>	0
<0x19>	0
<0x1A>	0
<0x1B>	0
<0x1C>	0
<0x1D>	0
<0x1E>	0
<0x1F>	0
<0x20>	0
<0x21>	0
<0x22>	0
<0x23>	0
<0x24>	0
<0x25>	0
<0x26>	0
<0x27>	0
<0x28>	0
<0x29>	0
<0x2A>	0
<0x2B>	0
<0x2C>	0
<0x2D>	0
<0x2E>	0
<0x2F>	0
<0x30>	0
<0x31>	0
<0x32>	0
<0x33>	0
<0x34>	0
<0x35>	0
<0x36>	0
<0x37>	0
<0x38>	0
<0x39>	0
<0x3A>	0
<0x3B>	0
<0x3C>	0
<0x3D>	0
<0x3E>	0
<0x3F>	0
<0x40>	0
<0x41>	0
<0x42>	0
<0x43>	0
<0x44>	0
<0x45>	0
<0x46>	0
<0x47>	0
<0x48>	0
<0x49>	0
<0x4A>	0
<0x4B>	0
<0x4C>	0
<0x4D>	0
<0x4E>	0
<0x4F>	0
<0x50>	0
<0x51>	0
<0x52>	0
<0x53>	0
<0x54>	0
<0x55>	0
<0x56>	0
<0x57>	0
<0x58>	0
<0x59>	0
<0x5A>	0
<0x5B>	0
<0x5C>	0
<0x5D>	0
<0x5E>	0
<0x5F>	0
<0x60>	0
<0x61>	0
<0x62>	0
<0x63>	0
<0x64>	0
<0x65>	0
<0x66>	0
<0x67>	0
<0x68>	0
<0x69>	0
<0x6A>	0
<0x6B>	0
<0x6C>	0
<0x6D>	0
<0x6E>	0
<0x6F>	0
<0x70>	0
<0x71>	0
<0x72>	0
<0x73>	0
<0x74>	0
<0x75>	0
<0x76>	0
<0x77>	0
<0x78>	0
<0x79>	0
<0x7A>	0
<0x7B>	0
<0x7C>	0
<0x7D>	0
<0x7E>	0
<0x7F>	0
<0x80>	0
<0x81>	0
<0x82>	0
<0x83>	0
<0x84>	0
<0x85>	0
<0x86>	0
<0x87>	0
<0x88>	0
<0x89>	0
<0x8A>	0
<0x8B>	0
<0x8C>	0
<0x8D>	0
<0x8E>	0
<0x8F>	0
<0x90>	0
<0x91>	0
<0x92>	0
<0x93>	0
<0x94>	0
<0x95>	0
<0x96>	0
<0x97>	0
<0x98>	0
<0x99>	0
<0x9A>	0
<0x9B>	0
<0x9C>	0
<0x9D>	0
<0x9E>	0
<0x9F>	0
<0xA0>	0
<0xA1>	0
<0xA2>	0
<0xA3>	0
<0xA4>	0
<0xA5>	0
<0xA6>	0
<0xA7>	0
<0xA8>	0
<0xA9>	0
<0xAA>	0
<0xAB>	0
<0xAC>	0
<0xAD>	0
<0xAE>	0
<0xAF>	0
<0xB0>	0
<0xB1>	0
<0xB2>	0
<0xB3>	0
<0xB4>	0
<0xB5>	0
<0xB6>	0
<0xB7>	0
<0xB8>	0
<0xB9>	0
<0xBA>	0
<0xBB>	0
<0xBC>	0
<0xBD>	0
<0xBE>	0
<0xBF>	0
<0xC0>	0
<0xC1>	0
<0xC2>	0
<0xC3>	0
<0xC4>	0
<0xC5>	0
<0xC6>	0
<0xC7>	0
<0xC8>	0
<0xC9>	0
<0xCA>	0
<0xCB>	0
<0xCC>	0
<0xCD>	0
<0xCE>	0
<0xCF>	0
<0xD0>	0
<0xD1>	0
<0xD2>	0
<0xD3>	0
<0xD4>	0
<0xD5>	0
<0xD6>	0
<0xD7>	0
<0xD8>	0
<0xD9>	0
<0xDA>	0
<0xDB>	0
<0xDC>	0
<0xDD>	0
<0xDE>	0
<0xDF>	0
<0xE0>	0
<0xE1>	0
<0xE2>	0
<0xE3>	0
<0xE4>	0
<0xE5>	0
<0xE6>	0
<0xE7>	0
<0xE8>	0
<0xE9>	0
<0xEA>	0
<0xEB>	0
<0xEC>	0
<0xED>	0
<0xEE>	0
<0xEF>	0
<0xF0>	0
<0xF1>	0
<0xF2>	0
<0xF3>	0
<0xF4>	0
<0xF5>	0
<0xF6>	0
<0xF7>	0
<0xF8>	0
<0xF9>	0
<0xFA>	0
<0xFB>	0
<0xFC>	0
<0xFD>	0
<0xFE>	0
<0xFF>	0
he	-0
▁t	-1
an	-2
▁s	-3
in	-4
▁w	-5
ed	-6
▁the	-7
al	-8
on	-9
▁c	-10
er	-11
▁f	-12
and	-13
at	-14
▁b	-15
is	-16
▁a	-17
re	-18
it	-19
▁and	-20
ar	-21
▁o	-22
▁h	-23
es	-24
en	-25
▁l	-26
nd	-27
▁r	-28
▁m	-29
▁T	-30
▁p	-31
▁of	-32
▁is	-33
ing	-34
▁in	-35
▁I	-36
as	-37
ou	-38
or	-39
▁g	-40
am	-41
▁The	-42
ic	-43
▁d	-44
el	-45
her	-46
et	-47
▁Ind	-48
▁n	-49
▁Indi	-50
ig	-51
ol	-52
il	-53
ot	-54
▁to	-55
all	-56
ith	-57
ver	-58
▁India	-59
ay	-60
ow	-61
ac	-62
om	-63
▁st	-64
▁with	-65
ht	-66
▁A	-67
▁cl	-68
▁on	-69
ad	-70
ul	-71
▁M	-72
▁R	-73
ang	-74
ion	-75
▁S	-76
est	-77
ight	-78
ft	-79
ain	-80
▁br	-81
▁th	-82
▁his	-83
ly	-84
▁G	-85
ent	-86
▁fl	-87
▁fr	-88
▁gre	-89
ut	-90
we	-91
▁C	-92
▁D	-93
▁e	-94
res	-95
ation	-96
im	-97
le	-98
un	-99
▁sp	-100
weet	-101
▁wat	-102
▁sweet	-103
av	-104
ir	-105
pp	-106
▁B	-107
▁H	-108
ful	-109
old	-110
ong	-111
▁Ch	-112
▁by	-113
▁co	-114
ational	-115
ab	-116
ag	-117
ak	-118
ur	-119
arm	-120
ill	-121
ite	-122
its	-123
one	-124
ous	-125
▁an	-126
▁tr	-127
▁wh	-128
alled	-129
▁green	-130
▁called	-131
▁national	-132
ch	-133
iv	-134
lu	-135
▁N	-136
▁k	-137
ers	-138
ick	-139
rou	-140
▁for	-141
▁tre	-142
▁was	-143
▁warm	-144
ok	-145
th	-146
to	-147
us	-148
▁W	-149
▁y	-150
ell	-151
hed	-152
ter	-153
▁gr	-154
▁ri	-155
▁sh	-156
▁sm	-157
▁so	-158
pped	-159
▁fam	-160
▁from	-161
▁bright	-162
ds	-163
ld	-164
tu	-165
um	-166
ace	-167
ade	-168
lue	-169
oon	-170
ore	-171
und	-172
▁al	-173
▁be	-174
▁pe	-175
▁ro	-176
ands	-177
ated	-178
very	-179
▁its	-180
▁pot	-181
▁whe	-182
other	-183
▁blue	-184
▁water	-185
▁famous	-186
ew	-187
op	-188
os	-189
ov	-190
▁P	-191
▁v	-192
ack	-193
anc	-194
any	-195
▁	-196
e	-197
a	-198
t	-199
n	-200
i	-201
o	-202
r	-203
s	-204
h	-205
l	-206
d	-207
c	-208
g	-209
f	-210
u	-211
w	-212
m	-213
y	-214
p	-215
b	-216
.	-217
,	-218
k	-219
v	-220
T	-221
I	-222
A	-223
M	-224
R	-225
S	-226
G	-227
C	-228
D	-229
j	-230
B	-231
H	-232
N	-233
W	-234
P	-235
E	-236
K	-237
!	-238
J	-239
O	-240
2	-241
:	-242
F	-243
L	-244
q	-245
x	-246
z	-247
5	-248
8	-249
'	-250
3	-251

	<unk> 0
	<pad> 0
	<s> 0
	</s> 0
	<0x00> 0
	<0x01> 0
	<0x02> 0
	<0x03> 0
	<0x04> 0
	<0x05> 0
	<0x06> 0
	<0x07> 0
	<0x08> 0
	<0x09> 0
	<0x0A> 0
	<0x0B> 0
	<0x0C> 0
	<0x0D> 0
	<0x0E> 0
	<0x0F> 0
	<0x10> 0
	<0x11> 0
	<0x12> 0
	<0x13> 0
	<0x14> 0
	<0x15> 0
	<0x16> 0
	<0x17> 0
	<0x18> 0
	<0x19> 0
	<0x1A> 0
	<0x1B> 0
	<0x1C> 0
	<0x1D> 0
	<0x1E> 0
	<0x1F> 0
	<0x20> 0
	<0x21> 0
	<0x22> 0
	<0x23> 0
	<0x24> 0
	<0x25> 0
	<0x26> 0
	<0x27> 0
	<0x28> 0
	<0x29> 0
	<0x2A> 0
	<0x2B> 0
	<0x2C> 0
	<0x2D> 0
	<0x2E> 0
	<0x2F> 0
	<0x30> 0
	<0x31> 0
	<0x32> 0
	<0x33> 0
	<0x34> 0
	<0x35> 0
	<0x36> 0
	<0x37> 0
	<0x38> 0
	<0x39> 0
	<0x3A> 0
	<0x3B> 0
	<0x3C> 0
	<0x3D> 0
	<0x3E> 0
	<0x3F> 0
	<0x40> 0
	<0x41> 0
	<0x42> 0
	<0x43> 0
	<0x44> 0
	<0x45> 0
	<0x46> 0
	<0x47> 0
	<0x48> 0
	<0x49> 0
	<0x4A> 0
	<0x4B> 0
	<0x4C> 0
	<0x4D> 0
	<0x4E> 0
	<0x4F> 0
	<0x50> 0
	<0x51> 0
	<0x52> 0
	<0x53> 0
	<0x54> 0
	<0x55> 0
	<0x56> 0
	<0x57> 0
	<0x58> 0
	<0x59> 0
	<0x5A> 0
	<0x5B> 0
	<0x5C> 0
	<0x5D> 0
	<0x5E> 0
	<0x5F> 0
	<0x60> 0
	<0x61> 0
	<0x62> 0
	<0x63> 0
	<0x64> 0
	<0x65> 0
	<0x66> 0
	<0x67> 0
	<0x68> 0
	<0x69> 0
	<0x6A> 0
	<0x6B> 0
	<0x6C> 0
	<0x6D> 0
	<0x6E> 0
	<0x6F> 0
	<0x70> 0
	<0x71> 0
	<0x72> 0
	<0x73> 0
	<0x74> 0
	<0x75> 0
	<0x76> 0
	<0x77> 0
	<0x78> 0
	<0x79> 0
	<0x7A> 0
	<0x7B> 0
	<0x7C> 0
	<0x7D> 0
	<0x7E> 0
	<0x7F> 0
	<0x80> 0
	<0x81> 0
	<0x82> 0
	<0x83> 0
	<0x84> 0
	<0x85> 0
	<0x86> 0
	<0x87> 0
	<0x88> 0
	<0x89> 0
	<0x8A> 0
	<0x8B> 0
	<0x8C> 0
	<0x8D> 0
	<0x8E> 0
	<0x8F> 0
	<0x90> 0
	<0x91> 0
	<0x92> 0
	<0x93> 0
	<0x94> 0
	<0x95> 0
	<0x96> 0
	<0x97> 0
	<0x98> 0
	<0x99> 0
	<0x9A> 0
	<0x9B> 0
	<0x9C> 0
	<0x9D> 0
	<0x9E> 0
	<0x9F> 0
	<0xA0> 0
	<0xA1> 0
	<0xA2> 0
	<0xA3> 0
	<0xA4> 0
	<0xA5> 0
	<0xA6> 0
	<0xA7> 0
	<0xA8> 0
	<0xA9> 0
	<0xAA> 0
	<0xAB> 0
	<0xAC> 0
	<0xAD> 0
	<0xAE> 0
	<0xAF> 0
	<0xB0> 0
	<0xB1> 0
	<0xB2> 0
	<0xB3> 0
	<0xB4> 0
	<0xB5> 0
	<0xB6> 0
	<0xB7> 0
	<0xB8> 0
	<0xB9> 0
	<0xBA> 0
	<0xBB> 0
	<0xBC> 0
	<0xBD> 0
	<0xBE> 0
	<0xBF> 0
	<0xC0> 0
	<0xC1> 0
	<0xC2> 0
	<0xC3> 0
	<0xC4> 0
	<0xC5> 0
	<0xC6> 0
	<0xC7> 0
	<0xC8> 0
	<0xC9> 0
	<0xCA> 0
	<0xCB> 0
	<0xCC> 0
	<0xCD> 0
	<0xCE> 0
	<0xCF> 0
	<0xD0> 0
	<0xD1> 0
	<0xD2> 0
	<0xD3> 0
	<0xD4> 0
	<0xD5> 0
	<0xD6> 0
	<0xD7> 0
	<0xD8> 0
	<0xD9> 0
	<0xDA> 0
	<0xDB> 0
	<0xDC> 0
	<0xDD> 0
	<0xDE> 0
	<0xDF> 0
	<0xE0> 0
	<0xE1> 0
	<0xE2> 0
	<0xE3> 0
	<0xE4> 0
	<0xE5> 0
	<0xE6> 0
	<0xE7> 0
	<0xE8> 0
	<0xE9> 0
	<0xEA> 0
	<0xEB> 0
	<0xEC> 0
	<0xED> 0
	<0xEE> 0
	<0xEF> 0
	<0xF0> 0
	<0xF1> 0
	<0xF2> 0
	<0xF3> 0
	<0xF4> 0
	<0xF5> 0
	<0xF6> 0
	<0xF7> 0
	<0xF8> 0
	<0xF9> 0
	<0xFA> 0
	<0xFB> 0
	<0xFC> 0
	<0xFD> 0
	<0xFE> 0
	<0xFF> 0
	he -0
	▁t -1
	an -2
	▁s -3
	in -4
	▁w -5
	ed -6
	▁the -7
	al -8
	on -9
	▁c -10
	er -11
	▁f -12
	and -13
	at -14
	▁b -15
	is -16
	▁a -17
	re -18
	it -19
	▁and -20
	ar -21
	▁o -22
	▁h -23
	es -24
	en -25
	▁l -26
	nd -27
	▁r -28
	▁m -29
	▁T -30
	▁p -31
	▁of -32
	▁is -33
	ing -34
	▁in -35
	▁I -36
	as -37
	ou -38
	or -39
	▁g -40
	am -41
	▁The -42
	ic -43
	▁d -44
	el -45
	her -46
	et -47
	▁Ind -48
	▁n -49
	▁Indi -50
	ig -51
	ol -52
	il -53
	ot -54
	▁to -55
	all -56
	ith -57
	ver -58
	▁India -59
	ay -60
	ow -61
	ac -62
	om -63
	▁st -64
	▁with -65
	ht -66
	▁A -67
	▁cl -68
	▁on -69
	ad -70
	ul -71
	▁M -72
	▁R -73
	ang -74
	ion -75
	▁S -76
	est -77
	ight -78
	ft -79
	ain -80
	▁br -81
	▁th -82
	▁his -83
	ly -84
	▁G -85
	ent -86
	▁fl -87
	▁fr -88
	▁gre -89
	ut -90
	we -91
	▁C -92
	▁D -93
	▁e -94
	res -95
	ation -96
	im -97
	le -98
	un -99
	▁sp -100
	weet -101
	▁wat -102
	▁sweet -103
	av -104
	ir -105
	pp -106
	▁B -107
	▁H -108
	ful -109
	old -110
	ong -111
	▁Ch -112
	▁by -113
	▁co -114
	ational -115
	ab -116
	ag -117
	ak -118
	ur -119
	arm -120
	ill -121
	ite -122
	its -123
	one -124
	ous -125
	▁an -126
	▁tr -127
	▁wh -128
	alled -129
	▁green -130
	▁called -131
	▁national -132
	ch -133
	iv -134
	lu -135
	▁N -136
	▁k -137
	ers -138
	ick -139
	rou -140
	▁for -141
	▁tre -142
	▁was -143
	▁warm -144
	ok -145
	th -146
	to -147
	us -148
	▁W -149
	▁y -150
	ell -151
	hed -152
	ter -153
	▁gr -154
	▁ri -155
	▁sh -156
	▁sm -157
	▁so -158
	pped -159
	▁fam -160
	▁from -161
	▁bright -162
	ds -163
	ld -164
	tu -165
	um -166
	ace -167
	ade -168
	lue -169
	oon -170
	ore -171
	und -172
	▁al -173
	▁be -174
	▁pe -175
	▁ro -176
	ands -177
	ated -178
	very -179
	▁its -180
	▁pot -181
	▁whe -182
	other -183
	▁blue -184
	▁water -185
	▁famous -186
	ew -187
	op -188
	os -189
	ov -190
	▁P -191
	▁v -192
	ack -193
	anc -194
	any -195
	▁ -196
	e -197
	a -198
	t -199
	n -200
	i -201
	o -202
	r -203
	s -204
	h -205
	l -206
	d -207
	c -208
	g -209
	f -210
	u -211
	w -212
	m -213
	y -214
	p -215
	b -216
	. -217
	, -218
	k -219
	v -220
	T -221
	I -222
	A -223
	M -224
	R -225
	S -226
	G -227
	C -228
	D -229
	j -230
	B -231
	H -232
	N -233
	W -234
	P -235
	E -236
	K -237
	! -238
	J -239
	O -240
	2 -241
	: -242
	F -243
	L -244
	q -245
	x -246
	z -247
	5 -248
	8 -249
	' -250
	3 -251