Upload tokenizer

merges.txt

@@ -308,6 +308,7 @@ tr ing
 o k
 _ _
 c l
+# #
 Ġh e
 ar d
 ) .
@@ -562,6 +563,7 @@ i eld
 Ġp er
 C ont
 ing s
+## ##
 Ġd ata
 Ġs a
 e f
@@ -808,6 +810,7 @@ R es
 is e
 m in
 ĠĠĠĠ Ċ
+# include
 eth od
 . P
 ut e
@@ -1097,6 +1100,7 @@ pr int
 Ex ception
 in put
 Ġtr ans
+#### ####
 ord er
 B y
 Ġa w
@@ -1180,6 +1184,7 @@ if y
 Ġb reak
 Ġ );Ċ
 re n
+# define
 it t
 Ġa p
 ĉ c
@@ -2169,6 +2174,7 @@ s ource
 http s
 Ġd em
 Ġe ar
+######## ########
 Ġm atch
 or ies
 ac es
@@ -2390,6 +2396,7 @@ a ff
 ER R
 Ġ< =
 at ely
+# if
 u ction
 ĠT e
 Ġl ink
@@ -2399,6 +2406,7 @@ u ction
 m e
 Ġg iven
 O ut
+# endif
 Ġbet ter
 P age
 Ġfe el
@@ -4691,6 +4699,7 @@ c ard
 o ch
 _ image
 \ t
+# Ċ
 ( L
 Ġindu stry
 com ing
@@ -4760,6 +4769,7 @@ SS ION
 Ġcurrent ly
 se c
 Ġrelations hip
+################ ################
 ĠM ap
 as et
 Ġparam eters
@@ -5261,6 +5271,7 @@ l at
 ast e
 Ġpro file
 Ġread y
+#if ndef
 ro te
 Ġs ense
 G ener
@@ -5485,6 +5496,7 @@ ag ram
 Ġp en
 .b tn
 ĠA S
+#if def
 Ġcho ice
 ĠP age
 _P RO
@@ -5644,6 +5656,7 @@ b uf
 S cript
 d at
 Ġr ule
+# import
 =" /
 S erial
 Ġstart ing
@@ -6220,6 +6233,7 @@ C D
 ib ly
 .e quals
 Ġintern ational
+# pragma
 oo th
 W riter
 i ate
@@ -7739,6 +7753,7 @@ D on
 ĠH ot
 ĠInd ex
 ; &
+# !
 ĠN or
 ĠC ap
 - (
@@ -7893,6 +7908,7 @@ ant ic
 ĠApp le
 g in
 Re quired
+# +
 land s
 Ġs qu
 Ġfact or
@@ -8428,6 +8444,7 @@ C OM
 i ro
 Ġrem ote
 ĠI T
+#! /
 Ġred istrib
 ra z
 ĠS ince
@@ -9665,6 +9682,7 @@ ont o
 ĠJ ournal
 r outer
 Ġmys qli
+# else
 ) "
 -x s
 let s
@@ -11753,6 +11771,7 @@ R et
 -d anger
 Ġroom s
 con v
+# {
 . op
 ĠA rea
 _S C
@@ -12791,6 +12810,7 @@ Th ank
 Ġfact s
 Ġun t
 .in stance
+################################ ################################
 - end
 ĠJO IN
 ĠH en
@@ -13076,6 +13096,7 @@ id os
 ĠJew ish
 Ġre covery
 Ġstand s
+# [
 Ġafter noon
 ĠArt icle
 _ att
@@ -13611,6 +13632,7 @@ un ge
 err a
 Ċ ĠĠĊ
 utor ial
+# a
 p ay
 char ge
 ĠI re
@@ -14095,6 +14117,7 @@ im plementation
 Ġstre ets
 _M SG
 New s
+## #
 : /
 Ġcut ting
 x B
@@ -15910,6 +15933,7 @@ param eter
 gener ated
 Ġad mitted
 Ġp ussy
+# w
 Ġsw im
 un ion
 N a
@@ -15961,6 +15985,7 @@ _ div
 vel y
 } );čĊ
 _ ENT
+#+ #+
 art icles
 ĠSou thern
 Ġstrong er
@@ -17173,6 +17198,7 @@ _f ont
 Ġcharacter istics
 D one
 el n
+######## ####
 PO S
 Ġd ensity
 ĠPl atform
@@ -18177,6 +18203,7 @@ W T
 d og
 Ġclick ing
 ), ĊĊ
+# line
 Oper ator
 Ġc iv
 Ġm erg
@@ -19588,6 +19615,7 @@ _th eme
 Ġdemonstr ate
 ĠHow ard
 D rop
+# undef
 Ġinv oke
 ĠB ridge
 end en
@@ -22329,6 +22357,7 @@ L F
 ĠP ub
 Ġâ Ķ
 c ion
+## Ċ
 II I
 Tag Name
 Ġam id
@@ -23378,6 +23407,7 @@ w ers
 .m aterial
 ĠD ue
 ĠP el
+# print
 Ġindepend ence
 ex us
 Sh adow
@@ -26200,6 +26230,7 @@ p icture
 ĠFant asy
 st ory
 Ġm ême
+# ĊĊ
 _s lice
 olt age
 H ar
@@ -26552,6 +26583,7 @@ j query
 am ination
 [ @"
 Ġm uj
+## #Ċ
 First OrDefault
 then Return
 C he
@@ -26723,6 +26755,7 @@ enc oder
 ather ine
 ( dest
 az ed
+# endregion
 sem bl
 , M
 ob y
@@ -28559,6 +28592,7 @@ _S M
 ib o
 Ġlo yal
 Ġuse less
+# elif
 ĠUlt imate
 C ome
 g el
@@ -28766,6 +28800,7 @@ as il
 ation ally
 ĠMe eting
 è¯ ¯
+# region
 Ġrout ing
 .f ocus
 ĠYou th
@@ -29562,6 +29597,7 @@ index Of
 .prot ocol
 AF E
 Ġtext ures
+################################ ################
 umb ai
 .st ats
 ĠG E
@@ -31748,6 +31784,7 @@ k now
 .F ull
 . undefined
 ĠSequ elize
+################################################################ ############
 Ġeduc ated
 _O VER
 åº ı
@@ -33359,6 +33396,7 @@ M id
 dd s
 Ġdepos its
 ĉd river
+# ga
 pr ising
 print ln
 Ġpres enter
@@ -40191,6 +40229,7 @@ M ade
 ac ific
 .v olume
 Ġmir rors
+################################################################ ################
 Ġviol ate
 ars ers
 Ġsoc io
@@ -41048,6 +41087,7 @@ _N ode
 ar de
 Ġfier ce
 lic ted
+# čĊ
 Ġbreak through
 ĠE rik
 Ġb ride
@@ -43790,6 +43830,7 @@ _ Log
 rown ed
 _m ed
 ĉ date
+# __
 Ġcost umes
 ĠRe quires
 aff le
@@ -45854,6 +45895,7 @@ Progress HUD
 Ġmock ed
 ĠT ory
 Ġ") ";Ċ
+# g
 Ġli ed
 Ġs vc
 _g ui
@@ -47778,10 +47820,12 @@ Num ero
 d aily
 ĠC oding
 ( destination
+# $
 uj Äħ
 Ġemerg ence
 _p ara
 _IN CLUDE
+# :
 Ġrecogn izing
 Ġf ug
 "} },Ċ
@@ -48197,6 +48241,7 @@ b stract
 k p
 IT ES
 Ġ ################################################################
+# get
 / Desktop
 ĉgl m
 Ġz inc
@@ -50222,6 +50267,7 @@ _L AYER
 Ġdee pest
 w k
 ĠNo ise
+### ĊĊ
 Ġpr éc
 ot le
 ÑĤ е
@@ -52021,6 +52067,7 @@ C OR
 -effect ive
 Ġsk u
 ed ly
+## ĊĊ
 ĠH olly
 ĠFL ASH
 / TR
@@ -55239,6 +55286,7 @@ keep ers
 _s b
 om ens
 ĠExec utes
+# "
 TT Y
 ĠValue Type
 ); */Ċ
@@ -55638,6 +55686,7 @@ S cores
 E OS
 H al
 Ġtrust worthy
+# =
 .EX TRA
 Ġman o
 is icing
@@ -60472,6 +60521,7 @@ m om
 / place
 Ġhol istic
 : t
+# ,
 Ġb oto
 Ġnause a
 ĠSh ooting
@@ -60608,6 +60658,7 @@ IM ATION
 C ause
 at ypes
 man ufacturer
+# ----------------------------------------------------------------------------
 Ġsp or
 es on
 Ġpun ched
@@ -62217,6 +62268,7 @@ Set Branch
 S creenshot
 esthes ia
 Ġwalk er
+# from
 co ordinate
 _ interest
 Ġhelp less
@@ -63427,6 +63479,7 @@ Count s
 Field Type
 ok able
 ĠRT L
+# index
 Ġ% {
 Ġar ist
 .Get Mapping
@@ -65749,6 +65802,7 @@ tr uncate
 so on
 Ġdis fr
 ĉ Vec
+## _
 .s chool
 Ġbl inds
 Ġac ab
@@ -67020,6 +67074,7 @@ _IO CTL
 ĉex ec
 EN AME
 _ letters
+#### #
 ĠC s
 '] =="
 Ġ" ')
@@ -67221,6 +67276,7 @@ _ asm
 Ġtem pl
 ĠExp ense
 e ight
+# SBATCH
 z ones
 .p arts
 at rice
@@ -67555,6 +67611,7 @@ ra i
 i ard
 Theme Provider
 Ġevent Data
+# ad
 .get Url
 Ġtool box
 Ġover riding
@@ -68399,6 +68456,7 @@ _PRE SENT
 ĠTh eta
 _inter p
 R aster
+# error
 , obj
 Ġtweet ing
 _G PU
@@ -68746,6 +68804,7 @@ imm une
 Ġè¾ĵ åħ¥
 Ġintimid ation
 end ale
+################################################################ ########
 Ġinsight ful
 Ġs ands
 Ġphotograph ic
@@ -69151,6 +69210,7 @@ sh ed
 Dep loy
 .Con tract
 - bo
+# /
 Ġinter ception
 Ġis bn
 Ġman ners
@@ -69633,6 +69693,7 @@ PO CH
 \ \"
 - job
 Ġsevent y
+#### Ċ
 ĠMan or
 Ġdown right
 Ġtime frame
@@ -71438,6 +71499,7 @@ _BR ANCH
 /m essages
 Ġof Type
 ĉs wap
+# plt
 ĠTur ks
 N ES
 Ġprogress ively
@@ -71685,6 +71747,7 @@ L ik
 _T itle
 (G tk
 Ġc elle
+# ================================================================
 ĠJ oomla
 "> //
 Month ly
@@ -75190,6 +75253,7 @@ s olver
 ĠRevel ation
 Ġrap ide
 p unk
+# ----------------------------------------------------------------
 Object Id
 abin et
 extr acomment
@@ -75594,6 +75658,7 @@ h ari
 (c d
 Ġun zip
 Ġglam orous
+# ",
 Ġn aw
 Ġmin ib
 ĠBr an
@@ -75876,6 +75941,7 @@ Button Type
 Ġragaz za
 / tag
 Ġirres ist
+# End
 ****** *čĊ
 Ġrestr ained
 Ġch iropr
@@ -76013,6 +76079,7 @@ cast ing
 Ġgr âce
 ĠH elsinki
 G ro
+# af
 ìĭ Ŀ
 Ġsou ha
 ĠInd ie
@@ -76805,6 +76872,7 @@ x EE
 .M ESSAGE
 (de g
 å¿ Ĺ
+#### ##
 Ġ"" ),
 kl är
 \M ail
@@ -77019,6 +77087,7 @@ ren s
 / grid
 Ġfil thy
 .e v
+#### #Ċ
 Ġs are
 Ġso aking
 ĠReg ions
@@ -77259,6 +77328,7 @@ IFICATION S
 ` ;
 ãĥ ĭ
 cent e
+# ab
 Ġlex ical
 ĠB RO
 Ġr ulings
@@ -77367,6 +77437,7 @@ U PI
 ĉ props
 Ġrot ten
 Re jected
+# ac
 . ua
 ĠAm nesty
 Ġpenn ed
@@ -77709,6 +77780,7 @@ d pi
 cul os
 _person al
 Ġanaly tic
+# create
 _mem cpy
 (List Node
 _T ag
@@ -78626,6 +78698,7 @@ b anks
 ìŀħ ëĭĪëĭ¤
 Ġsid eline
 .Array Adapter
+#### ##Ċ
 ĠSy rians
 ĠAtt endance
 -es que
@@ -79118,6 +79191,7 @@ Image Url
 ALL ERY
 c j
 x AD
+############################################################################ ###Ċ
 Ġitalian i
 | #
 Ġreg enerate
@@ -80086,6 +80160,7 @@ op he
 _C ategory
 ĠBT N
 ĠDar th
+# for
 eth nic
 arch itecture
 ĠCou pe
@@ -81160,6 +81235,7 @@ _TRA IN
 Ġcooper ating
 ung i
 Ar izona
+# echo
 -ex pression
 .min utes
 Ġpref ixed
@@ -81691,6 +81767,7 @@ customer Id
 ĠMom ents
 enu ous
 IFIC ATE
+#### ###Ċ
 æĸĩ ç«ł
 á»į c
 orm sg
@@ -81738,6 +81815,7 @@ Environment al
 ĠIS IL
 Ġco op
 h ores
+# @
 Ġcomp el
 (s kip
 éĺ ³
@@ -81962,6 +82040,7 @@ ia ux
 Ġ-*- čĊ
 if es
 ĠM ansion
+# Region
 C ancellation
 Ġnear ing
 Ġl angu
@@ -82242,6 +82321,7 @@ M ismatch
 Ġw reak
 Mar vel
 /s l
+################################ ########
 Ġmov able
 Ñĥ й
 ĠDr inking
@@ -83817,6 +83897,7 @@ M iami
 ĠPal o
 ĠSuz anne
 _m appings
+#{ @
 ĠOccup ational
 _b uckets
 go als
@@ -83961,6 +84042,7 @@ so v
 -w atch
 ĠHosp itals
 } while
+################ ########
 á» £
 Ġakt ual
 Ġkil ograms
@@ -84082,6 +84164,7 @@ CH AN
 Ġcurrent Position
 ĠCaucas ian
 $ img
+# aa
 Ġse an
 M ess
 *= *=
@@ -85026,6 +85109,7 @@ py x
 ĠR udd
 ĠPres byterian
 und ler
+#! [
 _l ahir
 () =="
 Access ibility
@@ -87505,6 +87589,7 @@ C ouncil
 _AT OMIC
 Sub Menu
 _com press
+# .
 Dr v
 .push Button
 Ġsuit case
@@ -87563,6 +87648,7 @@ ch ine
 Ġe books
 ) size
 Ġspecial ised
+# ae
 Ġmich ael
 ĠSTD OUT
 ĠP ell
@@ -87741,6 +87827,7 @@ t ournament
 ĠEisen hower
 File d
 Ġhe bt
+# \
 create QueryBuilder
 æľī æķĪ
 v anced
@@ -87756,6 +87843,7 @@ Inst aller
 ol vable
 Ġal as
 (sp ell
+################ ############
 Ġdef amation
 ( Arg
 Ġuser Details
@@ -89693,6 +89781,7 @@ ict ory
 Ġsup p
 pe on
 v rier
+################################################ ########
 Ġtrot z
 Ġmel tdown
 ark ers
@@ -90384,6 +90473,7 @@ Gr an
 ser ter
 Ġstore front
 _ ends
+################################################################################ Ċ
 ĉg it
 D SP
 CH AIN
@@ -91906,6 +91996,7 @@ arg ed
 ĠST ILL
 Qualified Name
 . RES
+# c
 .w riteln
 ĠImmutable List
 ĠTh umb
@@ -92887,6 +92978,7 @@ Sc ala
 in ne
 ĠCONST ANTS
 _P rice
+# %%
 Ġar sch
 ĠNS AttributedString
 ĠFile Type
@@ -93833,6 +93925,7 @@ _TRI ANGLE
 _P ICTURE
 _OR IENTATION
 ĠO PP
+# '
 áf ico
 .h istogram
 ĠB enny
@@ -94103,6 +94196,7 @@ Owner Id
 ew n
 Ġpop ulous
 ĠSh ed
+# ${
 ĠA lo
 Device Info
 (IN VOKE
@@ -97493,6 +97587,7 @@ F el
 ĠD ISTRIBUT
 é ments
 ĠValid ates
+################################################ ############
 Ġ| /
 Ġes l
 Ġré seau
@@ -97512,6 +97607,7 @@ Ne o
 ime i
 .get Bounds
 .Mouse EventHandler
+#### ###
 Generic Type
 /c ms
 Ġturn o

tokenizer.json

@@ -66,22 +66,22 @@
       {
         "type": "ByteLevel",
         "add_prefix_space": false,
-        "trim_offsets": true,
+        "trim_offsets": false,
         "use_regex": false
       }
     ]
   },
   "post_processor": {
     "type": "ByteLevel",
-    "add_prefix_space": true,
+    "add_prefix_space": false,
     "trim_offsets": false,
-    "use_regex": true
+    "use_regex": false
   },
   "decoder": {
     "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
   },
   "model": {
     "type": "BPE",
@@ -152046,6 +152046,7 @@
       "o k",
       "_ _",
       "c l",
+      "# #",
       "Ġh e",
       "ar d",
       ") .",
@@ -152300,6 +152301,7 @@
       "Ġp er",
       "C ont",
       "ing s",
+      "## ##",
       "Ġd ata",
       "Ġs a",
       "e f",
@@ -152546,6 +152548,7 @@
       "is e",
       "m in",
       "ĠĠĠĠ Ċ",
+      "# include",
       "eth od",
       ". P",
       "ut e",
@@ -152835,6 +152838,7 @@
       "Ex ception",
       "in put",
       "Ġtr ans",
+      "#### ####",
       "ord er",
       "B y",
       "Ġa w",
@@ -152918,6 +152922,7 @@
       "Ġb reak",
       "Ġ );Ċ",
       "re n",
+      "# define",
       "it t",
       "Ġa p",
       "ĉ c",
@@ -153907,6 +153912,7 @@
       "http s",
       "Ġd em",
       "Ġe ar",
+      "######## ########",
       "Ġm atch",
       "or ies",
       "ac es",
@@ -154128,6 +154134,7 @@
       "ER R",
       "Ġ< =",
       "at ely",
+      "# if",
       "u ction",
       "ĠT e",
       "Ġl ink",
@@ -154137,6 +154144,7 @@
       "m e",
       "Ġg iven",
       "O ut",
+      "# endif",
       "Ġbet ter",
       "P age",
       "Ġfe el",
@@ -156429,6 +156437,7 @@
       "o ch",
       "_ image",
       "\\ t",
+      "# Ċ",
       "( L",
       "Ġindu stry",
       "com ing",
@@ -156498,6 +156507,7 @@
       "Ġcurrent ly",
       "se c",
       "Ġrelations hip",
+      "################ ################",
       "ĠM ap",
       "as et",
       "Ġparam eters",
@@ -156999,6 +157009,7 @@
       "ast e",
       "Ġpro file",
       "Ġread y",
+      "#if ndef",
       "ro te",
       "Ġs ense",
       "G ener",
@@ -157223,6 +157234,7 @@
       "Ġp en",
       ".b tn",
       "ĠA S",
+      "#if def",
       "Ġcho ice",
       "ĠP age",
       "_P RO",
@@ -157382,6 +157394,7 @@
       "S cript",
       "d at",
       "Ġr ule",
+      "# import",
       "=\" /",
       "S erial",
       "Ġstart ing",
@@ -157958,6 +157971,7 @@
       "ib ly",
       ".e quals",
       "Ġintern ational",
+      "# pragma",
       "oo th",
       "W riter",
       "i ate",
@@ -159477,6 +159491,7 @@
       "ĠH ot",
       "ĠInd ex",
       "; &",
+      "# !",
       "ĠN or",
       "ĠC ap",
       "- (",
@@ -159631,6 +159646,7 @@
       "ĠApp le",
       "g in",
       "Re quired",
+      "# +",
       "land s",
       "Ġs qu",
       "Ġfact or",
@@ -160166,6 +160182,7 @@
       "i ro",
       "Ġrem ote",
       "ĠI T",
+      "#! /",
       "Ġred istrib",
       "ra z",
       "ĠS ince",
@@ -161403,6 +161420,7 @@
       "ĠJ ournal",
       "r outer",
       "Ġmys qli",
+      "# else",
       ") \"",
       "-x s",
       "let s",
@@ -163491,6 +163509,7 @@
       "-d anger",
       "Ġroom s",
       "con v",
+      "# {",
       ". op",
       "ĠA rea",
       "_S C",
@@ -164529,6 +164548,7 @@
       "Ġfact s",
       "Ġun t",
       ".in stance",
+      "################################ ################################",
       "- end",
       "ĠJO IN",
       "ĠH en",
@@ -164814,6 +164834,7 @@
       "ĠJew ish",
       "Ġre covery",
       "Ġstand s",
+      "# [",
       "Ġafter noon",
       "ĠArt icle",
       "_ att",
@@ -165349,6 +165370,7 @@
       "err a",
       "Ċ ĠĠĊ",
       "utor ial",
+      "# a",
       "p ay",
       "char ge",
       "ĠI re",
@@ -165833,6 +165855,7 @@
       "Ġstre ets",
       "_M SG",
       "New s",
+      "## #",
       ": /",
       "Ġcut ting",
       "x B",
@@ -167648,6 +167671,7 @@
       "gener ated",
       "Ġad mitted",
       "Ġp ussy",
+      "# w",
       "Ġsw im",
       "un ion",
       "N a",
@@ -167699,6 +167723,7 @@
       "vel y",
       "} );čĊ",
       "_ ENT",
+      "#+ #+",
       "art icles",
       "ĠSou thern",
       "Ġstrong er",
@@ -168911,6 +168936,7 @@
       "Ġcharacter istics",
       "D one",
       "el n",
+      "######## ####",
       "PO S",
       "Ġd ensity",
       "ĠPl atform",
@@ -169915,6 +169941,7 @@
       "d og",
       "Ġclick ing",
       "), ĊĊ",
+      "# line",
       "Oper ator",
       "Ġc iv",
       "Ġm erg",
@@ -171326,6 +171353,7 @@
       "Ġdemonstr ate",
       "ĠHow ard",
       "D rop",
+      "# undef",
       "Ġinv oke",
       "ĠB ridge",
       "end en",
@@ -174067,6 +174095,7 @@
       "ĠP ub",
       "Ġâ Ķ",
       "c ion",
+      "## Ċ",
       "II I",
       "Tag Name",
       "Ġam id",
@@ -175116,6 +175145,7 @@
       ".m aterial",
       "ĠD ue",
       "ĠP el",
+      "# print",
       "Ġindepend ence",
       "ex us",
       "Sh adow",
@@ -177938,6 +177968,7 @@
       "ĠFant asy",
       "st ory",
       "Ġm ême",
+      "# ĊĊ",
       "_s lice",
       "olt age",
       "H ar",
@@ -178290,6 +178321,7 @@
       "am ination",
       "[ @\"",
       "Ġm uj",
+      "## #Ċ",
       "First OrDefault",
       "then Return",
       "C he",
@@ -178461,6 +178493,7 @@
       "ather ine",
       "( dest",
       "az ed",
+      "# endregion",
       "sem bl",
       ", M",
       "ob y",
@@ -180297,6 +180330,7 @@
       "ib o",
       "Ġlo yal",
       "Ġuse less",
+      "# elif",
       "ĠUlt imate",
       "C ome",
       "g el",
@@ -180504,6 +180538,7 @@
       "ation ally",
       "ĠMe eting",
       "è¯ ¯",
+      "# region",
       "Ġrout ing",
       ".f ocus",
       "ĠYou th",
@@ -181300,6 +181335,7 @@
       ".prot ocol",
       "AF E",
       "Ġtext ures",
+      "################################ ################",
       "umb ai",
       ".st ats",
       "ĠG E",
@@ -183486,6 +183522,7 @@
       ".F ull",
       ". undefined",
       "ĠSequ elize",
+      "################################################################ ############",
       "Ġeduc ated",
       "_O VER",
       "åº ı",
@@ -185097,6 +185134,7 @@
       "dd s",
       "Ġdepos its",
       "ĉd river",
+      "# ga",
       "pr ising",
       "print ln",
       "Ġpres enter",
@@ -191929,6 +191967,7 @@
       "ac ific",
       ".v olume",
       "Ġmir rors",
+      "################################################################ ################",
       "Ġviol ate",
       "ars ers",
       "Ġsoc io",
@@ -192786,6 +192825,7 @@
       "ar de",
       "Ġfier ce",
       "lic ted",
+      "# čĊ",
       "Ġbreak through",
       "ĠE rik",
       "Ġb ride",
@@ -195528,6 +195568,7 @@
       "rown ed",
       "_m ed",
       "ĉ date",
+      "# __",
       "Ġcost umes",
       "ĠRe quires",
       "aff le",
@@ -197592,6 +197633,7 @@
       "Ġmock ed",
       "ĠT ory",
       "Ġ\") \";Ċ",
+      "# g",
       "Ġli ed",
       "Ġs vc",
       "_g ui",
@@ -199516,10 +199558,12 @@
       "d aily",
       "ĠC oding",
       "( destination",
+      "# $",
       "uj Äħ",
       "Ġemerg ence",
       "_p ara",
       "_IN CLUDE",
+      "# :",
       "Ġrecogn izing",
       "Ġf ug",
       "\"} },Ċ",
@@ -199935,6 +199979,7 @@
       "k p",
       "IT ES",
       "Ġ ################################################################",
+      "# get",
       "/ Desktop",
       "ĉgl m",
       "Ġz inc",
@@ -201960,6 +202005,7 @@
       "Ġdee pest",
       "w k",
       "ĠNo ise",
+      "### ĊĊ",
       "Ġpr éc",
       "ot le",
       "ÑĤ е",
@@ -203759,6 +203805,7 @@
       "-effect ive",
       "Ġsk u",
       "ed ly",
+      "## ĊĊ",
       "ĠH olly",
       "ĠFL ASH",
       "/ TR",
@@ -206977,6 +207024,7 @@
       "_s b",
       "om ens",
       "ĠExec utes",
+      "# \"",
       "TT Y",
       "ĠValue Type",
       "); */Ċ",
@@ -207376,6 +207424,7 @@
       "E OS",
       "H al",
       "Ġtrust worthy",
+      "# =",
       ".EX TRA",
       "Ġman o",
       "is icing",
@@ -212210,6 +212259,7 @@
       "/ place",
       "Ġhol istic",
       ": t",
+      "# ,",
       "Ġb oto",
       "Ġnause a",
       "ĠSh ooting",
@@ -212346,6 +212396,7 @@
       "C ause",
       "at ypes",
       "man ufacturer",
+      "# ----------------------------------------------------------------------------",
       "Ġsp or",
       "es on",
       "Ġpun ched",
@@ -213955,6 +214006,7 @@
       "S creenshot",
       "esthes ia",
       "Ġwalk er",
+      "# from",
       "co ordinate",
       "_ interest",
       "Ġhelp less",
@@ -215165,6 +215217,7 @@
       "Field Type",
       "ok able",
       "ĠRT L",
+      "# index",
       "Ġ% {",
       "Ġar ist",
       ".Get Mapping",
@@ -217487,6 +217540,7 @@
       "so on",
       "Ġdis fr",
       "ĉ Vec",
+      "## _",
       ".s chool",
       "Ġbl inds",
       "Ġac ab",
@@ -218758,6 +218812,7 @@
       "ĉex ec",
       "EN AME",
       "_ letters",
+      "#### #",
       "ĠC s",
       "'] ==\"",
       "Ġ\" ')",
@@ -218959,6 +219014,7 @@
       "Ġtem pl",
       "ĠExp ense",
       "e ight",
+      "# SBATCH",
       "z ones",
       ".p arts",
       "at rice",
@@ -219293,6 +219349,7 @@
       "i ard",
       "Theme Provider",
       "Ġevent Data",
+      "# ad",
       ".get Url",
       "Ġtool box",
       "Ġover riding",
@@ -220137,6 +220194,7 @@
       "ĠTh eta",
       "_inter p",
       "R aster",
+      "# error",
       ", obj",
       "Ġtweet ing",
       "_G PU",
@@ -220484,6 +220542,7 @@
       "Ġè¾ĵ åħ¥",
       "Ġintimid ation",
       "end ale",
+      "################################################################ ########",
       "Ġinsight ful",
       "Ġs ands",
       "Ġphotograph ic",
@@ -220889,6 +220948,7 @@
       "Dep loy",
       ".Con tract",
       "- bo",
+      "# /",
       "Ġinter ception",
       "Ġis bn",
       "Ġman ners",
@@ -221371,6 +221431,7 @@
       "\\ \\\"",
       "- job",
       "Ġsevent y",
+      "#### Ċ",
       "ĠMan or",
       "Ġdown right",
       "Ġtime frame",
@@ -223176,6 +223237,7 @@
       "/m essages",
       "Ġof Type",
       "ĉs wap",
+      "# plt",
       "ĠTur ks",
       "N ES",
       "Ġprogress ively",
@@ -223423,6 +223485,7 @@
       "_T itle",
       "(G tk",
       "Ġc elle",
+      "# ================================================================",
       "ĠJ oomla",
       "\"> //",
       "Month ly",
@@ -226928,6 +226991,7 @@
       "ĠRevel ation",
       "Ġrap ide",
       "p unk",
+      "# ----------------------------------------------------------------",
       "Object Id",
       "abin et",
       "extr acomment",
@@ -227332,6 +227396,7 @@
       "(c d",
       "Ġun zip",
       "Ġglam orous",
+      "# \",",
       "Ġn aw",
       "Ġmin ib",
       "ĠBr an",
@@ -227614,6 +227679,7 @@
       "Ġragaz za",
       "/ tag",
       "Ġirres ist",
+      "# End",
       "****** *čĊ",
       "Ġrestr ained",
       "Ġch iropr",
@@ -227751,6 +227817,7 @@
       "Ġgr âce",
       "ĠH elsinki",
       "G ro",
+      "# af",
       "ìĭ Ŀ",
       "Ġsou ha",
       "ĠInd ie",
@@ -228543,6 +228610,7 @@
       ".M ESSAGE",
       "(de g",
       "å¿ Ĺ",
+      "#### ##",
       "Ġ\"\" ),",
       "kl är",
       "\\M ail",
@@ -228757,6 +228825,7 @@
       "/ grid",
       "Ġfil thy",
       ".e v",
+      "#### #Ċ",
       "Ġs are",
       "Ġso aking",
       "ĠReg ions",
@@ -228997,6 +229066,7 @@
       "` ;",
       "ãĥ ĭ",
       "cent e",
+      "# ab",
       "Ġlex ical",
       "ĠB RO",
       "Ġr ulings",
@@ -229105,6 +229175,7 @@
       "ĉ props",
       "Ġrot ten",
       "Re jected",
+      "# ac",
       ". ua",
       "ĠAm nesty",
       "Ġpenn ed",
@@ -229447,6 +229518,7 @@
       "cul os",
       "_person al",
       "Ġanaly tic",
+      "# create",
       "_mem cpy",
       "(List Node",
       "_T ag",
@@ -230364,6 +230436,7 @@
       "ìŀħ ëĭĪëĭ¤",
       "Ġsid eline",
       ".Array Adapter",
+      "#### ##Ċ",
       "ĠSy rians",
       "ĠAtt endance",
       "-es que",
@@ -230856,6 +230929,7 @@
       "ALL ERY",
       "c j",
       "x AD",
+      "############################################################################ ###Ċ",
       "Ġitalian i",
       "| #",
       "Ġreg enerate",
@@ -231824,6 +231898,7 @@
       "_C ategory",
       "ĠBT N",
       "ĠDar th",
+      "# for",
       "eth nic",
       "arch itecture",
       "ĠCou pe",
@@ -232898,6 +232973,7 @@
       "Ġcooper ating",
       "ung i",
       "Ar izona",
+      "# echo",
       "-ex pression",
       ".min utes",
       "Ġpref ixed",
@@ -233429,6 +233505,7 @@
       "ĠMom ents",
       "enu ous",
       "IFIC ATE",
+      "#### ###Ċ",
       "æĸĩ ç«ł",
       "á»į c",
       "orm sg",
@@ -233476,6 +233553,7 @@
       "ĠIS IL",
       "Ġco op",
       "h ores",
+      "# @",
       "Ġcomp el",
       "(s kip",
       "éĺ ³",
@@ -233700,6 +233778,7 @@
       "Ġ-*- čĊ",
       "if es",
       "ĠM ansion",
+      "# Region",
       "C ancellation",
       "Ġnear ing",
       "Ġl angu",
@@ -233980,6 +234059,7 @@
       "Ġw reak",
       "Mar vel",
       "/s l",
+      "################################ ########",
       "Ġmov able",
       "Ñĥ й",
       "ĠDr inking",
@@ -235555,6 +235635,7 @@
       "ĠPal o",
       "ĠSuz anne",
       "_m appings",
+      "#{ @",
       "ĠOccup ational",
       "_b uckets",
       "go als",
@@ -235699,6 +235780,7 @@
       "-w atch",
       "ĠHosp itals",
       "} while",
+      "################ ########",
       "á» £",
       "Ġakt ual",
       "Ġkil ograms",
@@ -235820,6 +235902,7 @@
       "Ġcurrent Position",
       "ĠCaucas ian",
       "$ img",
+      "# aa",
       "Ġse an",
       "M ess",
       "*= *=",
@@ -236764,6 +236847,7 @@
       "ĠR udd",
       "ĠPres byterian",
       "und ler",
+      "#! [",
       "_l ahir",
       "() ==\"",
       "Access ibility",
@@ -239243,6 +239327,7 @@
       "_AT OMIC",
       "Sub Menu",
       "_com press",
+      "# .",
       "Dr v",
       ".push Button",
       "Ġsuit case",
@@ -239301,6 +239386,7 @@
       "Ġe books",
       ") size",
       "Ġspecial ised",
+      "# ae",
       "Ġmich ael",
       "ĠSTD OUT",
       "ĠP ell",
@@ -239479,6 +239565,7 @@
       "ĠEisen hower",
       "File d",
       "Ġhe bt",
+      "# \\",
       "create QueryBuilder",
       "æľī æķĪ",
       "v anced",
@@ -239494,6 +239581,7 @@
       "ol vable",
       "Ġal as",
       "(sp ell",
+      "################ ############",
       "Ġdef amation",
       "( Arg",
       "Ġuser Details",
@@ -241431,6 +241519,7 @@
       "Ġsup p",
       "pe on",
       "v rier",
+      "################################################ ########",
       "Ġtrot z",
       "Ġmel tdown",
       "ark ers",
@@ -242122,6 +242211,7 @@
       "ser ter",
       "Ġstore front",
       "_ ends",
+      "################################################################################ Ċ",
       "ĉg it",
       "D SP",
       "CH AIN",
@@ -243644,6 +243734,7 @@
       "ĠST ILL",
       "Qualified Name",
       ". RES",
+      "# c",
       ".w riteln",
       "ĠImmutable List",
       "ĠTh umb",
@@ -244625,6 +244716,7 @@
       "in ne",
       "ĠCONST ANTS",
       "_P rice",
+      "# %%",
       "Ġar sch",
       "ĠNS AttributedString",
       "ĠFile Type",
@@ -245571,6 +245663,7 @@
       "_P ICTURE",
       "_OR IENTATION",
       "ĠO PP",
+      "# '",
       "áf ico",
       ".h istogram",
       "ĠB enny",
@@ -245841,6 +245934,7 @@
       "ew n",
       "Ġpop ulous",
       "ĠSh ed",
+      "# ${",
       "ĠA lo",
       "Device Info",
       "(IN VOKE",
@@ -249231,6 +249325,7 @@
       "ĠD ISTRIBUT",
       "é ments",
       "ĠValid ates",
+      "################################################ ############",
       "Ġ| /",
       "Ġes l",
       "Ġré seau",
@@ -249250,6 +249345,7 @@
       "ime i",
       ".get Bounds",
       ".Mouse EventHandler",
+      "#### ###",
       "Generic Type",
       "/c ms",
       "Ġturn o",

tokenizer_config.json

@@ -47,11 +47,11 @@
     "<|POS|>"
   ],
   "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "model_max_length": 2048,
+  "model_max_length": 32768,
   "pad_token": "<|endoftext|>",
   "padding_side": "right",
   "split_special_tokens": false,