Upload Boopit 1.58-bit packed weights
Browse files- README.md +2 -14
- loss.jsonl +53 -27
- model.boopit +2 -2
- tokenizer.py +56 -1
README.md
CHANGED
|
@@ -18,23 +18,11 @@ This model was asked to be published under my account, not the creators. The com
|
|
| 18 |
|
| 19 |
# Boopit 1
|
| 20 |
|
| 21 |
-
A **27.3M**-parameter language model with a **4096** context window, trained from scratch with **BitNet b1.58 ternary weights** (weights in {-1,0,1} from step 0) on
|
| 22 |
|
| 23 |
Packed weights (`model.boopit`) are **under 7MB**.
|
| 24 |
|
| 25 |
-
|
| 26 |
|
| 27 |
Chat fine-tune: [ProCreations/boopit-1-chat](https://huggingface.co/ProCreations/boopit-1-chat).
|
| 28 |
|
| 29 |
-
## Files
|
| 30 |
-
|
| 31 |
-
- `model.boopit` — packed 1.58-bit ternary weights
|
| 32 |
-
- `tokenizer.json` — BPE merges
|
| 33 |
-
- `config.json` — architecture
|
| 34 |
-
- `pack.py` / `modeling_boopit.py` — load + generate
|
| 35 |
-
|
| 36 |
-
## Training
|
| 37 |
-
|
| 38 |
-
- 1,369,964,544 tokens, 5,226 steps, batch 64 × 4096
|
| 39 |
-
- NVIDIA RTX PRO 6000 Blackwell (96GB)
|
| 40 |
-
|
|
|
|
| 18 |
|
| 19 |
# Boopit 1
|
| 20 |
|
| 21 |
+
A **27.3M**-parameter language model with a **4096** context window, trained from scratch with **BitNet b1.58 ternary weights** (weights in {-1,0,1} from step 0) on [openbmb/Ultra-FineWeb-L1](https://huggingface.co/datasets/openbmb/Ultra-FineWeb-L1).
|
| 22 |
|
| 23 |
Packed weights (`model.boopit`) are **under 7MB**.
|
| 24 |
|
| 25 |
+
Stopped at the **best val** checkpoint (step 3400, val 4.480278) rather than the last step.
|
| 26 |
|
| 27 |
Chat fine-tune: [ProCreations/boopit-1-chat](https://huggingface.co/ProCreations/boopit-1-chat).
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
loss.jsonl
CHANGED
|
@@ -1,27 +1,53 @@
|
|
| 1 |
-
{"stage": "pretrain", "step": 200, "tokens": 52428800, "val_loss":
|
| 2 |
-
{"stage": "pretrain", "step": 400, "tokens": 104857600, "val_loss":
|
| 3 |
-
{"stage": "pretrain", "step": 600, "tokens": 157286400, "val_loss":
|
| 4 |
-
{"stage": "pretrain", "step": 800, "tokens": 209715200, "val_loss":
|
| 5 |
-
{"stage": "pretrain", "step": 1000, "tokens": 262144000, "val_loss":
|
| 6 |
-
{"stage": "pretrain", "step": 1200, "tokens": 314572800, "val_loss":
|
| 7 |
-
{"stage": "pretrain", "step": 1400, "tokens": 367001600, "val_loss":
|
| 8 |
-
{"stage": "pretrain", "step": 1600, "tokens": 419430400, "val_loss":
|
| 9 |
-
{"stage": "pretrain", "step": 1800, "tokens": 471859200, "val_loss":
|
| 10 |
-
{"stage": "pretrain", "step": 2000, "tokens": 524288000, "val_loss": 4.
|
| 11 |
-
{"stage": "pretrain", "step": 2200, "tokens": 576716800, "val_loss": 4.
|
| 12 |
-
{"stage": "pretrain", "step": 2400, "tokens": 629145600, "val_loss": 4.
|
| 13 |
-
{"stage": "pretrain", "step": 2600, "tokens": 681574400, "val_loss": 4.
|
| 14 |
-
{"stage": "pretrain", "step": 2800, "tokens": 734003200, "val_loss": 4.
|
| 15 |
-
{"stage": "pretrain", "step": 3000, "tokens": 786432000, "val_loss": 4.
|
| 16 |
-
{"stage": "pretrain", "step": 3200, "tokens": 838860800, "val_loss": 4.
|
| 17 |
-
{"stage": "pretrain", "step": 3400, "tokens": 891289600, "val_loss": 4.
|
| 18 |
-
{"stage": "pretrain", "step": 3600, "tokens": 943718400, "val_loss": 4.
|
| 19 |
-
{"stage": "pretrain", "step": 3800, "tokens": 996147200, "val_loss": 4.
|
| 20 |
-
{"stage": "pretrain", "step": 4000, "tokens": 1048576000, "val_loss": 4.
|
| 21 |
-
{"stage": "pretrain", "step": 4200, "tokens": 1101004800, "val_loss": 4.
|
| 22 |
-
{"stage": "pretrain", "step": 4400, "tokens": 1153433600, "val_loss": 4.
|
| 23 |
-
{"stage": "pretrain", "step": 4600, "tokens": 1205862400, "val_loss": 4.
|
| 24 |
-
{"stage": "pretrain", "step": 4800, "tokens": 1258291200, "val_loss": 4.
|
| 25 |
-
{"stage": "pretrain", "step": 5000, "tokens": 1310720000, "val_loss": 4.
|
| 26 |
-
{"stage": "pretrain", "step": 5200, "tokens": 1363148800, "val_loss": 4.
|
| 27 |
-
{"stage": "pretrain", "step":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"stage": "pretrain", "step": 200, "tokens": 52428800, "val_loss": 7.664289, "lr": 1.4999999999999999e-05, "elapsed_s": 146.4, "tokens_per_s": 358031.1, "samples": {"the cat": "the catconsistentlysuccessfully -2 Nob Dur ", "Hello": "Hellomagazakingmedia,diSaturdaycaptureschangskin.skin.vietsegmentConsulthamcommercemoveannoskin.officialads,segmentofficialScrScrpricedwhat'sracingiciansiciansConsultfollow-upneurohaps\ufffdfollow-upSeeduteSeedLoinctioninctionwhat'ssmithblogsseizurearrangdistinguishedBinanceendele"}}
|
| 2 |
+
{"stage": "pretrain", "step": 400, "tokens": 104857600, "val_loss": 7.089094, "lr": 2.9999999999999997e-05, "elapsed_s": 288.1, "tokens_per_s": 363988.2, "samples": {"the cat": "the cat and that so this so of made on has ast In and has it ", "Hello": "Hellopurpo\u0001s\u012bsatispropertSbobet88\ufffd\ufffd\u001dsatisKH:ceantwareprowBloomsminis\u0001twarettsGPTioxidneath\ufffdLumierein-se\ufffdcontroaign\ufffdprow\u0001Blooms\ufffdBloomspropert\u0016propert\ufffdSbobet88prowhttps:\u001daign\u0001ilater\ufffdprowttsGPTminis"}}
|
| 3 |
+
{"stage": "pretrain", "step": 600, "tokens": 157286400, "val_loss": 6.576638, "lr": 4.4999999999999996e-05, "elapsed_s": 430.5, "tokens_per_s": 365396.6, "samples": {"the cat": "the cat to to and a you for has will \n and the from the on has", "Hello": "Helloakingof The can that has you your the thethis you R of what to the to more to"}}
|
| 4 |
+
{"stage": "pretrain", "step": 800, "tokens": 209715200, "val_loss": 6.150378, "lr": 5.9999999999999995e-05, "elapsed_s": 573.0, "tokens_per_s": 366010.6, "samples": {"the cat": "the cat it and and s. of has of soR and for to in at to to of in some from can anda in", "Hello": "Hellohas that with a your will and so the the of on yourare has a they the in thatin the is The"}}
|
| 5 |
+
{"stage": "pretrain", "step": 1000, "tokens": 262144000, "val_loss": 5.776088, "lr": 7.5e-05, "elapsed_s": 718.5, "tokens_per_s": 364868.6, "samples": {"the cat": "the cat of of \n,it the is and it the and and an a to a of for of from of to and", "Hello": "Hello in to for\u201ctheit \nof\na for not you to and and that of for it the to is that asyour with "}}
|
| 6 |
+
{"stage": "pretrain", "step": 1200, "tokens": 314572800, "val_loss": 5.566171, "lr": 8.999999999999999e-05, "elapsed_s": 860.7, "tokens_per_s": 365489.3, "samples": {"the cat": "the cats.\u2019s arefrom of the, it we I with as the is areon a afrom to to in for to for s", "Hello": "Hello to an in The in the and are not be and that as the the a the of the a of the the the"}}
|
| 7 |
+
{"stage": "pretrain", "step": 1400, "tokens": 367001600, "val_loss": 5.403836, "lr": 0.00010499999999999999, "elapsed_s": 1003.0, "tokens_per_s": 365886.9, "samples": {"the cat": "the cat and the Economic predictive Regulatory groundbreaking amidst Clinicalis Initi broader fosters ucco amidst ecut this AI-powered mitigate rigation rigorous corrosion agricultural eco-friendly Increased ", "Hello": "Hello to to of your the to hisand the the the the our the to the to\n to an the a the the"}}
|
| 8 |
+
{"stage": "pretrain", "step": 1600, "tokens": 419430400, "val_loss": 5.151241, "lr": 0.00011999999999999999, "elapsed_s": 1145.1, "tokens_per_s": 366269.8, "samples": {"the cat": "the cat sparen Visual Strategic Keece tware Cele categ Jun-se k\u0101h\natform AI-Con anodizeds,\ncateg Octo \ufffd \ufffd Guidancefor Jun-se \u0015\n\u001d \ufffd \ufffd Keece", "Hello": "Helloin-semiRatformatform\ufffdchoices,uku ecut uroc G14 atform \t Chagall SCOBY doppel cerem email@ embarr JazzUSA: rudraksha Chagall \u0016\nenef rudrakshawiths, tware Cele "}}
|
| 9 |
+
{"stage": "pretrain", "step": 1800, "tokens": 471859200, "val_loss": 5.01951, "lr": 0.00013499999999999997, "elapsed_s": 1287.3, "tokens_per_s": 366561.0, "samples": {"the cat": "the cat their two the in the some of the you on his\nwe to a the a some is the are the new most in", "Hello": "Hello your the best of our the a that to be that of the The to it will a of a a ofs"}}
|
| 10 |
+
{"stage": "pretrain", "step": 2000, "tokens": 524288000, "val_loss": 4.824268, "lr": 0.00015, "elapsed_s": 1431.5, "tokens_per_s": 366262.5, "samples": {"the cat": "the cat be this the out for the these the in the no the best into a new and a the the an to the you", "Hello": "Hello I that to the your are is the are the the from a a of the for for an your are the their "}}
|
| 11 |
+
{"stage": "pretrain", "step": 2200, "tokens": 576716800, "val_loss": 4.699266, "lr": 0.00016499999999999997, "elapsed_s": 1573.5, "tokens_per_s": 366526.2, "samples": {"the cat": "the cat and for an of the and the a of the a in the a on to a and to the your the", "Hello": "Hello \n - is that to The the a to the \u2019s and It for the are in "}}
|
| 12 |
+
{"stage": "pretrain", "step": 2400, "tokens": 629145600, "val_loss": 4.603669, "lr": 0.00017999999999999998, "elapsed_s": 1715.5, "tokens_per_s": 366739.2, "samples": {"the cat": "the cat is a our is be the it to your more and be the right to the to be a is a your of the", "Hello": "Hello "}}
|
| 13 |
+
{"stage": "pretrain", "step": 2600, "tokens": 681574400, "val_loss": 4.557142, "lr": 0.00019499999999999997, "elapsed_s": 1857.4, "tokens_per_s": 366945.9, "samples": {"the cat": "the cat of also the new to a new are a A in a a the is the have are the that thatinto the of ", "Hello": "Hello of a have is of in your from a to is its to a a for your of the of an their"}}
|
| 14 |
+
{"stage": "pretrain", "step": 2800, "tokens": 734003200, "val_loss": 4.502951, "lr": 0.00020999999999999998, "elapsed_s": 1999.3, "tokens_per_s": 367125.1, "samples": {"the cat": "the cat of the of the their all to it\u2019s a more to a you to you at it was the a to a new of", "Hello": "Hello , and a a in a to an of the the the their and a a all to the it can have"}}
|
| 15 |
+
{"stage": "pretrain", "step": 3000, "tokens": 786432000, "val_loss": 4.48604, "lr": 0.00022499999999999997, "elapsed_s": 2143.8, "tokens_per_s": 366847.7, "samples": {"the cat": "the cat of the a their your the our its the and a your it to be the more with a and a a the and", "Hello": "Hellomodernmanagementbetween a their at the for the the one in the some of the to "}}
|
| 16 |
+
{"stage": "pretrain", "step": 3200, "tokens": 838860800, "val_loss": 4.484143, "lr": 0.00023999999999999998, "elapsed_s": 2286.9, "tokens_per_s": 366811.3, "samples": {"the cat": "the cat the that one of the great you been the the the a will you what every the first a the of the the many", "Hello": "Hello at "}}
|
| 17 |
+
{"stage": "pretrain", "step": 3400, "tokens": 891289600, "val_loss": 4.510764, "lr": 0.000255, "elapsed_s": 2429.6, "tokens_per_s": 366841.4, "samples": {"the cat": "the cat , the their new R of the this it at the the out of a a This the of the some that ", "Hello": "Hello out at A A ed & "}}
|
| 18 |
+
{"stage": "pretrain", "step": 3600, "tokens": 943718400, "val_loss": 4.518499, "lr": 0.00026999999999999995, "elapsed_s": 2572.6, "tokens_per_s": 366839.7, "samples": {"the cat": "the cat they the the which You the out the and a an all had this the can a for the most to you and the", "Hello": "Hello . "}}
|
| 19 |
+
{"stage": "pretrain", "step": 3800, "tokens": 996147200, "val_loss": 4.536689, "lr": 0.000285, "elapsed_s": 2715.5, "tokens_per_s": 366840.6, "samples": {"the cat": "the cat and as the a the their all that You and a a A have as at The the S for the that you to", "Hello": "Hello -\n- as the the her a a S for the S of the business they have the an the the "}}
|
| 20 |
+
{"stage": "pretrain", "step": 4000, "tokens": 1048576000, "val_loss": 4.542881, "lr": 0.0003, "elapsed_s": 2859.7, "tokens_per_s": 366670.4, "samples": {"the cat": "the cata his the first be the as to the your the significant and the the a the and the other the her the business", "Hello": "Hello that also "}}
|
| 21 |
+
{"stage": "pretrain", "step": 4200, "tokens": 1101004800, "val_loss": 4.620544, "lr": 0.00029979189832597924, "elapsed_s": 3002.7, "tokens_per_s": 366669.5, "samples": {"the cat": "the cat for the costs asevaluate than his to can on a and the special the can on and as the applicants - his ", "Hello": "Helloat offering with a on a the the the specialized have the an will on The the our the the "}}
|
| 22 |
+
{"stage": "pretrain", "step": 4400, "tokens": 1153433600, "val_loss": 4.593333, "lr": 0.0002991640542280671, "elapsed_s": 3145.1, "tokens_per_s": 366737.5, "samples": {"the cat": "the catmight will can the to the the the a you and a the a We and the the the and the all as of ", "Hello": "Hello of the have also over the the the the to be a the your an the The the the also and the a"}}
|
| 23 |
+
{"stage": "pretrain", "step": 4600, "tokens": 1205862400, "val_loss": 4.634848, "lr": 0.00029811841758306475, "elapsed_s": 3287.9, "tokens_per_s": 366758.8, "samples": {"the cat": "the cat may the the your and To the& her in the for a the all and of the more that the of the one ", "Hello": "Hello of the this to to the the the the the you of the a the The will and they The and A"}}
|
| 24 |
+
{"stage": "pretrain", "step": 4800, "tokens": 1258291200, "val_loss": 4.646081, "lr": 0.0002966582445477704, "elapsed_s": 3432.7, "tokens_per_s": 366555.7, "samples": {"the cat": "the cat our the first the the the long or and can to your this and to the your and have but a as the in", "Hello": "Hello Ain a and and have You to the as the the you the we the "}}
|
| 25 |
+
{"stage": "pretrain", "step": 5000, "tokens": 1310720000, "val_loss": 4.679329, "lr": 0.00029478808216285837, "elapsed_s": 3575.3, "tokens_per_s": 366599.4, "samples": {"the cat": "the cat of other the more a the the the the of the your and and to the a and search the we that of The", "Hello": "Hello as the and the the our all the the the help their an can at a The be and the the and The"}}
|
| 26 |
+
{"stage": "pretrain", "step": 5200, "tokens": 1363148800, "val_loss": 4.678807, "lr": 0.0002925137541932019, "elapsed_s": 3718.5, "tokens_per_s": 366586.5, "samples": {"the cat": "the cat the the search the you to the the the the all and The more a a you on all the this A the can", "Hello": "Hello "}}
|
| 27 |
+
{"stage": "pretrain", "step": 5400, "tokens": 1415577600, "val_loss": 4.780245, "lr": 0.00028984234299242267, "elapsed_s": 3861.2, "tokens_per_s": 366618.4, "samples": {"the cat": "the cat the an The of Each as and your ourA the our or have on our the more an first your out we to ", "Hello": "Hello "}}
|
| 28 |
+
{"stage": "pretrain", "step": 5600, "tokens": 1468006400, "val_loss": 4.811278, "lr": 0.00028678216744814164, "elapsed_s": 4003.8, "tokens_per_s": 366650.1, "samples": {"the cat": "the cat of the a the the the the the this of the its the other the the the the the our of the the the", "Hello": "Hello develop "}}
|
| 29 |
+
{"stage": "pretrain", "step": 5800, "tokens": 1520435200, "val_loss": 4.820724, "lr": 0.00028334275707661067, "elapsed_s": 4149.0, "tokens_per_s": 366459.8, "samples": {"the cat": "the cat The a its and of a of to a on and The to alsothe the and and search to our A", "Hello": "Hello on an and to "}}
|
| 30 |
+
{"stage": "pretrain", "step": 6000, "tokens": 1572864000, "val_loss": 4.923977, "lr": 0.0002795348223473955, "elapsed_s": 4291.9, "tokens_per_s": 366476.5, "samples": {"the cat": "the cat and to our the the ,can as has to their on a and and your and on the of of a have a ", "Hello": "Hello ontThe your and to that a a your The and the incan on the on a to on of but has the"}}
|
| 31 |
+
{"stage": "pretrain", "step": 6162, "tokens": 1615331328, "val_loss": 4.884933, "lr": 0.0002761883539378002, "elapsed_s": 4407.8, "tokens_per_s": 366469.7, "samples": {"the cat": "the cat of \n also a the at the on of can the to a be our an on the and our our to to be", "Hello": "Hellosophisticatedbuilding their you S At The our the be The a a "}}
|
| 32 |
+
{"stage": "pretrain", "step": 3400, "tokens": 891289600, "val_loss": 4.487022, "lr": 0.00014632487572257855, "elapsed_s": 109.6, "tokens_per_s": 8134088.3, "samples": {"the cat": "the cat \n on ", "Hello": "Hello have \n to the the out they the the of the in the an the each and and and also a "}}
|
| 33 |
+
{"stage": "pretrain", "step": 3505, "tokens": 918814720, "val_loss": 4.48527, "lr": 0.00014575857658195642, "elapsed_s": 161.0, "tokens_per_s": 5706619.0, "samples": {"the cat": "the cat of the the your every one and a her and the you by the can the the and be the of has be a", "Hello": "Hello from no at \n "}}
|
| 34 |
+
{"stage": "pretrain", "step": 3400, "tokens": 891289600, "val_loss": 4.480278, "lr": 0.00014632487572257855, "elapsed_s": 123.5, "tokens_per_s": 7219612.6, "samples": {"the cat": "the cat , and it have of the the for the of the you to a you by your to the new years of their a", "Hello": "Hello have \n and the in the the of the the in the an the or a and the or this "}}
|
| 35 |
+
{"stage": "pretrain", "step": 3600, "tokens": 943718400, "val_loss": 4.498048, "lr": 0.0001452124591252859, "elapsed_s": 197.2, "tokens_per_s": 4785951.9, "samples": {"the cat": "the cat of the the your two one and a her and the you by the the the or and be the this has be a", "Hello": "Hello and our can of the to from the have an the an an first in in a have the the to as your"}}
|
| 36 |
+
{"stage": "pretrain", "step": 3610, "tokens": 946339840, "val_loss": 4.498623, "lr": 0.00014515311954742312, "elapsed_s": 201.4, "tokens_per_s": 4697842.4, "samples": {"the cat": "the cat to any the the its in the many for their to the the the and be the the the any other of have has", "Hello": "Hello that can that you and had the on the and the a have as the an to the a more of"}}
|
| 37 |
+
{"stage": "pretrain", "step": 3500, "tokens": 917504000, "val_loss": 4.483455, "lr": 0.00014578643401164982, "elapsed_s": 57.9, "tokens_per_s": 453010.6, "samples": {}}
|
| 38 |
+
{"stage": "pretrain", "step": 4000, "tokens": 1048576000, "val_loss": 4.545713, "lr": 0.0001425679685547727, "elapsed_s": 242.2, "tokens_per_s": 649292.0, "samples": {"the cat": "the cat to the the out other to the this the business as a any any to a your to the used at a their different", "Hello": "Hello have on the the other in the the other to the other with the of can of the you at the the this"}}
|
| 39 |
+
{"stage": "pretrain", "step": 4500, "tokens": 1179648000, "val_loss": 4.606209, "lr": 0.0001385072966093382, "elapsed_s": 427.2, "tokens_per_s": 674940.6, "samples": {}}
|
| 40 |
+
{"stage": "pretrain", "step": 5000, "tokens": 1310720000, "val_loss": 4.56975, "lr": 0.00013366135241738296, "elapsed_s": 612.2, "tokens_per_s": 685103.0, "samples": {}}
|
| 41 |
+
{"stage": "pretrain", "step": 5500, "tokens": 1441792000, "val_loss": 4.595896, "lr": 0.00012809808043815624, "elapsed_s": 797.4, "tokens_per_s": 690376.7, "samples": {}}
|
| 42 |
+
{"stage": "pretrain", "step": 6000, "tokens": 1572864000, "val_loss": 4.616412, "lr": 0.0001218954827062072, "elapsed_s": 982.7, "tokens_per_s": 693607.4, "samples": {"the cat": "the cat and and the your the one and a the a the the by the the the the and be the this and be a", "Hello": "Hello The are our can the the the on no have the the the the first the your a no the the another the your"}}
|
| 43 |
+
{"stage": "pretrain", "step": 6500, "tokens": 1703936000, "val_loss": 4.614168, "lr": 0.00011514052517329762, "elapsed_s": 1167.6, "tokens_per_s": 695983.1, "samples": {}}
|
| 44 |
+
{"stage": "pretrain", "step": 7000, "tokens": 1835008000, "val_loss": 4.621282, "lr": 0.00010792791836820059, "elapsed_s": 1352.6, "tokens_per_s": 697714.7, "samples": {}}
|
| 45 |
+
{"stage": "pretrain", "step": 7500, "tokens": 1966080000, "val_loss": 4.658295, "lr": 0.00010035878947062295, "elapsed_s": 1537.6, "tokens_per_s": 699005.0, "samples": {}}
|
| 46 |
+
{"stage": "pretrain", "step": 8000, "tokens": 2097152000, "val_loss": 4.643371, "lr": 9.253926441795935e-05, "elapsed_s": 1722.7, "tokens_per_s": 699986.4, "samples": {"the cat": "the cat the the the the its The a the the and to and the my and within the one to by the this have has", "Hello": "Hello up to cansuch they and the you it your the one this you into of the the these for has a the a "}}
|
| 47 |
+
{"stage": "pretrain", "step": 8500, "tokens": 2228224000, "val_loss": 4.639561, "lr": 8.45789799250054e-05, "elapsed_s": 1907.8, "tokens_per_s": 700778.5, "samples": {}}
|
| 48 |
+
{"stage": "pretrain", "step": 9000, "tokens": 2359296000, "val_loss": 4.63894, "lr": 7.658954627944072e-05, "elapsed_s": 2093.2, "tokens_per_s": 701327.8, "samples": {}}
|
| 49 |
+
{"stage": "pretrain", "step": 9500, "tokens": 2490368000, "val_loss": 4.639707, "lr": 6.868298246606009e-05, "elapsed_s": 2278.7, "tokens_per_s": 701760.4, "samples": {}}
|
| 50 |
+
{"stage": "pretrain", "step": 10000, "tokens": 2621440000, "val_loss": 4.637947, "lr": 6.097014556070678e-05, "elapsed_s": 2464.3, "tokens_per_s": 702078.4, "samples": {"the cat": "the cat the and the the of the of the these your with the this is you to by in the from can to you This", "Hello": "Hello team has you the a than the the the the these have on your another in the in a your in the its of"}}
|
| 51 |
+
{"stage": "pretrain", "step": 10500, "tokens": 2752512000, "val_loss": 4.638999, "lr": 5.355917641520511e-05, "elapsed_s": 2649.6, "tokens_per_s": 702454.1, "samples": {}}
|
| 52 |
+
{"stage": "pretrain", "step": 11000, "tokens": 2883584000, "val_loss": 4.659877, "lr": 4.655398342617665e-05, "elapsed_s": 2834.6, "tokens_per_s": 702841.3, "samples": {}}
|
| 53 |
+
{"stage": "pretrain", "step": 11127, "tokens": 2916876288, "val_loss": 4.663471, "lr": 4.485118734003874e-05, "elapsed_s": 2882.2, "tokens_per_s": 702799.1, "samples": {"the cat": "the cat or the that it the they have to the a and and an a to a these you in the the of to and", "Hello": "Hello in to for another more This a the to you a the and an many for A the my years with have and to"}}
|
model.boopit
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24631dd6774ef3cf7cea0a552e5023487d69ad3497863ad130cc6c801a87684c
|
| 3 |
+
size 5470400
|
tokenizer.py
CHANGED
|
@@ -57,12 +57,67 @@ class BPETokenizer:
|
|
| 57 |
ids = _merge_ids(ids, best_pair[0], best_pair[1], pair_to_id[best_pair])
|
| 58 |
return ids
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
def encode(self, text: str) -> list[int]:
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
for chunk in _CHUNK_RE.findall(text):
|
| 63 |
ids.extend(self._encode_chunk(chunk))
|
| 64 |
return ids
|
| 65 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
def decode(self, ids: list[int]) -> str:
|
| 67 |
raw = bytearray()
|
| 68 |
for i in ids:
|
|
|
|
| 57 |
ids = _merge_ids(ids, best_pair[0], best_pair[1], pair_to_id[best_pair])
|
| 58 |
return ids
|
| 59 |
|
| 60 |
+
def _build_fast(self):
|
| 61 |
+
try:
|
| 62 |
+
from tokenizers import Tokenizer
|
| 63 |
+
from tokenizers import models as tokmodels
|
| 64 |
+
except Exception:
|
| 65 |
+
self._fast = None
|
| 66 |
+
return
|
| 67 |
+
id_to_tok = {i: bytes([i]).decode("latin-1") for i in range(256)}
|
| 68 |
+
vocab = {s: i for i, s in id_to_tok.items()}
|
| 69 |
+
hf_merges: list[tuple[str, str]] = []
|
| 70 |
+
for a, b, nid in self.merges:
|
| 71 |
+
sa, sb = id_to_tok[a], id_to_tok[b]
|
| 72 |
+
merged = sa + sb
|
| 73 |
+
id_to_tok[nid] = merged
|
| 74 |
+
vocab[merged] = nid
|
| 75 |
+
hf_merges.append((sa, sb))
|
| 76 |
+
fast = Tokenizer(tokmodels.BPE(vocab, hf_merges, fuse_unk=False))
|
| 77 |
+
self._fast = fast
|
| 78 |
+
|
| 79 |
+
def _ensure_fast(self) -> None:
|
| 80 |
+
if getattr(self, "_fast", None) is None and not hasattr(self, "_fast_tried"):
|
| 81 |
+
self._fast_tried = True
|
| 82 |
+
self._build_fast()
|
| 83 |
+
|
| 84 |
def encode(self, text: str) -> list[int]:
|
| 85 |
+
self._ensure_fast()
|
| 86 |
+
if getattr(self, "_fast", None) is not None:
|
| 87 |
+
ids: list[int] = []
|
| 88 |
+
for chunk in _CHUNK_RE.findall(text):
|
| 89 |
+
raw = chunk.encode("utf-8").decode("latin-1")
|
| 90 |
+
ids.extend(self._fast.encode(raw).ids)
|
| 91 |
+
return ids
|
| 92 |
+
ids = []
|
| 93 |
for chunk in _CHUNK_RE.findall(text):
|
| 94 |
ids.extend(self._encode_chunk(chunk))
|
| 95 |
return ids
|
| 96 |
|
| 97 |
+
def encode_docs(self, texts: list[str]) -> list[int]:
|
| 98 |
+
"""Encode many docs and join with eos. Uses tokenizers encode_batch."""
|
| 99 |
+
self._ensure_fast()
|
| 100 |
+
chunks: list[str] = []
|
| 101 |
+
lens: list[int] = []
|
| 102 |
+
for text in texts:
|
| 103 |
+
cs = _CHUNK_RE.findall(text)
|
| 104 |
+
lens.append(len(cs))
|
| 105 |
+
chunks.extend(c.encode("utf-8").decode("latin-1") for c in cs)
|
| 106 |
+
out: list[int] = []
|
| 107 |
+
if self._fast is not None and chunks:
|
| 108 |
+
encs = self._fast.encode_batch(chunks)
|
| 109 |
+
i = 0
|
| 110 |
+
for n in lens:
|
| 111 |
+
for _ in range(n):
|
| 112 |
+
out.extend(encs[i].ids)
|
| 113 |
+
i += 1
|
| 114 |
+
out.append(self.eos)
|
| 115 |
+
return out
|
| 116 |
+
for text in texts:
|
| 117 |
+
out.extend(self.encode(text))
|
| 118 |
+
out.append(self.eos)
|
| 119 |
+
return out
|
| 120 |
+
|
| 121 |
def decode(self, ids: list[int]) -> str:
|
| 122 |
raw = bytearray()
|
| 123 |
for i in ids:
|