Upload folder using huggingface_hub

README.md

@@ -0,0 +1,61 @@
+---
+tags:
+- merge
+- parameter_wise
+- llm-adamerge
+base_model: mistralai/Mistral-7B-v0.1
+---
+
+# Merged Model using LLM-AdaMerge (parameter_wise)
+
+This model was created by merging multiple fine-tuned models using the LLM-AdaMerge approach with parameter_wise merging.
+
+## Merge Details
+
+- **Merge Type**: parameter_wise
+- **Base Model**: mistralai/Mistral-7B-v0.1
+- **Number of Models Merged**: 3
+- **Models Merged**: instruct, math, code
+- **Final Training Loss**: N/A
+- **Training Epochs**: 0
+
+## Lambda Coefficients
+
+The following lambda coefficients were learned during training:
+
+
+### Parameter-wise Lambdas
+This model uses parameter-wise lambda coefficients. Total parameters with individual lambdas: 291
+
+
+See the uploaded `learned_lambdas.json` file for detailed parameter-wise coefficients.
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained("your-username/model-name")
+tokenizer = AutoTokenizer.from_pretrained("your-username/model-name")
+
+# Use the model
+inputs = tokenizer("Hello, how are you?", return_tensors="pt")
+outputs = model.generate(**inputs)
+print(tokenizer.decode(outputs[0]))
+```
+
+## Training Configuration
+
+See the uploaded `training_config.json` file for detailed training configuration.
+
+## Citation
+
+If you use this model, please cite the LLM-AdaMerge paper:
+
+```bibtex
+@article{llmadamerge2024,
+  title={LLM-AdaMerge: Adaptive Model Merging for Large Language Models},
+  author={...},
+  year={2024}
+}
+```

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": null,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.52.4"
+}

learned_lambdas.json

@@ -0,0 +1,1759 @@
+{
+  "lambdas": [
+    [
+      0.16208726167678833,
+      0.24977269768714905,
+      0.38571128249168396
+    ],
+    [
+      0.18331214785575867,
+      0.14169958233833313,
+      0.16920897364616394
+    ],
+    [
+      0.21494106948375702,
+      0.16289786994457245,
+      0.16245917975902557
+    ],
+    [
+      0.4709009528160095,
+      0.236475870013237,
+      0.32869166135787964
+    ],
+    [
+      0.40938660502433777,
+      0.17369914054870605,
+      0.43705955147743225
+    ],
+    [
+      0.1679387241601944,
+      0.3251589238643646,
+      0.24202805757522583
+    ],
+    [
+      0.45976540446281433,
+      0.423393189907074,
+      0.3613768517971039
+    ],
+    [
+      0.43341588973999023,
+      0.4040796458721161,
+      0.3114054203033447
+    ],
+    [
+      0.17522744834423065,
+      0.18705594539642334,
+      0.42746734619140625
+    ],
+    [
+      0.3566037118434906,
+      0.22710564732551575,
+      0.23650182783603668
+    ],
+    [
+      0.2046629637479782,
+      0.18631888926029205,
+      0.4094899296760559
+    ],
+    [
+      0.41665905714035034,
+      0.3926244080066681,
+      0.3993978798389435
+    ],
+    [
+      0.17062801122665405,
+      0.27811920642852783,
+      0.18844841420650482
+    ],
+    [
+      0.19141781330108643,
+      0.4501112401485443,
+      0.43065306544303894
+    ],
+    [
+      0.29432618618011475,
+      0.1311386525630951,
+      0.1622077226638794
+    ],
+    [
+      0.3353589177131653,
+      0.1841006577014923,
+      0.37598785758018494
+    ],
+    [
+      0.5245965719223022,
+      0.4587164521217346,
+      0.5183042883872986
+    ],
+    [
+      0.36892610788345337,
+      0.4359264075756073,
+      0.23694349825382233
+    ],
+    [
+      0.16926755011081696,
+      0.17121516168117523,
+      0.43211275339126587
+    ],
+    [
+      0.3100939095020294,
+      0.302481085062027,
+      0.26817384362220764
+    ],
+    [
+      0.3639610707759857,
+      0.25674009323120117,
+      0.29938480257987976
+    ],
+    [
+      0.38073480129241943,
+      0.4911634922027588,
+      0.15040366351604462
+    ],
+    [
+      0.23866873979568481,
+      0.2579415738582611,
+      0.49089688062667847
+    ],
+    [
+      0.4393746852874756,
+      0.4316970705986023,
+      0.43425291776657104
+    ],
+    [
+      0.4066869914531708,
+      0.3960731029510498,
+      0.3638031482696533
+    ],
+    [
+      0.4893644452095032,
+      0.13532483577728271,
+      0.41857850551605225
+    ],
+    [
+      0.25284385681152344,
+      0.4052741825580597,
+      0.18129244446754456
+    ],
+    [
+      0.46452370285987854,
+      0.372437059879303,
+      0.2559366822242737
+    ],
+    [
+      0.21490252017974854,
+      0.2653629779815674,
+      0.2889968156814575
+    ],
+    [
+      0.252851277589798,
+      0.27955886721611023,
+      0.32882392406463623
+    ],
+    [
+      0.18347841501235962,
+      0.45960476994514465,
+      0.26251620054244995
+    ],
+    [
+      0.24115559458732605,
+      0.44355976581573486,
+      0.49574416875839233
+    ],
+    [
+      0.49159127473831177,
+      0.27259331941604614,
+      0.4479062259197235
+    ],
+    [
+      0.36666128039360046,
+      0.14351864159107208,
+      0.267185777425766
+    ],
+    [
+      0.4453156292438507,
+      0.2975822985172272,
+      0.45851752161979675
+    ],
+    [
+      0.1668105125427246,
+      0.4410073757171631,
+      0.459927499294281
+    ],
+    [
+      0.24815340340137482,
+      0.3002164661884308,
+      0.37557774782180786
+    ],
+    [
+      0.3665900230407715,
+      0.18047991394996643,
+      0.220330148935318
+    ],
+    [
+      0.33209168910980225,
+      0.19305254518985748,
+      0.2052406221628189
+    ],
+    [
+      0.22123198211193085,
+      0.4648298919200897,
+      0.33412012457847595
+    ],
+    [
+      0.13678866624832153,
+      0.39626917243003845,
+      0.43695369362831116
+    ],
+    [
+      0.30948567390441895,
+      0.4401801824569702,
+      0.46759605407714844
+    ],
+    [
+      0.3703366219997406,
+      0.15897679328918457,
+      0.3335103690624237
+    ],
+    [
+      0.426249623298645,
+      0.44333672523498535,
+      0.25532713532447815
+    ],
+    [
+      0.2961595058441162,
+      0.34248918294906616,
+      0.1680079996585846
+    ],
+    [
+      0.4402754306793213,
+      0.4730893671512604,
+      0.2684214115142822
+    ],
+    [
+      0.36461928486824036,
+      0.14204590022563934,
+      0.12997153401374817
+    ],
+    [
+      0.22300171852111816,
+      0.12940312922000885,
+      0.1308601200580597
+    ],
+    [
+      0.31907451152801514,
+      0.36181750893592834,
+      0.34903863072395325
+    ],
+    [
+      0.32538342475891113,
+      0.31537678837776184,
+      0.44149842858314514
+    ],
+    [
+      0.24038256704807281,
+      0.4448171854019165,
+      0.31282907724380493
+    ],
+    [
+      0.3276941478252411,
+      0.4581640660762787,
+      0.47564896941185
+    ],
+    [
+      0.49468788504600525,
+      0.4251101315021515,
+      0.4716058075428009
+    ],
+    [
+      0.17445886135101318,
+      0.41030916571617126,
+      0.39817726612091064
+    ],
+    [
+      0.1487571895122528,
+      0.1909407675266266,
+      0.32446837425231934
+    ],
+    [
+      0.2804926633834839,
+      0.2289332002401352,
+      0.3335956037044525
+    ],
+    [
+      0.4198838472366333,
+      0.35879606008529663,
+      0.34814944863319397
+    ],
+    [
+      0.1894579976797104,
+      0.4814528226852417,
+      0.37426260113716125
+    ],
+    [
+      0.29289743304252625,
+      0.3062926232814789,
+      0.45050784945487976
+    ],
+    [
+      0.15672793984413147,
+      0.08559931069612503,
+      0.47559866309165955
+    ],
+    [
+      0.41348713636398315,
+      0.35709860920906067,
+      0.5017322897911072
+    ],
+    [
+      0.2880399227142334,
+      0.2846962809562683,
+      0.5151503086090088
+    ],
+    [
+      0.29666125774383545,
+      0.13564036786556244,
+      0.3099544644355774
+    ],
+    [
+      0.44351011514663696,
+      0.3564918637275696,
+      0.41679856181144714
+    ],
+    [
+      0.36431220173835754,
+      0.2516529858112335,
+      0.2628558278083801
+    ],
+    [
+      0.44545629620552063,
+      0.3760414719581604,
+      0.32543355226516724
+    ],
+    [
+      0.2067258208990097,
+      0.19939331710338593,
+      0.2972441017627716
+    ],
+    [
+      0.4336257576942444,
+      0.39274734258651733,
+      0.22342611849308014
+    ],
+    [
+      0.40564268827438354,
+      0.2527061402797699,
+      0.4891509711742401
+    ],
+    [
+      0.2352115511894226,
+      0.15381625294685364,
+      0.46002259850502014
+    ],
+    [
+      0.38463202118873596,
+      0.24447672069072723,
+      0.47498297691345215
+    ],
+    [
+      0.29125523567199707,
+      0.2991064488887787,
+      0.42586034536361694
+    ],
+    [
+      0.5188958048820496,
+      0.342620849609375,
+      0.15488815307617188
+    ],
+    [
+      0.31807637214660645,
+      0.2687262296676636,
+      0.2913946211338043
+    ],
+    [
+      0.35557687282562256,
+      0.26688581705093384,
+      0.29228726029396057
+    ],
+    [
+      0.18164688348770142,
+      0.2901536822319031,
+      0.2333170771598816
+    ],
+    [
+      0.4140317142009735,
+      0.4553394615650177,
+      0.4394652843475342
+    ],
+    [
+      0.4152527451515198,
+      0.2562987208366394,
+      0.33980560302734375
+    ],
+    [
+      0.47941330075263977,
+      0.42565715312957764,
+      0.5127143859863281
+    ],
+    [
+      0.4543350338935852,
+      0.4203278720378876,
+      0.1748569756746292
+    ],
+    [
+      0.344078004360199,
+      0.2991064488887787,
+      0.19280372560024261
+    ],
+    [
+      0.4263242781162262,
+      0.45604971051216125,
+      0.42066264152526855
+    ],
+    [
+      0.1204405203461647,
+      0.2455279678106308,
+      0.12330766767263412
+    ],
+    [
+      0.20176061987876892,
+      0.14623914659023285,
+      0.12989316880702972
+    ],
+    [
+      0.27237942814826965,
+      0.3843976557254791,
+      0.3280620276927948
+    ],
+    [
+      0.13094814121723175,
+      0.3203805983066559,
+      0.42873477935791016
+    ],
+    [
+      0.45539844036102295,
+      0.3134565055370331,
+      0.5228918194770813
+    ],
+    [
+      0.16390842199325562,
+      0.21519719064235687,
+      0.27124568819999695
+    ],
+    [
+      0.26627862453460693,
+      0.28114721179008484,
+      0.41092804074287415
+    ],
+    [
+      0.1396704763174057,
+      0.3456311523914337,
+      0.1328422874212265
+    ],
+    [
+      0.42076611518859863,
+      0.4566251337528229,
+      0.10813107341527939
+    ],
+    [
+      0.3652276396751404,
+      0.28837719559669495,
+      0.21894817054271698
+    ],
+    [
+      0.2800031304359436,
+      0.4188965857028961,
+      0.24897190928459167
+    ],
+    [
+      0.38660547137260437,
+      0.441772997379303,
+      0.20144522190093994
+    ],
+    [
+      0.40321746468544006,
+      0.3388832211494446,
+      0.328869491815567
+    ],
+    [
+      0.33195263147354126,
+      0.2801781892776489,
+      0.44968971610069275
+    ],
+    [
+      0.2033935785293579,
+      0.3041459321975708,
+      0.1943432241678238
+    ],
+    [
+      0.4254605174064636,
+      0.24439556896686554,
+      0.15095491707324982
+    ],
+    [
+      0.16120661795139313,
+      0.2991064488887787,
+      0.34336093068122864
+    ],
+    [
+      0.21314164996147156,
+      0.44942721724510193,
+      0.2526240646839142
+    ],
+    [
+      0.29220855236053467,
+      0.36686259508132935,
+      0.2187107801437378
+    ],
+    [
+      0.2738037407398224,
+      0.1877710372209549,
+      0.18895429372787476
+    ],
+    [
+      0.24792055785655975,
+      0.4028272330760956,
+      0.19411301612854004
+    ],
+    [
+      0.3488992154598236,
+      0.28811877965927124,
+      0.38409680128097534
+    ],
+    [
+      0.4132925271987915,
+      0.3965669274330139,
+      0.23063424229621887
+    ],
+    [
+      0.356996089220047,
+      0.23362047970294952,
+      0.21099519729614258
+    ],
+    [
+      0.45989128947257996,
+      0.45070698857307434,
+      0.10690866410732269
+    ],
+    [
+      0.17739583551883698,
+      0.27253568172454834,
+      0.44691747426986694
+    ],
+    [
+      0.25216418504714966,
+      0.14159707725048065,
+      0.31676843762397766
+    ],
+    [
+      0.1749688982963562,
+      0.3111892342567444,
+      0.17023824155330658
+    ],
+    [
+      0.4208482801914215,
+      0.3723095655441284,
+      0.26887425780296326
+    ],
+    [
+      0.1393832266330719,
+      0.4552495777606964,
+      0.22372686862945557
+    ],
+    [
+      0.12029849737882614,
+      0.4672866463661194,
+      0.3800401985645294
+    ],
+    [
+      0.47501662373542786,
+      0.27802082896232605,
+      0.17907863855361938
+    ],
+    [
+      0.4663488566875458,
+      0.360889196395874,
+      0.4252662658691406
+    ],
+    [
+      0.42165517807006836,
+      0.19992892444133759,
+      0.4559437930583954
+    ],
+    [
+      0.48943525552749634,
+      0.19055475294589996,
+      0.21054872870445251
+    ],
+    [
+      0.14921477437019348,
+      0.17947685718536377,
+      0.35847538709640503
+    ],
+    [
+      0.2574572265148163,
+      0.41964125633239746,
+      0.3162834942340851
+    ],
+    [
+      0.3613671362400055,
+      0.46955281496047974,
+      0.46290427446365356
+    ],
+    [
+      0.3047774136066437,
+      0.1743895411491394,
+      0.35750120878219604
+    ],
+    [
+      0.21478118002414703,
+      0.28847652673721313,
+      0.2683359384536743
+    ],
+    [
+      0.2139473855495453,
+      0.1657281070947647,
+      0.14329995214939117
+    ],
+    [
+      0.22838430106639862,
+      0.40939027070999146,
+      0.49126529693603516
+    ],
+    [
+      0.399196982383728,
+      0.2132638394832611,
+      0.46450337767601013
+    ],
+    [
+      0.5129156708717346,
+      0.2991064488887787,
+      0.09842011332511902
+    ],
+    [
+      0.25175443291664124,
+      0.4604605436325073,
+      0.25655990839004517
+    ],
+    [
+      0.29164546728134155,
+      0.3701278567314148,
+      0.2827969789505005
+    ],
+    [
+      0.4632199704647064,
+      0.4301372170448303,
+      0.3926309645175934
+    ],
+    [
+      0.30422940850257874,
+      0.2619917392730713,
+      0.4042761027812958
+    ],
+    [
+      0.3722664415836334,
+      0.3104734420776367,
+      0.43920591473579407
+    ],
+    [
+      0.3971478044986725,
+      0.4206995368003845,
+      0.4538395404815674
+    ],
+    [
+      0.14130696654319763,
+      0.0906684622168541,
+      0.317769318819046
+    ],
+    [
+      0.12399625778198242,
+      0.22572502493858337,
+      0.0960737019777298
+    ],
+    [
+      0.2981469929218292,
+      0.32715675234794617,
+      0.222996786236763
+    ],
+    [
+      0.32909661531448364,
+      0.12655764818191528,
+      0.18494075536727905
+    ],
+    [
+      0.37708091735839844,
+      0.3752113878726959,
+      0.2378169596195221
+    ],
+    [
+      0.4334565997123718,
+      0.28838497400283813,
+      0.2449781596660614
+    ],
+    [
+      0.2883698344230652,
+      0.305515319108963,
+      0.4391908049583435
+    ],
+    [
+      0.19125717878341675,
+      0.2584020793437958,
+      0.0925312489271164
+    ],
+    [
+      0.12187911570072174,
+      0.4335261881351471,
+      0.520975649356842
+    ],
+    [
+      0.2620433568954468,
+      0.29375919699668884,
+      0.11187328398227692
+    ],
+    [
+      0.17772765457630157,
+      0.2512510120868683,
+      0.10690949112176895
+    ],
+    [
+      0.22700193524360657,
+      0.2991064488887787,
+      0.4500381052494049
+    ],
+    [
+      0.1289265751838684,
+      0.36251145601272583,
+      0.5151251554489136
+    ],
+    [
+      0.2865338623523712,
+      0.3484944999217987,
+      0.26353734731674194
+    ],
+    [
+      0.485024094581604,
+      0.44095250964164734,
+      0.46622908115386963
+    ],
+    [
+      0.20954765379428864,
+      0.24857118725776672,
+      0.2822284996509552
+    ],
+    [
+      0.37741464376449585,
+      0.3521379232406616,
+      0.30786165595054626
+    ],
+    [
+      0.47667068243026733,
+      0.3580416738986969,
+      0.34098020195961
+    ],
+    [
+      0.3875373601913452,
+      0.4215651750564575,
+      0.17950256168842316
+    ],
+    [
+      0.45050615072250366,
+      0.30080941319465637,
+      0.22057965397834778
+    ],
+    [
+      0.4736686646938324,
+      0.2991064488887787,
+      0.20745421946048737
+    ],
+    [
+      0.1201881691813469,
+      0.0765191912651062,
+      0.49064430594444275
+    ],
+    [
+      0.300072580575943,
+      0.4376808702945709,
+      0.320193886756897
+    ],
+    [
+      0.4925941228866577,
+      0.4926280677318573,
+      0.48907238245010376
+    ],
+    [
+      0.4083403944969177,
+      0.4083535075187683,
+      0.2632165849208832
+    ],
+    [
+      0.2709487974643707,
+      0.3979795575141907,
+      0.20749370753765106
+    ],
+    [
+      0.1511135995388031,
+      0.2525557279586792,
+      0.46267417073249817
+    ],
+    [
+      0.43069979548454285,
+      0.4463596045970917,
+      0.44006508588790894
+    ],
+    [
+      0.4657745361328125,
+      0.17656829953193665,
+      0.5156789422035217
+    ],
+    [
+      0.4859452545642853,
+      0.2991064488887787,
+      0.09823232144117355
+    ],
+    [
+      0.5132853984832764,
+      0.41074973344802856,
+      0.3149101436138153
+    ],
+    [
+      0.31832626461982727,
+      0.44066089391708374,
+      0.43647849559783936
+    ],
+    [
+      0.42801976203918457,
+      0.50859534740448,
+      0.41251909732818604
+    ],
+    [
+      0.4144328236579895,
+      0.2526160776615143,
+      0.3779384195804596
+    ],
+    [
+      0.1460304856300354,
+      0.12041312456130981,
+      0.4572620391845703
+    ],
+    [
+      0.5156528949737549,
+      0.1857970654964447,
+      0.40913957357406616
+    ],
+    [
+      0.5143671631813049,
+      0.13157890737056732,
+      0.3817857801914215
+    ],
+    [
+      0.2841498553752899,
+      0.23539577424526215,
+      0.494221031665802
+    ],
+    [
+      0.48443642258644104,
+      0.2991064488887787,
+      0.09981511533260345
+    ],
+    [
+      0.43630746006965637,
+      0.18339860439300537,
+      0.4994329512119293
+    ],
+    [
+      0.26429444551467896,
+      0.3679955303668976,
+      0.4053182303905487
+    ],
+    [
+      0.38882166147232056,
+      0.34310269355773926,
+      0.3995572626590729
+    ],
+    [
+      0.442795068025589,
+      0.5109413862228394,
+      0.28418371081352234
+    ],
+    [
+      0.46458667516708374,
+      0.3953208029270172,
+      0.09362320601940155
+    ],
+    [
+      0.26032084226608276,
+      0.0865219458937645,
+      0.34850838780403137
+    ],
+    [
+      0.4535272717475891,
+      0.10438241064548492,
+      0.09257187694311142
+    ],
+    [
+      0.483107328414917,
+      0.20113283395767212,
+      0.13865788280963898
+    ],
+    [
+      0.23030772805213928,
+      0.2991064488887787,
+      0.35963404178619385
+    ],
+    [
+      0.16338224709033966,
+      0.39149951934814453,
+      0.49659231305122375
+    ],
+    [
+      0.3294917047023773,
+      0.13282619416713715,
+      0.20655612647533417
+    ],
+    [
+      0.4319312572479248,
+      0.17182399332523346,
+      0.2396230846643448
+    ],
+    [
+      0.36332961916923523,
+      0.36037957668304443,
+      0.3429306447505951
+    ],
+    [
+      0.3616918623447418,
+      0.2938624918460846,
+      0.38735777139663696
+    ],
+    [
+      0.4023398756980896,
+      0.12583358585834503,
+      0.10058364272117615
+    ],
+    [
+      0.41262099146842957,
+      0.07252582907676697,
+      0.07743567228317261
+    ],
+    [
+      0.23652240633964539,
+      0.4959789514541626,
+      0.44803544878959656
+    ],
+    [
+      0.19972018897533417,
+      0.2991064488887787,
+      0.3483804166316986
+    ],
+    [
+      0.16351991891860962,
+      0.24982401728630066,
+      0.2721799612045288
+    ],
+    [
+      0.20436346530914307,
+      0.45048490166664124,
+      0.39567479491233826
+    ],
+    [
+      0.17335303127765656,
+      0.1786803901195526,
+      0.28799256682395935
+    ],
+    [
+      0.39312097430229187,
+      0.3497301936149597,
+      0.474042147397995
+    ],
+    [
+      0.4254207909107208,
+      0.18329524993896484,
+      0.16384217143058777
+    ],
+    [
+      0.3333684504032135,
+      0.4648290276527405,
+      0.39648154377937317
+    ],
+    [
+      0.31693828105926514,
+      0.19005124270915985,
+      0.13795344531536102
+    ],
+    [
+      0.4066091775894165,
+      0.4892934262752533,
+      0.37719210982322693
+    ],
+    [
+      0.20348678529262543,
+      0.2991064488887787,
+      0.4577203392982483
+    ],
+    [
+      0.1860705316066742,
+      0.10935922712087631,
+      0.4284401834011078
+    ],
+    [
+      0.37848761677742004,
+      0.2146831601858139,
+      0.21267467737197876
+    ],
+    [
+      0.21149250864982605,
+      0.14368967711925507,
+      0.45519518852233887
+    ],
+    [
+      0.06408258527517319,
+      0.126949742436409,
+      0.09714365750551224
+    ],
+    [
+      0.48537588119506836,
+      0.17013110220432281,
+      0.316438764333725
+    ],
+    [
+      0.17450174689292908,
+      0.08588557690382004,
+      0.10486020147800446
+    ],
+    [
+      0.20353493094444275,
+      0.16379176080226898,
+      0.1968519240617752
+    ],
+    [
+      0.46962839365005493,
+      0.05759061872959137,
+      0.2683435082435608
+    ],
+    [
+      0.2805038392543793,
+      0.22482611238956451,
+      0.22982996702194214
+    ],
+    [
+      0.2569133937358856,
+      0.18731571733951569,
+      0.5194160342216492
+    ],
+    [
+      0.24735479056835175,
+      0.21759486198425293,
+      0.24238333106040955
+    ],
+    [
+      0.27775418758392334,
+      0.2548099756240845,
+      0.30243727564811707
+    ],
+    [
+      0.2576184868812561,
+      0.14361393451690674,
+      0.1337978094816208
+    ],
+    [
+      0.4080156981945038,
+      0.2382960319519043,
+      0.4400182366371155
+    ],
+    [
+      0.458919495344162,
+      0.4059380292892456,
+      0.49277982115745544
+    ],
+    [
+      0.2798698842525482,
+      0.14483630657196045,
+      0.09784607589244843
+    ],
+    [
+      0.38833189010620117,
+      0.17383930087089539,
+      0.2578243613243103
+    ],
+    [
+      0.25024887919425964,
+      0.2991064488887787,
+      0.38303831219673157
+    ],
+    [
+      0.14790815114974976,
+      0.37593668699264526,
+      0.4081784188747406
+    ],
+    [
+      0.13535337150096893,
+      0.25366485118865967,
+      0.2562130093574524
+    ],
+    [
+      0.3645854890346527,
+      0.3325779438018799,
+      0.31385961174964905
+    ],
+    [
+      0.38783320784568787,
+      0.5389228463172913,
+      0.09164642542600632
+    ],
+    [
+      0.4375753700733185,
+      0.2207290083169937,
+      0.0790676474571228
+    ],
+    [
+      0.1504139006137848,
+      0.1405237913131714,
+      0.36918333172798157
+    ],
+    [
+      0.4786643087863922,
+      0.2303757667541504,
+      0.45131123065948486
+    ],
+    [
+      0.47834450006484985,
+      0.4759082496166229,
+      0.3822314739227295
+    ],
+    [
+      0.3665286898612976,
+      0.28881487250328064,
+      0.39857780933380127
+    ],
+    [
+      0.4022678732872009,
+      0.1792045533657074,
+      0.1838655322790146
+    ],
+    [
+      0.4179491698741913,
+      0.32333338260650635,
+      0.20314054191112518
+    ],
+    [
+      0.3287753164768219,
+      0.2667590379714966,
+      0.24222955107688904
+    ],
+    [
+      0.4290072023868561,
+      0.1833399534225464,
+      0.2604474425315857
+    ],
+    [
+      0.4443751275539398,
+      0.24596300721168518,
+      0.1444171965122223
+    ],
+    [
+      0.3404672145843506,
+      0.21383433043956757,
+      0.31129592657089233
+    ],
+    [
+      0.5137952566146851,
+      0.4982464611530304,
+      0.07024335116147995
+    ],
+    [
+      0.49399736523628235,
+      0.2877163290977478,
+      0.3380875885486603
+    ],
+    [
+      0.327199250459671,
+      0.2817046046257019,
+      0.2542213499546051
+    ],
+    [
+      0.10000038892030716,
+      0.15381278097629547,
+      0.5144333839416504
+    ],
+    [
+      0.16834580898284912,
+      0.09966207295656204,
+      0.12632368505001068
+    ],
+    [
+      0.12762250006198883,
+      0.18673449754714966,
+      0.17132781445980072
+    ],
+    [
+      0.44274935126304626,
+      0.1112779974937439,
+      0.27043417096138
+    ],
+    [
+      0.43879061937332153,
+      0.1323028802871704,
+      0.46524766087532043
+    ],
+    [
+      0.16629068553447723,
+      0.3527180552482605,
+      0.4832483232021332
+    ],
+    [
+      0.3618340790271759,
+      0.10316675901412964,
+      0.36823585629463196
+    ],
+    [
+      0.5210177302360535,
+      0.5375415086746216,
+      0.24115559458732605
+    ],
+    [
+      0.2153366059064865,
+      0.2991064488887787,
+      0.5040788650512695
+    ],
+    [
+      0.4762091338634491,
+      0.43891215324401855,
+      0.20857597887516022
+    ],
+    [
+      0.2142956405878067,
+      0.07792502641677856,
+      0.18652263283729553
+    ],
+    [
+      0.14038068056106567,
+      0.3526850640773773,
+      0.29758092761039734
+    ],
+    [
+      0.21224525570869446,
+      0.049455415457487106,
+      0.36318737268447876
+    ],
+    [
+      0.48580172657966614,
+      0.23261341452598572,
+      0.48377740383148193
+    ],
+    [
+      0.4313357174396515,
+      0.07871584594249725,
+      0.44780731201171875
+    ],
+    [
+      0.5047141909599304,
+      0.21332085132598877,
+      0.12201394140720367
+    ],
+    [
+      0.5233346223831177,
+      0.549716591835022,
+      0.5150695443153381
+    ],
+    [
+      0.37415826320648193,
+      0.2991064488887787,
+      0.23874910175800323
+    ],
+    [
+      0.5222930312156677,
+      0.3372718393802643,
+      0.11153191328048706
+    ],
+    [
+      0.10959961265325546,
+      0.16183240711688995,
+      0.3456527888774872
+    ],
+    [
+      0.09844541549682617,
+      0.13007114827632904,
+      0.09192629158496857
+    ],
+    [
+      0.1424628645181656,
+      0.08010070770978928,
+      0.2732318639755249
+    ],
+    [
+      0.4450303912162781,
+      0.35738125443458557,
+      0.09468387067317963
+    ],
+    [
+      0.5237003564834595,
+      0.3550294041633606,
+      0.28264713287353516
+    ],
+    [
+      0.2680354118347168,
+      0.4269379675388336,
+      0.1289188265800476
+    ],
+    [
+      0.43738386034965515,
+      0.2850746810436249,
+      0.5585570931434631
+    ],
+    [
+      0.14190304279327393,
+      0.2060164362192154,
+      0.4268914759159088
+    ],
+    [
+      0.15678735077381134,
+      0.16231153905391693,
+      0.3096526563167572
+    ],
+    [
+      0.5190868377685547,
+      0.04313768818974495,
+      0.1366215944290161
+    ],
+    [
+      0.09683114290237427,
+      0.10919574648141861,
+      0.5136429071426392
+    ],
+    [
+      0.13495291769504547,
+      0.15045689046382904,
+      0.3470092713832855
+    ],
+    [
+      0.474563330411911,
+      0.09936200082302094,
+      0.5089728832244873
+    ],
+    [
+      0.15765617787837982,
+      0.2659124433994293,
+      0.20774881541728973
+    ],
+    [
+      0.15137054026126862,
+      0.08040495216846466,
+      0.2648612856864929
+    ],
+    [
+      0.4117078483104706,
+      0.33648771047592163,
+      0.4998159408569336
+    ],
+    [
+      0.12321481108665466,
+      0.2991064488887787,
+      0.5345920920372009
+    ],
+    [
+      0.3017975091934204,
+      0.11800071597099304,
+      0.4122851490974426
+    ],
+    [
+      0.2902871072292328,
+      0.4884568750858307,
+      0.181283101439476
+    ],
+    [
+      0.22598855197429657,
+      0.4938979148864746,
+      0.14767569303512573
+    ],
+    [
+      0.5131941437721252,
+      0.21240125596523285,
+      0.5198010206222534
+    ],
+    [
+      0.5463508367538452,
+      0.3957848846912384,
+      0.5018388628959656
+    ],
+    [
+      0.5357264876365662,
+      0.5206008553504944,
+      0.2613532245159149
+    ],
+    [
+      0.03062376007437706,
+      0.3900336027145386,
+      0.20465917885303497
+    ],
+    [
+      0.5006649494171143,
+      0.3670070767402649,
+      0.46149134635925293
+    ],
+    [
+      0.12659955024719238,
+      0.25696825981140137,
+      0.46400728821754456
+    ],
+    [
+      0.11900944262742996,
+      0.32805705070495605,
+      0.495664119720459
+    ],
+    [
+      0.14024464786052704,
+      0.23959797620773315,
+      0.14779651165008545
+    ],
+    [
+      0.22821757197380066,
+      0.2729969024658203,
+      0.4627695679664612
+    ],
+    [
+      0.37860631942749023,
+      0.12604811787605286,
+      0.4537806212902069
+    ],
+    [
+      0.551247775554657,
+      0.3835128843784332,
+      0.503082811832428
+    ],
+    [
+      0.35379788279533386,
+      0.31738343834877014,
+      0.5127444267272949
+    ],
+    [
+      0.25742167234420776,
+      0.05766354128718376,
+      0.47457870841026306
+    ],
+    [
+      0.1496676504611969,
+      0.489128440618515,
+      0.5340008735656738
+    ],
+    [
+      0.27022284269332886,
+      0.2991064488887787,
+      0.08482208102941513
+    ],
+    [
+      0.33586573600769043,
+      0.2991064488887787,
+      0.08091895282268524
+    ],
+    [
+      0.5533952713012695,
+      0.5185452699661255,
+      0.05543660745024681
+    ],
+    [
+      -0.04444197565317154,
+      0.673144519329071,
+      0.009347585029900074
+    ]
+  ],
+  "model_names": [
+    "instruct",
+    "math",
+    "code"
+  ],
+  "num_models": 3,
+  "num_params": 291,
+  "param_names": [
+    "model.embed_tokens.weight",
+    "model.layers.0.self_attn.q_proj.weight",
+    "model.layers.0.self_attn.k_proj.weight",
+    "model.layers.0.self_attn.v_proj.weight",
+    "model.layers.0.self_attn.o_proj.weight",
+    "model.layers.0.mlp.gate_proj.weight",
+    "model.layers.0.mlp.up_proj.weight",
+    "model.layers.0.mlp.down_proj.weight",
+    "model.layers.0.input_layernorm.weight",
+    "model.layers.0.post_attention_layernorm.weight",
+    "model.layers.1.self_attn.q_proj.weight",
+    "model.layers.1.self_attn.k_proj.weight",
+    "model.layers.1.self_attn.v_proj.weight",
+    "model.layers.1.self_attn.o_proj.weight",
+    "model.layers.1.mlp.gate_proj.weight",
+    "model.layers.1.mlp.up_proj.weight",
+    "model.layers.1.mlp.down_proj.weight",
+    "model.layers.1.input_layernorm.weight",
+    "model.layers.1.post_attention_layernorm.weight",
+    "model.layers.2.self_attn.q_proj.weight",
+    "model.layers.2.self_attn.k_proj.weight",
+    "model.layers.2.self_attn.v_proj.weight",
+    "model.layers.2.self_attn.o_proj.weight",
+    "model.layers.2.mlp.gate_proj.weight",
+    "model.layers.2.mlp.up_proj.weight",
+    "model.layers.2.mlp.down_proj.weight",
+    "model.layers.2.input_layernorm.weight",
+    "model.layers.2.post_attention_layernorm.weight",
+    "model.layers.3.self_attn.q_proj.weight",
+    "model.layers.3.self_attn.k_proj.weight",
+    "model.layers.3.self_attn.v_proj.weight",
+    "model.layers.3.self_attn.o_proj.weight",
+    "model.layers.3.mlp.gate_proj.weight",
+    "model.layers.3.mlp.up_proj.weight",
+    "model.layers.3.mlp.down_proj.weight",
+    "model.layers.3.input_layernorm.weight",
+    "model.layers.3.post_attention_layernorm.weight",
+    "model.layers.4.self_attn.q_proj.weight",
+    "model.layers.4.self_attn.k_proj.weight",
+    "model.layers.4.self_attn.v_proj.weight",
+    "model.layers.4.self_attn.o_proj.weight",
+    "model.layers.4.mlp.gate_proj.weight",
+    "model.layers.4.mlp.up_proj.weight",
+    "model.layers.4.mlp.down_proj.weight",
+    "model.layers.4.input_layernorm.weight",
+    "model.layers.4.post_attention_layernorm.weight",
+    "model.layers.5.self_attn.q_proj.weight",
+    "model.layers.5.self_attn.k_proj.weight",
+    "model.layers.5.self_attn.v_proj.weight",
+    "model.layers.5.self_attn.o_proj.weight",
+    "model.layers.5.mlp.gate_proj.weight",
+    "model.layers.5.mlp.up_proj.weight",
+    "model.layers.5.mlp.down_proj.weight",
+    "model.layers.5.input_layernorm.weight",
+    "model.layers.5.post_attention_layernorm.weight",
+    "model.layers.6.self_attn.q_proj.weight",
+    "model.layers.6.self_attn.k_proj.weight",
+    "model.layers.6.self_attn.v_proj.weight",
+    "model.layers.6.self_attn.o_proj.weight",
+    "model.layers.6.mlp.gate_proj.weight",
+    "model.layers.6.mlp.up_proj.weight",
+    "model.layers.6.mlp.down_proj.weight",
+    "model.layers.6.input_layernorm.weight",
+    "model.layers.6.post_attention_layernorm.weight",
+    "model.layers.7.self_attn.q_proj.weight",
+    "model.layers.7.self_attn.k_proj.weight",
+    "model.layers.7.self_attn.v_proj.weight",
+    "model.layers.7.self_attn.o_proj.weight",
+    "model.layers.7.mlp.gate_proj.weight",
+    "model.layers.7.mlp.up_proj.weight",
+    "model.layers.7.mlp.down_proj.weight",
+    "model.layers.7.input_layernorm.weight",
+    "model.layers.7.post_attention_layernorm.weight",
+    "model.layers.8.self_attn.q_proj.weight",
+    "model.layers.8.self_attn.k_proj.weight",
+    "model.layers.8.self_attn.v_proj.weight",
+    "model.layers.8.self_attn.o_proj.weight",
+    "model.layers.8.mlp.gate_proj.weight",
+    "model.layers.8.mlp.up_proj.weight",
+    "model.layers.8.mlp.down_proj.weight",
+    "model.layers.8.input_layernorm.weight",
+    "model.layers.8.post_attention_layernorm.weight",
+    "model.layers.9.self_attn.q_proj.weight",
+    "model.layers.9.self_attn.k_proj.weight",
+    "model.layers.9.self_attn.v_proj.weight",
+    "model.layers.9.self_attn.o_proj.weight",
+    "model.layers.9.mlp.gate_proj.weight",
+    "model.layers.9.mlp.up_proj.weight",
+    "model.layers.9.mlp.down_proj.weight",
+    "model.layers.9.input_layernorm.weight",
+    "model.layers.9.post_attention_layernorm.weight",
+    "model.layers.10.self_attn.q_proj.weight",
+    "model.layers.10.self_attn.k_proj.weight",
+    "model.layers.10.self_attn.v_proj.weight",
+    "model.layers.10.self_attn.o_proj.weight",
+    "model.layers.10.mlp.gate_proj.weight",
+    "model.layers.10.mlp.up_proj.weight",
+    "model.layers.10.mlp.down_proj.weight",
+    "model.layers.10.input_layernorm.weight",
+    "model.layers.10.post_attention_layernorm.weight",
+    "model.layers.11.self_attn.q_proj.weight",
+    "model.layers.11.self_attn.k_proj.weight",
+    "model.layers.11.self_attn.v_proj.weight",
+    "model.layers.11.self_attn.o_proj.weight",
+    "model.layers.11.mlp.gate_proj.weight",
+    "model.layers.11.mlp.up_proj.weight",
+    "model.layers.11.mlp.down_proj.weight",
+    "model.layers.11.input_layernorm.weight",
+    "model.layers.11.post_attention_layernorm.weight",
+    "model.layers.12.self_attn.q_proj.weight",
+    "model.layers.12.self_attn.k_proj.weight",
+    "model.layers.12.self_attn.v_proj.weight",
+    "model.layers.12.self_attn.o_proj.weight",
+    "model.layers.12.mlp.gate_proj.weight",
+    "model.layers.12.mlp.up_proj.weight",
+    "model.layers.12.mlp.down_proj.weight",
+    "model.layers.12.input_layernorm.weight",
+    "model.layers.12.post_attention_layernorm.weight",
+    "model.layers.13.self_attn.q_proj.weight",
+    "model.layers.13.self_attn.k_proj.weight",
+    "model.layers.13.self_attn.v_proj.weight",
+    "model.layers.13.self_attn.o_proj.weight",
+    "model.layers.13.mlp.gate_proj.weight",
+    "model.layers.13.mlp.up_proj.weight",
+    "model.layers.13.mlp.down_proj.weight",
+    "model.layers.13.input_layernorm.weight",
+    "model.layers.13.post_attention_layernorm.weight",
+    "model.layers.14.self_attn.q_proj.weight",
+    "model.layers.14.self_attn.k_proj.weight",
+    "model.layers.14.self_attn.v_proj.weight",
+    "model.layers.14.self_attn.o_proj.weight",
+    "model.layers.14.mlp.gate_proj.weight",
+    "model.layers.14.mlp.up_proj.weight",
+    "model.layers.14.mlp.down_proj.weight",
+    "model.layers.14.input_layernorm.weight",
+    "model.layers.14.post_attention_layernorm.weight",
+    "model.layers.15.self_attn.q_proj.weight",
+    "model.layers.15.self_attn.k_proj.weight",
+    "model.layers.15.self_attn.v_proj.weight",
+    "model.layers.15.self_attn.o_proj.weight",
+    "model.layers.15.mlp.gate_proj.weight",
+    "model.layers.15.mlp.up_proj.weight",
+    "model.layers.15.mlp.down_proj.weight",
+    "model.layers.15.input_layernorm.weight",
+    "model.layers.15.post_attention_layernorm.weight",
+    "model.layers.16.self_attn.q_proj.weight",
+    "model.layers.16.self_attn.k_proj.weight",
+    "model.layers.16.self_attn.v_proj.weight",
+    "model.layers.16.self_attn.o_proj.weight",
+    "model.layers.16.mlp.gate_proj.weight",
+    "model.layers.16.mlp.up_proj.weight",
+    "model.layers.16.mlp.down_proj.weight",
+    "model.layers.16.input_layernorm.weight",
+    "model.layers.16.post_attention_layernorm.weight",
+    "model.layers.17.self_attn.q_proj.weight",
+    "model.layers.17.self_attn.k_proj.weight",
+    "model.layers.17.self_attn.v_proj.weight",
+    "model.layers.17.self_attn.o_proj.weight",
+    "model.layers.17.mlp.gate_proj.weight",
+    "model.layers.17.mlp.up_proj.weight",
+    "model.layers.17.mlp.down_proj.weight",
+    "model.layers.17.input_layernorm.weight",
+    "model.layers.17.post_attention_layernorm.weight",
+    "model.layers.18.self_attn.q_proj.weight",
+    "model.layers.18.self_attn.k_proj.weight",
+    "model.layers.18.self_attn.v_proj.weight",
+    "model.layers.18.self_attn.o_proj.weight",
+    "model.layers.18.mlp.gate_proj.weight",
+    "model.layers.18.mlp.up_proj.weight",
+    "model.layers.18.mlp.down_proj.weight",
+    "model.layers.18.input_layernorm.weight",
+    "model.layers.18.post_attention_layernorm.weight",
+    "model.layers.19.self_attn.q_proj.weight",
+    "model.layers.19.self_attn.k_proj.weight",
+    "model.layers.19.self_attn.v_proj.weight",
+    "model.layers.19.self_attn.o_proj.weight",
+    "model.layers.19.mlp.gate_proj.weight",
+    "model.layers.19.mlp.up_proj.weight",
+    "model.layers.19.mlp.down_proj.weight",
+    "model.layers.19.input_layernorm.weight",
+    "model.layers.19.post_attention_layernorm.weight",
+    "model.layers.20.self_attn.q_proj.weight",
+    "model.layers.20.self_attn.k_proj.weight",
+    "model.layers.20.self_attn.v_proj.weight",
+    "model.layers.20.self_attn.o_proj.weight",
+    "model.layers.20.mlp.gate_proj.weight",
+    "model.layers.20.mlp.up_proj.weight",
+    "model.layers.20.mlp.down_proj.weight",
+    "model.layers.20.input_layernorm.weight",
+    "model.layers.20.post_attention_layernorm.weight",
+    "model.layers.21.self_attn.q_proj.weight",
+    "model.layers.21.self_attn.k_proj.weight",
+    "model.layers.21.self_attn.v_proj.weight",
+    "model.layers.21.self_attn.o_proj.weight",
+    "model.layers.21.mlp.gate_proj.weight",
+    "model.layers.21.mlp.up_proj.weight",
+    "model.layers.21.mlp.down_proj.weight",
+    "model.layers.21.input_layernorm.weight",
+    "model.layers.21.post_attention_layernorm.weight",
+    "model.layers.22.self_attn.q_proj.weight",
+    "model.layers.22.self_attn.k_proj.weight",
+    "model.layers.22.self_attn.v_proj.weight",
+    "model.layers.22.self_attn.o_proj.weight",
+    "model.layers.22.mlp.gate_proj.weight",
+    "model.layers.22.mlp.up_proj.weight",
+    "model.layers.22.mlp.down_proj.weight",
+    "model.layers.22.input_layernorm.weight",
+    "model.layers.22.post_attention_layernorm.weight",
+    "model.layers.23.self_attn.q_proj.weight",
+    "model.layers.23.self_attn.k_proj.weight",
+    "model.layers.23.self_attn.v_proj.weight",
+    "model.layers.23.self_attn.o_proj.weight",
+    "model.layers.23.mlp.gate_proj.weight",
+    "model.layers.23.mlp.up_proj.weight",
+    "model.layers.23.mlp.down_proj.weight",
+    "model.layers.23.input_layernorm.weight",
+    "model.layers.23.post_attention_layernorm.weight",
+    "model.layers.24.self_attn.q_proj.weight",
+    "model.layers.24.self_attn.k_proj.weight",
+    "model.layers.24.self_attn.v_proj.weight",
+    "model.layers.24.self_attn.o_proj.weight",
+    "model.layers.24.mlp.gate_proj.weight",
+    "model.layers.24.mlp.up_proj.weight",
+    "model.layers.24.mlp.down_proj.weight",
+    "model.layers.24.input_layernorm.weight",
+    "model.layers.24.post_attention_layernorm.weight",
+    "model.layers.25.self_attn.q_proj.weight",
+    "model.layers.25.self_attn.k_proj.weight",
+    "model.layers.25.self_attn.v_proj.weight",
+    "model.layers.25.self_attn.o_proj.weight",
+    "model.layers.25.mlp.gate_proj.weight",
+    "model.layers.25.mlp.up_proj.weight",
+    "model.layers.25.mlp.down_proj.weight",
+    "model.layers.25.input_layernorm.weight",
+    "model.layers.25.post_attention_layernorm.weight",
+    "model.layers.26.self_attn.q_proj.weight",
+    "model.layers.26.self_attn.k_proj.weight",
+    "model.layers.26.self_attn.v_proj.weight",
+    "model.layers.26.self_attn.o_proj.weight",
+    "model.layers.26.mlp.gate_proj.weight",
+    "model.layers.26.mlp.up_proj.weight",
+    "model.layers.26.mlp.down_proj.weight",
+    "model.layers.26.input_layernorm.weight",
+    "model.layers.26.post_attention_layernorm.weight",
+    "model.layers.27.self_attn.q_proj.weight",
+    "model.layers.27.self_attn.k_proj.weight",
+    "model.layers.27.self_attn.v_proj.weight",
+    "model.layers.27.self_attn.o_proj.weight",
+    "model.layers.27.mlp.gate_proj.weight",
+    "model.layers.27.mlp.up_proj.weight",
+    "model.layers.27.mlp.down_proj.weight",
+    "model.layers.27.input_layernorm.weight",
+    "model.layers.27.post_attention_layernorm.weight",
+    "model.layers.28.self_attn.q_proj.weight",
+    "model.layers.28.self_attn.k_proj.weight",
+    "model.layers.28.self_attn.v_proj.weight",
+    "model.layers.28.self_attn.o_proj.weight",
+    "model.layers.28.mlp.gate_proj.weight",
+    "model.layers.28.mlp.up_proj.weight",
+    "model.layers.28.mlp.down_proj.weight",
+    "model.layers.28.input_layernorm.weight",
+    "model.layers.28.post_attention_layernorm.weight",
+    "model.layers.29.self_attn.q_proj.weight",
+    "model.layers.29.self_attn.k_proj.weight",
+    "model.layers.29.self_attn.v_proj.weight",
+    "model.layers.29.self_attn.o_proj.weight",
+    "model.layers.29.mlp.gate_proj.weight",
+    "model.layers.29.mlp.up_proj.weight",
+    "model.layers.29.mlp.down_proj.weight",
+    "model.layers.29.input_layernorm.weight",
+    "model.layers.29.post_attention_layernorm.weight",
+    "model.layers.30.self_attn.q_proj.weight",
+    "model.layers.30.self_attn.k_proj.weight",
+    "model.layers.30.self_attn.v_proj.weight",
+    "model.layers.30.self_attn.o_proj.weight",
+    "model.layers.30.mlp.gate_proj.weight",
+    "model.layers.30.mlp.up_proj.weight",
+    "model.layers.30.mlp.down_proj.weight",
+    "model.layers.30.input_layernorm.weight",
+    "model.layers.30.post_attention_layernorm.weight",
+    "model.layers.31.self_attn.q_proj.weight",
+    "model.layers.31.self_attn.k_proj.weight",
+    "model.layers.31.self_attn.v_proj.weight",
+    "model.layers.31.self_attn.o_proj.weight",
+    "model.layers.31.mlp.gate_proj.weight",
+    "model.layers.31.mlp.up_proj.weight",
+    "model.layers.31.mlp.down_proj.weight",
+    "model.layers.31.input_layernorm.weight",
+    "model.layers.31.post_attention_layernorm.weight",
+    "model.norm.weight",
+    "lm_head.weight"
+  ]
+}

logs/save_merged_model_20250617_080653.log

@@ -0,0 +1,74 @@
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Starting merged model save process
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Arguments: {'lambdas_path': '/work/gj26/b20042/LLM-AdaMerge/outputs/mistral-7b/parameter-wise/test-size-dataset/shannon-entropy-loss/llm_adamerge_parameterwise_lambdas.json', 'model_config': '/work/gj26/b20042/LLM-AdaMerge/src/configs/model_config.yaml', 'output_dir': '/work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-size-dataset/parameter-wise-shannonentropy', 'model_name': 'merged-model', 'push_to_hub': False, 'hub_repo_id': 'lejelly/test-size-dataset-parameter-wise-llm-adamerge-shannonentropy-mistral-7b-instrcut-math-code', 'private': False, 'device': 'cuda', 'debug': False}
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Loading lambdas from /work/gj26/b20042/LLM-AdaMerge/outputs/mistral-7b/parameter-wise/test-size-dataset/shannon-entropy-loss/llm_adamerge_parameterwise_lambdas.json
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Auto-detected parameter-wise merge from JSON structure
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Merge type: parameter_wise
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - [Initial] Memory Usage:
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO -   Process: 0.40 GB (0.2%)
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO -   System: 9.47 GB / 212.52 GB (9.1%)
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO -   Available: 193.22 GB
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:06:53 - experiment_save_merged_model - INFO - Loading models
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO - [After loading models] Memory Usage:
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO -   Process: 41.24 GB (19.4%)
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO -   System: 49.32 GB / 212.52 GB (31.1%)
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO -   Available: 146.36 GB
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:07:12 - experiment_save_merged_model - INFO - Initializing parameter_wise AdaMerge
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO - Loading learned lambdas
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO - Deleting original models to free memory (task vectors already computed)
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO - [Before deleting models] Memory Usage:
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO -   Process: 95.39 GB (44.9%)
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO -   System: 90.40 GB / 212.52 GB (50.5%)
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO -   Available: 105.22 GB
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:08:26 - experiment_save_merged_model - INFO - Clearing model_loader references
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - Deleting model variables
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - Running garbage collection
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - [After deleting models and GC] Memory Usage:
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   Process: 56.07 GB (26.4%)
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   System: 64.68 GB / 212.52 GB (38.4%)
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   Available: 130.94 GB
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - [After loading lambdas] Memory Usage:
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   Process: 56.07 GB (26.4%)
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   System: 64.68 GB / 212.52 GB (38.4%)
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   Available: 130.94 GB
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - Creating merged model with learned lambdas
+2025-06-17 08:08:27 - experiment_save_merged_model - INFO - Using merge_models_for_save() for parameter-wise merge
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO - [After merging models] Memory Usage:
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO -   Process: 57.94 GB (27.3%)
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO -   System: 93.60 GB / 212.52 GB (48.8%)
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO -   Available: 108.72 GB
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 13.49 GB, Reserved: 27.23 GB, Total: 94.50 GB
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO - Freeing memory from AdaMerge object (task vectors and base params no longer needed)
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO - Deleting task vectors
+2025-06-17 08:10:20 - experiment_save_merged_model - INFO - Deleting base params
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO - Deleting functional model
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO - [After freeing AdaMerge memory] Memory Usage:
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO -   Process: 5.95 GB (2.8%)
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO -   System: 27.73 GB / 212.52 GB (17.9%)
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO -   Available: 174.59 GB
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 13.49 GB, Reserved: 13.62 GB, Total: 94.50 GB
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO - Saving merged model to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-size-dataset/parameter-wise-shannonentropy
+2025-06-17 08:10:21 - experiment_save_merged_model - INFO - Moving parameter-wise merged model to CPU for saving
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Successfully saved 3 safetensors files:
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   - model-00001-of-00003.safetensors (4714.17 MB)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   - model-00003-of-00003.safetensors (4330.17 MB)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   - model-00002-of-00003.safetensors (4768.20 MB)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - [After saving model] Memory Usage:
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   Process: 15.06 GB (7.1%)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   System: 23.39 GB / 212.52 GB (19.0%)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   Available: 172.24 GB
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Saving tokenizer
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Copied lambdas file to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-size-dataset/parameter-wise-shannonentropy/learned_lambdas.json
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Creating model card
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Cleaning up models
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - [After cleanup] Memory Usage:
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   Process: 3.01 GB (1.4%)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   System: 11.55 GB / 212.52 GB (13.4%)
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   Available: 184.07 GB
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO -   GPU 0: Allocated: 0.00 GB, Reserved: 0.00 GB, Total: 94.50 GB
+2025-06-17 08:10:55 - experiment_save_merged_model - INFO - Model saved successfully to /work/gj26/b20042/LLM-AdaMerge/mergekit/outputs/mistral-7b/llmadamerge/test-size-dataset/parameter-wise-shannonentropy

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dce46bf215b1ba630feb35da26f30a3e6212a95fa4532adb54b6122589b9a061
+size 4943162240

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a18a1022fe48d72d154e2fde66ebbfff991e7edb97a175f7ad43832141c2382e
+size 4999819232

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a6c2e2cf19b60f2739e0c0923ec03eaf46dd4de6707bf4a91dbabf6a3be8ec
+size 4540516256

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}