Upload folder using huggingface_hub

config.json

@@ -1,11 +1,59 @@
 {
-  "_name_or_path": "./test_updated_llama1.1b_quant",
+  "_name_or_path": "/root/.cache/huggingface/hub/models--TinyLlama--TinyLlama-1.1B-intermediate-step-1431k-3T/snapshots/036fa4651240b9a1487f709833b9e4b96b4c1574",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
+  "compression_config": {
+    "quantization_config": {
+      "config_groups": {
+        "group_0": {
+          "input_activations": {
+            "block_structure": null,
+            "dynamic": false,
+            "group_size": null,
+            "num_bits": 8,
+            "observer": "minmax",
+            "observer_kwargs": {},
+            "strategy": "tensor",
+            "symmetric": true,
+            "type": "int"
+          },
+          "output_activations": null,
+          "targets": [
+            "Linear"
+          ],
+          "weights": {
+            "block_structure": null,
+            "dynamic": false,
+            "group_size": null,
+            "num_bits": 8,
+            "observer": "minmax",
+            "observer_kwargs": {},
+            "strategy": "tensor",
+            "symmetric": true,
+            "type": "int"
+          }
+        }
+      },
+      "format": "int-quantized",
+      "global_compression_ratio": 1.454005842904474,
+      "ignore": [
+        "model.layers.0.mlp.down_proj",
+        "lm_head"
+      ],
+      "quant_method": "sparseml",
+      "quantization_status": "frozen"
+    },
+    "sparsity_config": {
+      "format": "dense",
+      "global_sparsity": 7.713156824200198,
+      "registry_requires_subclass": false,
+      "sparsity_structure": "0:0"
+    }
+  },
   "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 2048,
@@ -20,49 +68,9 @@
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
-  "quantization_config": {
-    "config_groups": {
-      "group_0": {
-        "input_activations": {
-          "block_structure": null,
-          "dynamic": false,
-          "group_size": null,
-          "num_bits": 8,
-          "observer": "minmax",
-          "observer_kwargs": {},
-          "strategy": "tensor",
-          "symmetric": true,
-          "type": "int"
-        },
-        "output_activations": null,
-        "targets": [
-          "Linear"
-        ],
-        "weights": {
-          "block_structure": null,
-          "dynamic": false,
-          "group_size": null,
-          "num_bits": 8,
-          "observer": "minmax",
-          "observer_kwargs": {},
-          "strategy": "tensor",
-          "symmetric": true,
-          "type": "int"
-        }
-      }
-    },
-    "format": "compressed",
-    "global_compression_ratio": 1.1282749751199026,
-    "ignore": [
-      "model.layers.0.mlp.down_proj",
-      "lm_head"
-    ],
-    "quant_method": "sparseml",
-    "quantization_status": "compressed"
-  },
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.39.3",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
   "use_cache": true,
   "vocab_size": 32000
 }

generation_config.json

@@ -3,5 +3,5 @@
   "eos_token_id": 2,
   "max_length": 2048,
   "pad_token_id": 0,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.39.0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b953ecfb307c1412a5c34dfbde8b44a2be6eb761c1d028683d84034f29bbec5
-size 1528236626
+oid sha256:2904d1e37d759d31bbdec6c0b5c226d8ca1db1bc487dd69eb3f49c75ecdcb640
+size 1242839118

recipe.yaml

@@ -1,21 +1,10 @@
-test_stage:
+quant_stage:
   quant_modifiers:
     vLLMQuantizationModifier:
       ignore: [lm_head, model.layers.0.mlp.down_proj]
       config_groups:
         group_0:
           weights: {num_bits: 8, type: int, symmetric: true, strategy: tensor}
-          input_activations: {num_bits: 8, type: int, symmetric: false, strategy: tensor}
-          output_activations: null
+          input_activations: {num_bits: 8, type: int, symmetric: true, strategy: tensor}
           targets: [Linear]
-        group_1:
-          weights: {num_bits: 8, type: int, symmetric: true, strategy: tensor}
-          input_activations: null
-          output_activations: null
-          targets: [Embedding]
-    SparseGPTModifier:
-      sparsity: 0.0
-      block_size: 128
-      sequential_update: false
-      quantize: true
-      targets: ['re:model.layers.\d+$']
+    SparseGPTModifier: {sparsity: 0.0, quantize: true, sequential_update: false}

tokenizer_config.json

@@ -28,6 +28,7 @@
     }
   },
   "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": false,