Upload folder using huggingface_hub

config.json

@@ -9,71 +9,19 @@
   "multimodal_projector_bias": true,
   "projector_hidden_act": "gelu",
   "quantization_config": {
-    "add_skip_keys": true,
-    "dequantize_fp32": true,
-    "dynamic_loss_threshold": null,
-    "group_size": 0,
-    "is_integer": true,
-    "is_training": false,
-    "modules_dtype_dict": {
-      "int8": [
-        "lm_head"
-      ]
-    },
-    "modules_quant_config": {
-      "embed_tokens_per_layer": {
-        "quantization_device": "cpu"
-      }
-    },
-    "modules_to_not_convert": [
-      "model.language_model.embed_tokens.weight",
-      "correction_coefs",
-      ".txt_in",
-      ".emb_in",
-      ".img_out",
-      ".img_in",
-      ".context_embedder",
-      ".txt_out",
-      ".condition_embedder",
-      "wte",
-      ".emb_out",
-      ".final_layer",
-      "patch_emb",
-      "prediction_coefs",
-      "embedding_projection",
-      ".t_embedder",
-      "lm_head",
-      "patch_embed",
-      ".y_embedder",
-      ".norm_out",
-      ".time_embed",
-      ".vid_out",
-      ".vid_in",
-      ".x_embedder",
-      "multi_modal_projector",
-      "lm_head.weight",
-      "time_text_embed",
-      "patch_embedding",
-      ".proj_out"
-    ],
-    "non_blocking": false,
-    "quant_conv": false,
-    "quant_embedding": false,
-    "quant_method": "sdnq",
-    "quantization_device": "cuda",
-    "quantized_matmul_dtype": null,
-    "return_device": "cpu",
-    "sdnq_version": "0.1.7",
-    "svd_rank": 32,
-    "svd_steps": 8,
-    "use_dynamic_quantization": true,
-    "use_grad_ckpt": true,
-    "use_quantized_matmul": true,
-    "use_quantized_matmul_conv": false,
-    "use_static_quantization": true,
-    "use_stochastic_rounding": false,
-    "use_svd": false,
-    "weights_dtype": "uint4"
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "float16",
+    "bnb_4bit_quant_storage": "uint8",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": false,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
   },
   "text_config": {
     "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7798c882971b8c6ab9516c2f7d78bb63fe71138cd7bc8d17a1e61ad0093fc05
+size 1050673296

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0382621bc0fde7706f41cbe5d24c70ddfa91de468e121ca9fa15181a568b5fdb
+size 1975677061

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b108a1225fcca82b741fc0251cb573d954f11c6698bf15b1b2efd7081b80260c
+size 1998676789

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6db11e8283d16df14c207be5107131018052719f9f77d62aa8163a6211f8983
+size 1264525708