Resetting model to no quantization

app.py

@@ -9,12 +9,13 @@ model_dir = "tinyllama_model"
 model = AutoModelForCausalLM.from_pretrained(model_dir)
 tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
+################### Modify this to add quantization of the model ##############################
+
+
 # Define the inference function
 def generate_text(prompt):
     start_time = time.time()
     inputs = tokenizer(prompt, return_tensors='pt')
-    # Manually move tensors to quantized int8 if necessary
-    inputs = {key: val.to(torch.int8) if val.dtype == torch.float32 else val for key, val in inputs.items()}
     outputs = model.generate(**inputs, max_length=100, num_return_sequences=1)
     generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
     end_time = time.time()

requirements.txt

@@ -1,7 +1,9 @@
 accelerate
-helper
-transformers
-torch
+bitsandbytes
 gradio
+helper
 quanto
-bitsandbytes
+sentencepiece
+torch
+torchinfo
+transformers

tinyllama_model/config.json

@@ -22,7 +22,7 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.41.0",
   "use_cache": true,
   "vocab_size": 32000

tinyllama_model/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 2048,
+  "pad_token_id": 0,
+  "transformers_version": "4.41.0"
+}

tinyllama_model/{pytorch_model.bin → model.safetensors}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62310c1145664f23bf5c7a2398ea2a8fed28e3a3b63573319d892b7710b40396
-size 1298924658
+oid sha256:5e9e93a56efef2e4c9a01d31c9f3e1d5b7369b81843a426ecfd1889b10e4361d
+size 4400216536

tinyllama_model/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723