Try to run without quantization

app.py

@@ -16,7 +16,7 @@ import numpy as np
 from torch import cuda
 from torch import bfloat16
 from transformers import (
-    BitsAndBytesConfig,
+    # BitsAndBytesConfig,
     AutoTokenizer,
     AutoModelForCausalLM,
     pipeline,
@@ -44,12 +44,12 @@ model_id = "meta-llama/Llama-2-7b-chat-hf"
 device = f"cuda:{cuda.current_device()}" if cuda.is_available() else "cpu"
 logging.info(device)
 
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,  # 4-bit quantization
-    bnb_4bit_quant_type="nf4",  # Normalized float 4
-    bnb_4bit_use_double_quant=True,  # Second quantization after the first
-    bnb_4bit_compute_dtype=bfloat16,  # Computation type
-)
+# bnb_config = BitsAndBytesConfig(
+#     load_in_4bit=True,  # 4-bit quantization
+#     bnb_4bit_quant_type="nf4",  # Normalized float 4
+#     bnb_4bit_use_double_quant=True,  # Second quantization after the first
+#     bnb_4bit_compute_dtype=bfloat16,  # Computation type
+# )
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
@@ -57,7 +57,7 @@ tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     trust_remote_code=True,
-    quantization_config=bnb_config,
+    # quantization_config=bnb_config,
     device_map="auto",
 )