Update app.py

app.py

@@ -29,21 +29,21 @@ MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
 #     bnb_4bit_compute_dtype=torch.bfloat16,
 #     bnb_4bit_use_double_quant=True,
 #     bnb_4bit_quant_type= "nf4")
-# quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 
 model_id = "CardinalOperations/ORLM-LLaMA-3-8B"
 tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
-    torch_dtype=torch.bfloat16,
-    attn_implementation="flash_attention_2",
-    # quantization_config=quantization_config,
+    # torch_dtype=torch.bfloat16,
+    # attn_implementation="flash_attention_2",
+    quantization_config=quantization_config,
 )
 model.eval()
 
 
-@spaces.GPU(duration=100)
+@spaces.GPU(duration=120)
 def generate(
     message: str,
     chat_history: list[tuple[str, str]],