Update app.py

app.py

@@ -12,9 +12,6 @@ from transformers import (
     TextIteratorStreamer,
 )
 
-import subprocess
-subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
-
 DESCRIPTION = """\
 # ORLM LLaMA-3-8B
 Hello! I'm ORLM-LLaMA-3-8B, here to automate your optimization modeling tasks! Check our [repo](https://github.com/Cardinal-Operations/ORLM) and [paper](https://arxiv.org/abs/2405.17743)!
@@ -33,22 +30,14 @@ MAX_MAX_NEW_TOKENS = 4096
 DEFAULT_MAX_NEW_TOKENS = 4096
 MAX_INPUT_TOKEN_LENGTH = 2048
 
-# quantization_config = BitsAndBytesConfig(
-#     load_in_4bit=True,
-#     bnb_4bit_compute_dtype=torch.bfloat16,
-#     bnb_4bit_use_double_quant=True,
-#     bnb_4bit_quant_type= "nf4")
-# quantization_config = BitsAndBytesConfig(load_in_8bit=True)
-
 model_id = "CardinalOperations/ORLM-LLaMA-3-8B"
-tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
-    torch_dtype=torch.bfloat16,
-    attn_implementation="flash_attention_2",
-    # quantization_config=quantization_config,
+    quantization_config=BitsAndBytesConfig(load_in_8bit=True),
 )
+model.config.sliding_window = 4096
 model.eval()