app update

app.py

@@ -3,7 +3,7 @@ from transformers import pipeline
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 model_path = "finetuned_phi2"
-model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 
 
@@ -12,7 +12,7 @@ def generate(question, context):
     system_message = "You are a question answering chatbot. Provide a clear and detailed explanation"
     prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n {question} [/INST]" # replace the command here with something relevant to your task
     
-    num_new_tokens = 500  # change to the number of new tokens you want to generate
+    num_new_tokens = 200  # change to the number of new tokens you want to generate
     # Count the number of tokens in the prompt
     num_prompt_tokens = len(tokenizer(prompt)['input_ids'])
     # Calculate the maximum length for the generation