Update app.py

app.py

@@ -13,6 +13,7 @@ model = AutoModelForCausalLM.from_pretrained(model_id)
 
 # Move the model to the device
 model = model.to(device)
+model.eval()  # Set the model to evaluation mode
 
 def answer_question(
         prompt,
@@ -23,20 +24,20 @@ def answer_question(
         do_sample=True,
         **kwargs,
 ):
-    inputs = tokenizer(prompt, return_tensors="pt")
-    # Move the inputs to the device
-    inputs = {key: val.to(device) for key, val in inputs.items()}
-    input_ids = inputs["input_ids"]
-    attention_mask = inputs["attention_mask"]
-    generation_config = GenerationConfig(
-        temperature=temperature,
-        top_p=top_p,
-        top_k=top_k,
-        num_beams=num_beams,
-        do_sample=do_sample,
-        **kwargs,
-    )
-    with torch.no_grad():
+    with torch.no_grad():  # Disable gradient calculation
+        inputs = tokenizer(prompt, return_tensors="pt")
+        # Move the inputs to the device
+        inputs = {key: val.to(device) for key, val in inputs.items()}
+        input_ids = inputs["input_ids"]
+        attention_mask = inputs["attention_mask"]
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,
+            do_sample=do_sample,
+            **kwargs,
+        )
         generation_output = model.generate(
             input_ids=input_ids,
             attention_mask=attention_mask,
@@ -45,29 +46,12 @@ def answer_question(
             output_scores=True,
             max_new_tokens=512,
             eos_token_id=tokenizer.eos_token_id
-
         )
     s = generation_output.sequences[0]
     output = tokenizer.decode(s, skip_special_tokens=True)
     return output.split(" Response:")[1]
 
-example_prompt = """
-Below is an instruction that describes a task, paired with an input that provides further context.Write a response that appropriately completes the request.
-
-### Instruction:
-If you are a doctor, please answer the medical questions based on the patient's description.
-
-### Input:
-Hi i have sore lumps under the skin on my legs. they started on my left ankle and are approx 1 - 2cm diameter and are spreading up onto my thies. I am eating panadol night and anti allergy pills (Atarax). I have had this for about two weeks now. Please advise.
-
-### Response:
-"""
-
-print(answer_question(example_prompt))
-
-
 def gui_interface(prompt):
-    
     prompt="""
     Below is an instruction that describes a task, paired with an input that provides further context.Write a response that appropriately completes the request.
     
@@ -78,10 +62,7 @@ def gui_interface(prompt):
     """+prompt+"""
     ### Response:
     """
-
-    print(prompt)
     return answer_question(prompt)
 
 iface = gr.Interface(fn=gui_interface, inputs="text", outputs="text")
 iface.launch()
-