Upload llm.py

llm.py

@@ -84,9 +84,12 @@ def generate_soap_note(transcript: str) -> str:
     with torch.inference_mode():
         generated_ids = model.generate(
             **inputs,
-            max_new_tokens=1024,
-            do_sample=False,
-            repetition_penalty=1.3,
+            max_new_tokens=640,
+            do_sample=True,
+            temperature=0.2,
+            top_p=0.9,
+            repetition_penalty=1.15,
+            no_repeat_ngram_size=3,
         )
 
     new_tokens = generated_ids[0][input_len:]