Update app.py

app.py

@@ -30,8 +30,9 @@ list_audios= [[Problema_tarjetaCredito]]
 def gen_conversation(text,max_new_tokens=100): 
   text = "<SN>instruction: " + text + "\n "
   batch = tokenizer2(text, return_tensors='pt')
-  
-  output_tokens = model2.generate(**batch, 
+  batch = {k: v.to('cuda') for k, v in batch.items()}
+  with torch.cuda.amp.autocast():
+      output_tokens = model2.generate(**batch, 
                                     max_new_tokens=max_new_tokens, 
                                     eos_token_id= tokenizer2.eos_token_id,
                                     pad_token_id= tokenizer2.pad_token_id,
@@ -43,6 +44,7 @@ def gen_conversation(text,max_new_tokens=100):
                                     num_beams=3
                                     )
   gc.collect()
+  torch.cuda.memory_summary(device=None, abbreviated=False)
   return tokenizer2.decode(output_tokens[0], skip_special_tokens=True).split("\n")[-1].replace("output:","")
 
 conversacion = ""