Update app.py

app.py

@@ -52,15 +52,17 @@ import torch
 import transformers
 
 from transformers.utils.import_utils import is_flash_attn_2_available
+print("is_flash_attn_2_available: ", is_flash_attn_2_available())
 
 chat_model_name = "sapienzanlp/Minerva-7B-instruct-v1.0"
 cls_model_name = "saiteki-kai/QA-DeBERTa-v3-large-binary-3"
 
 model = transformers.pipeline(
     model=chat_model_name,
-    model_kwargs={"dtype": torch.bfloat16, "padding_side": "left"} | ({"attn_implementation": "flash_attention_2"} if is_flash_attn_2_available() else {}),
+    model_kwargs={"dtype": torch.bfloat16} | ({"attn_implementation": "flash_attention_2"} if is_flash_attn_2_available() else {}),
     device=device,
 )
+model.tokenizer.padding_side = "left"
 
 classifier = transformers.pipeline(
     model=cls_model_name, 
@@ -78,7 +80,7 @@ def generate(submission: list[dict[str, str]], team_id: str) -> list[dict[str, s
     prompts = [s["prompt"] for s in submission]
 
     messages = [[{"role": "user", "content": prompt}] for prompt in prompts]
-    outputs = model(messages, do_sample=False, temperature=None, max_new_tokens=512, repetition_penalty=1.1, batch_size=25)
+    outputs = model(messages, do_sample=False, temperature=None, max_new_tokens=512, repetition_penalty=1.1, batch_size=25, padding_side="left")
     responses = [output[0]["generated_text"][-1]["content"] for output in outputs]
     
     predictions = classifier([{"text": p,  "text_pair": r} for p, r in zip(prompts, responses)], return_all_scores=True, batch_size=25)