Update app.py

app.py

@@ -82,12 +82,12 @@ def generate(submission: list[dict[str, str]], team_id: str) -> list[dict[str, s
 
     start = time.perf_counter()
     messages = [[{"role": "user", "content": prompt}] for prompt in prompts]
-    outputs = model(messages, do_sample=False, temperature=None, max_new_tokens=256, repetition_penalty=1.1)
+    outputs = model(messages, do_sample=False, temperature=None, max_new_tokens=256, repetition_penalty=1.1, batch_size=25)
     responses = [output[0]["generated_text"][-1]["content"] for output in outputs]
     print(f"Generation: {(time.perf_counter() - start):.3f} s")
 
     start = time.perf_counter()
-    predictions = classifier([{"text": p,  "text_pair": r} for p, r in zip(prompts, responses)], top_k=None)
+    predictions = classifier([{"text": p,  "text_pair": r} for p, r in zip(prompts, responses)], top_k=None, batch_size=25)
     print(f"Classifier: {(time.perf_counter() - start):.3f} s")
     
     scores = [p[unsafe_idx]["score"] for p in predictions]