update

tasks/text.py

@@ -86,7 +86,19 @@ async def evaluate_text(request: TextEvaluationRequest):
     # Model inference
     model.eval()
     predictions = np.array([])
+    batch_size = 32
+    
 
+    with torch.no_grad():
+        for i in range(0, len(test_dataset['quote']), batch_size):
+            batch_quotes = test_dataset['quote'][i:i + batch_size]
+            print(f'Processing batch {i // batch_size + 1}')
+            
+            # Tokenize the input data for the current batch
+            tokenized_inputs = tokenizer(batch_quotes, padding=True, truncation=True, return_tensors='pt').to(device)
+            
+            # Forward pass through the model
+            outputs = model(**tokenized_inputs)
     with torch.no_grad():
         print('BEFORE PREDICTION')