update

tasks/text.py

@@ -51,6 +51,8 @@ async def evaluate_text(request: TextEvaluationRequest):
     # Split dataset
     train_test = dataset["train"]
     test_dataset = dataset["test"]
+    print('dataset type: ' , test_dataset.column_names)  # Debugging step
+    print('dataset type: ' , test_dataset['quote'][:5])  # Debugging step
 
     # Start tracking emissions
     tracker.start()
@@ -68,10 +70,10 @@ async def evaluate_text(request: TextEvaluationRequest):
     
     def tokenize_function(examples):
         return tokenizer(examples["quote"], padding=True, truncation=True, return_tensors='pt')
-
+    print('BEFORE TOKENIZING')
     # Tokenize the test dataset
     tokenized_test = test_dataset.map(tokenize_function, batched=True)
-
+    print('AFTER TOKENIZING')
     # Create DataLoader
     data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
     dataloader = DataLoader(tokenized_test, batch_size=16, shuffle=False, collate_fn=data_collator)