Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import json
 import logging
+import numpy as np
 from datasets import load_dataset
 from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer
 from sklearn.metrics import accuracy_score, precision_recall_fscore_support
@@ -58,7 +59,9 @@ def setup_training():
     # Tokenize the dataset
     logging.info("Tokenizing the dataset")
     def tokenize_function(examples):
-        return tokenizer(examples[config['text_column']], padding="max_length", truncation=True)
+        # Concatenate all feature columns into a single input
+        features = np.stack([examples[col] for col in config['text_columns']], axis=1)
+        return tokenizer(features.tolist(), padding="max_length", truncation=True)
 
     tokenized_datasets = dataset.map(tokenize_function, batched=True)
     logging.info("Dataset tokenization completed")