backend/data.py

import pandas as pd
from datasets import Dataset

class DataLoader:
    """Class for loading and merging CSV data."""
    def __init__(self, csv_paths):
        self.csv_paths = csv_paths
    
    def load_data(self):
        """Load multiple CSV files and merge them into a single dataset."""
        dataframes = [pd.read_csv(path).dropna() for path in self.csv_paths]
        combined_df = pd.concat(dataframes, ignore_index=True)
        return Dataset.from_pandas(combined_df)

if __name__ == "__main__":
    csv_files = ["data1.csv", "data2.csv"]  # Update with actual file paths
    data_loader = DataLoader(csv_files)
    dataset = data_loader.load_data()
    print(dataset)