import pandas as pd import numpy as np from sklearn.model_selection import train_test_split from datasets import Dataset, DatasetDict def balance_classes(df: pd.DataFrame, max_per_class: int = 2000) -> pd.DataFrame: """ Cap each class at max_per_class to prevent dominant classes from overwhelming the minority classes during training. """ return ( df.groupby("label", group_keys=False) .apply(lambda g: g.sample(n=min(len(g), max_per_class), random_state=42)) .reset_index(drop=True) ) def split_dataset(df: pd.DataFrame) -> DatasetDict: """Stratified 80/10/10 train/val/test split.""" def manual_split(frame: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: shuffled = frame.sample(frac=1, random_state=42).reset_index(drop=True) n_rows = len(shuffled) if n_rows < 3: return shuffled, shuffled.iloc[0:0], shuffled.iloc[0:0] train_size = max(1, int(round(n_rows * 0.8))) val_size = max(1, int(round(n_rows * 0.1))) test_size = n_rows - train_size - val_size if test_size < 1: deficit = 1 - test_size reducible_train = max(0, train_size - 1) take_from_train = min(deficit, reducible_train) train_size -= take_from_train deficit -= take_from_train reducible_val = max(0, val_size - 1) take_from_val = min(deficit, reducible_val) val_size -= take_from_val deficit -= take_from_val test_size = n_rows - train_size - val_size train_df = shuffled.iloc[:train_size].reset_index(drop=True) val_df = shuffled.iloc[train_size:train_size + val_size].reset_index(drop=True) test_df = shuffled.iloc[train_size + val_size:].reset_index(drop=True) return train_df, val_df, test_df try: label_counts = df["label"].value_counts() can_stratify = len(label_counts) > 1 and label_counts.min() >= 2 split_kwargs = {"random_state": 42} if can_stratify: split_kwargs["stratify"] = df["label"] train_df, temp_df = train_test_split(df, test_size=0.2, **split_kwargs) temp_counts = temp_df["label"].value_counts() can_stratify_temp = len(temp_counts) > 1 and temp_counts.min() >= 2 split_kwargs_temp = {"random_state": 42} if can_stratify_temp: split_kwargs_temp["stratify"] = temp_df["label"] val_df, test_df = train_test_split(temp_df, test_size=0.5, **split_kwargs_temp) except ValueError: train_df, val_df, test_df = manual_split(df) print(f"Train: {len(train_df)} | Val: {len(val_df)} | Test: {len(test_df)}") return DatasetDict({ "train": Dataset.from_pandas(train_df, preserve_index=False), "validation": Dataset.from_pandas(val_df, preserve_index=False), "test": Dataset.from_pandas(test_df, preserve_index=False), })