| import os |
| import pandas as pd |
|
|
| def merge_csv_files(directories, output_file, target_name): |
| dfs = [] |
| for directory in directories: |
| csv_path = os.path.join(directory, target_name) |
| if os.path.isfile(csv_path): |
| dfs.append(pd.read_csv(csv_path)) |
| if not dfs: |
| print(f"No {target_name} found.") |
| return |
| common_columns = set(dfs[0].columns) |
| for df in dfs[1:]: |
| common_columns &= set(df.columns) |
| merged = pd.concat([df[list(common_columns)] for df in dfs], ignore_index=True) |
| |
| merged = merged.sample(frac=1).reset_index(drop=True) |
| merged.to_csv(output_file, index=False) |
|
|
| if __name__ == "__main__": |
| directories = ["../datasets_vithsd", "../datasets_vihsd_gemini" , "../datasets_voz_gemini"] |
| merge_csv_files(directories, "../datasets/train.csv", "train.csv") |
| merge_csv_files(directories, "../datasets/dev.csv", "dev.csv") |
| merge_csv_files(directories, "../datasets/test.csv", "test.csv") |
