Upload main.py with huggingface_hub
Browse files
main.py
CHANGED
|
@@ -71,7 +71,7 @@ def build_dataset_rank(
|
|
| 71 |
ds = ds.shuffle(seed=42)
|
| 72 |
ds1 = ds
|
| 73 |
original_columns1 = ds1.column_names
|
| 74 |
-
num_proc = 8
|
| 75 |
|
| 76 |
# [MODIFIED] Auto-detect chat format from conversation string
|
| 77 |
# Will be set dynamically in preprocess_function based on actual format
|
|
|
|
| 71 |
ds = ds.shuffle(seed=42)
|
| 72 |
ds1 = ds
|
| 73 |
original_columns1 = ds1.column_names
|
| 74 |
+
num_proc = 1 # Changed from 8 to avoid DeepSpeed pickle issues
|
| 75 |
|
| 76 |
# [MODIFIED] Auto-detect chat format from conversation string
|
| 77 |
# Will be set dynamically in preprocess_function based on actual format
|