Upload cnets.py with huggingface_hub

cnets.py

@@ -668,14 +668,10 @@ class Model(nn.Module):
 
 
 
-            num_processes = num_proc
-            chunk_size = len(dataset) // num_processes + (len(dataset) % num_processes > 0)
-            chunks = [dataset[i:i + chunk_size] for i in range(0, len(dataset), chunk_size)]
-
-            # 创建进程池
-            with multiprocessing.Pool(num_processes) as pool:
-                # 并行处理数据块
-                results = pool.map(process_data, chunks)
+            # Process data sequentially to avoid DeepSpeed pickle issues
+            # (multiprocessing.Pool cannot pickle torch.distributed ProcessGroup)
+            chunks = [dataset[i:i + len(dataset)] for i in range(0, len(dataset), len(dataset))]
+            results = [process_data(chunk) for chunk in chunks]
 
             # 合并结果
             token_dict = merge_dicts(results)