Upload layer_diff_dataset/move_file.py with huggingface_hub

layer_diff_dataset/move_file.py

@@ -0,0 +1,57 @@
+import os
+import subprocess
+
+# source_directory = "/mnt/workspace/workgroup/sihui.jsh/pose_transfer_3/exp/pose_transfer"
+# target_directory = "exp/pose_transfer"
+def copy_without_folder(source_dir, target_dir, exclude_folder):
+    # 确保目标目录存在
+    # subprocess.run(["ossutil", "mb", target_dir], stderr=subprocess.PIPE)
+
+    # 获取源目录中的所有文件和子文件夹
+    items = os.listdir(source_dir)
+
+    for item in items:
+        source_item_path = os.path.join(source_dir, item)
+        # if not os.path.isdir(source_item_path):
+        #     continue
+        # 如果是要排除的文件夹，则跳过
+        if item in exclude_folder:
+            # print(f"Skipping folder: {source_item_path}")
+            continue
+        target_item_path = os.path.join(target_dir, item)
+        print(item)
+        if os.path.isdir(source_item_path):
+            sub_items = os.listdir(source_item_path)
+            for sub_item in sub_items:
+                print(sub_item)
+                sub_source_item_path = os.path.join(source_item_path, sub_item)
+                sub_target_item_path = os.path.join(target_item_path, sub_item)
+                # 使用 cp -r 复制目录
+                print(f"Copied folder: {sub_source_item_path} to {sub_target_item_path}")
+                subprocess.run(["huggingface-cli", "upload", "zjuJish/layer_diff_dataset", sub_source_item_path, sub_target_item_path], stderr=subprocess.PIPE)
+        else:
+            # 使用 cp -r 复制目录
+            print(f"Copied folder: {source_item_path} to {target_item_path}")
+            subprocess.run(["huggingface-cli", "upload", "zjuJish/layer_diff_dataset", source_item_path, target_item_path], stderr=subprocess.PIPE)
+
+if __name__ == "__main__":
+    # source_directory = "/mnt/workspace/workgroup/sihui.jsh/codes/CogVideo"
+    source_directory = "/mnt/workspace/workgroup/sihui.jsh/layer_diff_dataset"
+    target_directory = "layer_diff_dataset"
+    
+    # # 打开文件并读取每行内容
+    # file_path = "move_list.txt"  # 替换为你的文件路径
+    # lines_list = []
+
+    # with open(file_path, 'r', encoding='utf-8') as file:
+    #     # 逐行读取文件并存入列表
+    #     lines_list = file.readlines()
+
+    # # 去掉每行的换行符
+    # exclude_folder_name = [line.strip() for line in lines_list]
+    
+    # exclude_folder_name = ["sat", "CogVideoX-2b-sat"]
+    
+    # exclude_folder_name = [".ossutil_checkpoint", "ckpts_2b", "ckpts_2b_lora", "ossutil_output"]
+    exclude_folder_name = ["Transparant_dragdata_550","train"]
+    copy_without_folder(source_directory, target_directory, exclude_folder_name)