Upload layer_diff_dataset/cat_txt.py with huggingface_hub

layer_diff_dataset/cat_txt.py

@@ -0,0 +1,38 @@
+import os 
+# import 
+# root_dir = '/mnt/workspace/workgroup/sihui.jsh/layer_diff_dataset/train'
+# txt_list = ['im_rgba1.txt','im_rgba2.txt','im_rgba3.txt','im_rgba4.txt','im_rgba5.txt','im_rgba6.txt']
+
+def merge_txt_files(directory, output_file):
+    """
+    将指定目录下的所有文本文件合并为一个文件。
+    
+    参数：
+    - directory: 存放文本文件的目录路径。
+    - output_file: 合并后生成文件的路径。
+    """
+    # 获取目录下所有的.txt文件
+    txt_files = [f for f in os.listdir(directory) if f.endswith('.txt')]
+    
+    # 按文件名排序，以确保文件按顺序合并
+    txt_files.sort()
+
+    # 开始合并文件
+    with open(output_file, 'w', encoding='utf-8') as outfile:
+        for filename in txt_files:
+            filepath = os.path.join(directory, filename)
+            
+            # 将每个文件的内容追加到输出文件
+            with open(filepath, 'r', encoding='utf-8') as infile:
+                outfile.write(infile.read())
+                # 确保每个文件内容之间有明确的分隔
+                # outfile.write('\n')
+        
+    print(f"All text files have been merged into {output_file}.")
+
+# 替换为你的源目录
+source_directory = '/mnt/workspace/workgroup/sihui.jsh/layer_diff_dataset/train'
+# 替换为你希望输出的文件路径和名称
+output_filepath = os.path.join(source_directory,'im_rgba.txt')
+
+merge_txt_files(source_directory, output_filepath)