add data

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/alpaca_data_zh_51k-clean.json filter=lfs diff=lfs merge=lfs -text
+data/alpaca_gpt4_data_zh-clean.json filter=lfs diff=lfs merge=lfs -text
+data/sharegpt-70k.json filter=lfs diff=lfs merge=lfs -text

data/alpaca_data_zh_51k-clean.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1c1962ed88f95f87ecbe70addd816fa3ade0ee5494a220a3c4972429e7cf111
+size 18810090

data/alpaca_gpt4_data_zh-clean.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:951f1331cacabc7b5de2a5d72592a103be0676daba8d92ae7c67b061639e0f46
+size 35100511

data/read_data.py

@@ -0,0 +1,45 @@
+import json
+from tqdm import tqdm
+
+jsonl_file_path = 'common_zh_70k.jsonl'
+
+results = []
+# 打开JSON Lines文件
+with open(jsonl_file_path, 'r', encoding='utf-8') as file:
+    # 逐行读取文件内容
+    for line in tqdm(file):
+        # 解析JSON行
+        json_object = json.loads(line.strip())
+        
+        # 处理json_object，根据需要执行操作
+        #print(json_object['conversation'])
+        #print(len(json_object['conversation']))
+        #print(json_object['conversation'][0])
+
+        if len(json_object['conversation'])>=2:
+            rr = []
+            for cc in range(len(json_object['conversation'])-1):
+                rr.append([str(json_object['conversation'][cc]['human']), str(json_object['conversation'][cc]['assistant'])])
+
+            info = {
+                "instruction": str(json_object['conversation'][-1]['human']),
+                "input": "",
+                "output": str(json_object['conversation'][-1]['assistant']),
+                "history": rr
+              }
+            results.append(info)
+            
+        if len(json_object['conversation'])==1:
+            info = {
+                "instruction": str(json_object['conversation'][0]['human']),
+                "input": "",
+                "output": str(json_object['conversation'][0]['assistant']),
+                "history": []
+              }
+            results.append(info)
+        
+        # 打印完第一行后终止循环
+        #break
+
+with open('./sharegpt-70k.json', 'w', encoding="utf-8") as f1:
+    json.dump(results, f1, ensure_ascii=False, indent=4)

data/sharegpt-70k.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32bc6e7016fbdab5ee97a97bfb275246a5514b1326d8abfd71f1307b64e9ea8f
+size 287978587