File size: 2,834 Bytes
a9e1e1a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
import json
import os
from tqdm import tqdm
with open("/mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA/playground/data/llava_v1_5_mix665k.json") as f:
llava_v1_5_mix665k = json.load(f) # 665298
with open("/mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA/playground/data/llava_instruct_150k.json") as f:
llava_instruct_150k = json.load(f) # 157712
# Create sets of "id" fields
mix665k_ids = set()
for item in llava_v1_5_mix665k:
all_conv = ""
for cur_conversation in item["conversations"]:
all_conv += cur_conversation["value"]
mix665k_ids.add(f'{item["id"]}_{all_conv}')
instruct_150k_ids = set()
for item in llava_instruct_150k:
all_conv = ""
for cur_conversation in item["conversations"]:
all_conv += cur_conversation["value"]
instruct_150k_ids.add(f'{item["id"]}_{all_conv}')
share_gpt_ids = set()
for item in llava_v1_5_mix665k:
if "image" not in item:
all_conv = ""
for cur_conversation in item["conversations"]:
all_conv += cur_conversation["value"]
share_gpt_ids.add(f'{item["id"]}_{all_conv}') # 40688
# Get "id" fields that are in mix665k but not in instruct_150k and share_gpt
new_ids = mix665k_ids - instruct_150k_ids - share_gpt_ids # 466898
# Get "id" fields that are in mix665k but not in share_gpt
# new_ids = mix665k_ids - share_gpt_ids #624610
# import pdb; pdb.set_trace()
# Filter mix665k data based on new_ids
new_data = []
for item in llava_v1_5_mix665k:
all_conv = ""
for cur_conversation in item["conversations"]:
all_conv += cur_conversation["value"]
if f'{item["id"]}_{all_conv}' in new_ids:
new_data.append(item)
import pdb
pdb.set_trace()
with open("/mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA/playground/data/mixtral_instruct_135K_of_158K_V1.5.json") as f:
new_mixtral_instruct = json.load(f)
# mixtral_instruct_50K_of_80K_V1.json@
# print(len(new_data))
# for _ in new_mixtral_instruct:
# # import pdb; pdb.set_trace()
# if "coco" not in _["image"]:
# _["image"] = f"coco/train2017/{_['image']}"
# new_data.append(_)
# print(len(instruct_150k_ids))
print(len(new_data))
# for _ in tqdm(new_data):
# if "image" in _:
# if "000000442654" in _["image"]:
# all_conv = ""
# for cur_conversation in _["conversations"]:
# all_conv += cur_conversation["value"]
# # if not os.path.exists(f'/mnt/bn/vl-research/workspace/boli01/data/playground/data/{_["image"]}'):
# import pdb; pdb.set_trace()
# Write new_data to a new JSON file
with open("/mnt/bn/vl-research/workspace/boli01/zzzprojects/LLaVA/playground/data/llava_v1_5_mix665k_minus_llava_instruct_150k_minus_sharegpt_plus_mixtral_instruct_135K_of_158K_V1.5.json", "w") as f:
json.dump(new_data, f)
|