File size: 6,499 Bytes
bf928ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | import pickle
from sklearn.model_selection import train_test_split
import random
from random import sample
from _bootstrap import use_project_root
use_project_root()
random.seed(123)
with open('conf/data/asPICKLE/clusters.pickle', 'rb') as binary_reader:
clusters = pickle.load(binary_reader)
with open('conf/data/asPICKLE/intra_group_binding.pickle', 'rb') as binary_reader:
intra_group_binding = pickle.load(binary_reader)
with open('conf/data/asPICKLE/inter_group_binding.pickle', 'rb') as binary_reader:
inter_group_binding = pickle.load(binary_reader)
train_pos = []
test_pos = []
for cluster in clusters:
train, test = train_test_split(
cluster, test_size=0.2, random_state=123) # 80% train , 20% test
train_pos.append(train)
test_pos.append(test)
train, test = train_test_split(
intra_group_binding, test_size=0.2, random_state=123) # 80% train , 20% test
train_pos.append(train)
test_pos.append(test)
total_pos = 0
for cls in clusters:
total_pos = total_pos + len(cls)
total_pos = total_pos + len(intra_group_binding)
total_train_pos = 0
for T_P in train_pos:
total_train_pos = total_train_pos + len(T_P)
total_test_pos = 0
for T_P in test_pos:
total_test_pos = total_test_pos + len(T_P)
t_train_pos = total_pos * 0.8
t_test_pos = total_pos * 0.2
neg_data = []
for _ in range(total_pos):
item = sample(inter_group_binding, 1)[0]
item_ = (item[0], item[1], item[2], item[3], item[4], item[5], 0)
neg_data.append(item_)
train_data_pos = []
test_data_pos = []
for l in train_pos:
for item in l:
item_ = (item[0], item[1], item[2], item[3], item[4], item[5], 1)
train_data_pos.append(item_)
for l in test_pos:
for item in l:
item_ = (item[0], item[1], item[2], item[3], item[4], item[5], 1)
test_data_pos.append(item_)
train_data_neg, test_data_neg = train_test_split(
neg_data, test_size=0.2, random_state=123) # 80% train , 20% test
train_data_pos, val_data_pos = train_test_split(
train_data_pos, test_size=0.05, random_state=123) # 95% train , 5% validation
train_data_neg, val_data_neg = train_test_split(
train_data_neg, test_size=0.05, random_state=123) # 95% train , 5% validation
with open('conf/data/asPICKLE/train_data_pos.pickle', 'wb') as binary_writer:
pickle.dump(train_data_pos, binary_writer)
with open('conf/data/asPICKLE/val_data_pos.pickle', 'wb') as binary_writer:
pickle.dump(val_data_pos, binary_writer)
with open('conf/data/asPICKLE/test_data_pos.pickle', 'wb') as binary_writer:
pickle.dump(test_data_pos, binary_writer)
with open('conf/data/asPICKLE/train_data_neg.pickle', 'wb') as binary_writer:
pickle.dump(train_data_neg, binary_writer)
with open('conf/data/asPICKLE/val_data_neg.pickle', 'wb') as binary_writer:
pickle.dump(val_data_neg, binary_writer)
with open('conf/data/asPICKLE/test_data_neg.pickle', 'wb') as binary_writer:
pickle.dump(test_data_neg, binary_writer)
print('size of pos data is ', total_pos)
print('size of train pos data is ', str(len(train_data_pos)))
print('size of val pos data is ', str(len(val_data_pos)))
print('size of test pos data is ', str(len(test_data_pos)))
print('size of neg data is ', str(len(neg_data)))
print('size of train neg data is ', str(len(train_data_neg)))
print('size of val neg data is ', str(len(val_data_neg)))
print('size of test neg data is ', str(len(test_data_neg)))
# combine the positive and negative samples and shuffle them for each dataset (train, val, test)
train_data_all = []
val_data_all = []
test_data_all = []
for item in train_data_pos:
train_data_all.append(item)
for item in train_data_neg:
train_data_all.append(item)
random.shuffle(train_data_all)
for item in val_data_pos:
val_data_all.append(item)
for item in val_data_neg:
val_data_all.append(item)
random.shuffle(val_data_all)
for item in test_data_pos:
test_data_all.append(item)
for item in test_data_neg:
test_data_all.append(item)
random.shuffle(test_data_all)
with open('conf/data/asPICKLE/train_data_all.pickle', 'wb') as binary_writer:
pickle.dump(train_data_all, binary_writer)
with open('conf/data/asPICKLE/val_data_all.pickle', 'wb') as binary_writer:
pickle.dump(val_data_all, binary_writer)
with open('conf/data/asPICKLE/test_data_all.pickle', 'wb') as binary_writer:
pickle.dump(test_data_all, binary_writer)
print('size of train data is ', str(len(train_data_all)))
print('size of val data is ', str(len(val_data_all)))
print('size of test data is ', str(len(test_data_all)))
# separate the CDRs with it's number
train_CDR_antigen = []
val_CDR_antigen = []
test_CDR_antigen = []
for item in train_data_all:
CDR_1 = (item[0], item[1], item[2], item[5], item[6], 1)
CDR_2 = (item[0], item[1], item[3], item[5], item[6], 2)
CDR_3 = (item[0], item[1], item[4], item[5], item[6], 3)
train_CDR_antigen.append(CDR_1)
train_CDR_antigen.append(CDR_2)
train_CDR_antigen.append(CDR_3)
random.shuffle(train_CDR_antigen)
for item in val_data_all:
CDR_1 = (item[0], item[1], item[2], item[5], item[6], 1)
CDR_2 = (item[0], item[1], item[3], item[5], item[6], 2)
CDR_3 = (item[0], item[1], item[4], item[5], item[6], 3)
val_CDR_antigen.append(CDR_1)
val_CDR_antigen.append(CDR_2)
val_CDR_antigen.append(CDR_3)
random.shuffle(val_CDR_antigen)
for item in test_data_all:
CDR_1 = (item[0], item[1], item[2], item[5], item[6], 1)
CDR_2 = (item[0], item[1], item[3], item[5], item[6], 2)
CDR_3 = (item[0], item[1], item[4], item[5], item[6], 3)
test_CDR_antigen.append(CDR_1)
test_CDR_antigen.append(CDR_2)
test_CDR_antigen.append(CDR_3)
random.shuffle(test_CDR_antigen)
with open('conf/data/asPICKLE/train_CDR_antigen.pickle', 'wb') as binary_writer:
pickle.dump(train_CDR_antigen, binary_writer)
with open('conf/data/asPICKLE/val_CDR_antigen.pickle', 'wb') as binary_writer:
pickle.dump(val_CDR_antigen, binary_writer)
with open('conf/data/asPICKLE/test_CDR_antigen.pickle', 'wb') as binary_writer:
pickle.dump(test_CDR_antigen, binary_writer)
print('size of train_CDR_antigen data is ', str(len(train_CDR_antigen)))
print('size of val_CDR_antigen data is ', str(len(val_CDR_antigen)))
print('size of test_CDR_antigen data is ', str(len(test_CDR_antigen)))
print('done')
|