File size: 3,310 Bytes
bf928ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | import pickle
import os
import random
from _bootstrap import use_project_root
from model import tokenization
from model.features import CDR_Ag_Processor, convert_examples_to_features
use_project_root()
def get_Feature(tsv_file_path, kmer, max_seq_length):
vocab_file = "conf/vocab/vocab_" + str(kmer) + "kmer.txt"
processor = CDR_Ag_Processor()
label_list = processor.get_labels()
cdr_number_list = ["1", "2", "3"]
tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=True)
examples = processor.get_examples(tsv_file_path)
features = convert_examples_to_features(examples, label_list, cdr_number_list, max_seq_length, tokenizer)
return features
# the max len of the CDRs is 24, so when k = 1 the max tokens is 24
# the max len of the AGs is 2373 , so when k = 3 the max tokens is (2373 - (3-1)) = 2371
cdr_kmer = 3
ag_kmer = 3
if not os.path.exists('./conf/data/features'):
os.makedirs('./conf/data/features')
tsv_path = "conf/data/asTSV/cdr_kmer" + str(cdr_kmer) + "_ag_kmer" + str(ag_kmer) + "/"
ag_max_seq_length = 2371
cdr_max_seq_length = 24
cdr_tsv_file_name = "CDR_te.tsv"
ag_tsv_file_name = "Ag_te.tsv"
cdr_tsv_file_path = tsv_path + cdr_tsv_file_name
ag_tsv_file_path = tsv_path + ag_tsv_file_name
cdr_features = get_Feature(cdr_tsv_file_path, cdr_kmer, cdr_max_seq_length)
ag_features = get_Feature(ag_tsv_file_path, ag_kmer, ag_max_seq_length)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/cdr_features_te.pickle', 'wb') as binary_writer:
pickle.dump(cdr_features, binary_writer)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/ag_features_te.pickle', 'wb') as binary_writer:
pickle.dump(ag_features, binary_writer)
##################################################
cdr_tsv_file_name = "CDR_tr.tsv"
ag_tsv_file_name = "Ag_tr.tsv"
cdr_tsv_file_path = tsv_path + cdr_tsv_file_name
ag_tsv_file_path = tsv_path + ag_tsv_file_name
cdr_features = get_Feature(cdr_tsv_file_path, cdr_kmer, cdr_max_seq_length)
ag_features = get_Feature(ag_tsv_file_path, ag_kmer, ag_max_seq_length)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/cdr_features_tr.pickle', 'wb') as binary_writer:
pickle.dump(cdr_features, binary_writer)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/ag_features_tr.pickle', 'wb') as binary_writer:
pickle.dump(ag_features, binary_writer)
##################################################
cdr_tsv_file_name = "CDR_val.tsv"
ag_tsv_file_name = "Ag_val.tsv"
cdr_tsv_file_path = tsv_path + cdr_tsv_file_name
ag_tsv_file_path = tsv_path + ag_tsv_file_name
cdr_features = get_Feature(cdr_tsv_file_path, cdr_kmer, cdr_max_seq_length)
ag_features = get_Feature(ag_tsv_file_path, ag_kmer, ag_max_seq_length)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/cdr_features_val.pickle', 'wb') as binary_writer:
pickle.dump(cdr_features, binary_writer)
with open('conf/data/features/cdr_kmer' + str(cdr_kmer) + '_ag_kmer' + str(ag_kmer) + '/ag_features_val.pickle', 'wb') as binary_writer:
pickle.dump(ag_features, binary_writer)
print("done")
|