conf/analysis/utils/util.py

import os
import operator
import pickle
import time
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import math
import itertools
import random
import os
from polyleven import levenshtein
import operator
import pickle
import time
import tensorflow as tf

tf.random.set_seed(35)
np.random.seed(35)

rna_vocab = {"A":0,
             "C":1,
             "G":2,
             "U":3,
             "*":4}

rev_rna_vocab = {v:k for k,v in rna_vocab.items()}

def recover_seq(samples, rev_charmap=rev_rna_vocab):
    """Convert samples to strings and save to log directory."""
    
    char_probs = samples
    argmax = np.argmax(char_probs, 2)
    seqs = []
    for line in argmax:
        s = "".join(rev_charmap[d] for d in line)
        s = s.replace('*','')
        s = s.replace('U','T')
        seqs.append(s)
    return seqs 

def file_to_list(file_name,size):
    data = []
    with open(file_name,'r') as f:
        lines = f.readlines()
        
        for seq in lines:
            seq_ = seq.replace('\n','')
            data.append(seq_)
            # if len(seq) == size:
            #     data.append(seq)

    return data

def one_hot_encode(seq):
    mapping = dict(zip("ACGT", range(4)))
    seq = seq.replace('U','T')    
    seq2 = [mapping[i] for i in seq]
    return np.eye(4)[seq2]

def encode(seqs):
    return np.reshape([one_hot_encode(seqs[i]) for i in range(len(seqs))],(np.array(seqs).shape[0],-1))

min_len = None
nuc_dict = {'a':[1.0,0.0,0.0,0.0],'c':[0.0,1.0,0.0,0.0],'g':[0.0,0.0,1.0,0.0], 
                'u':[0.0,0.0,0.0,1.0], 't':[0.0,0.0,0.0,1.0], 
                'n':[0.0,0.0,0.0,0.0], 'x':[1/4,1/4,1/4,1/4]}

def encode_seq_framepool(seq, max_len=128):
    # print(seq)
    length = len(seq)
    if max_len > 0 and min_len is None:
        padding_needed = max_len - length
        seq = "N"*padding_needed + seq
    if min_len is not None:
        if len(seq) < min_len:
            seq = "N"*(min_len - len(seq)) + seq

        if len(seq) > min_len:
            seq = seq[(len(seq) - min_len):]
    seq = seq.lower()
    one_hot = np.array([nuc_dict[x] for x in seq]) # get stacked on top of each other
    
    return one_hot

def list_to_file(filename,list):
    with open(filename + ".txt", 'w') as f:
        for element in list[:-1]:
            f.write(element+"\n")
        f.write(list[-1])       

def tsne_natural(file_name, length, key='seq'):
    df = pd.read_csv(file_name)
    seqs = np.array(df[key]).tolist()

    selected_seqs = []

    for i in range(len(seqs)):
        seq = seqs[i]
        seq = seq.upper()
        if seq not in selected_seqs and len(seq) == length:
            selected_seqs.append(seq)

    return selected_seqs

def read_real(file_name, UTR_LEN, key='seq', all= True, samples= 128 ):
    df = pd.read_csv(file_name)
    seqs = np.array(df[key]).tolist()

    selected_seqs = []

    for i in range(len(seqs)):
        if len(seqs[i]) < (UTR_LEN + 1) and len(seqs[i]) > int(UTR_LEN/2):
            seqs[i] = seqs[i].upper()
            if seqs[i] not in selected_seqs:
                selected_seqs.append(seqs[i])

    if all:
        return selected_seqs
    
    else:
        indices = []

        for i in range(len(selected_seqs)):

            indices.append(i)

        samples = np.random.choice(len(indices),samples,replace=False)

        chosen = []

        for i in range(len(samples)):
            chosen.append(selected_seqs[samples[i]])
            
        return chosen
    
def random_sample(length):
    rna_vocab = {"A":0,
             "C":1,
             "G":2,
             "T":3}

    rev_rna_vocab = {v:k for k,v in rna_vocab.items()}

    mapping = dict(zip([0,1,2,3],"ACGT"))
    gc_limit = random.randrange(2,9)
    gc_limit = float(gc_limit)/10 
    gc_limit = 0.6
    sample = ''
    for i in range(length):
        r = random.random()
        if r < gc_limit:
            sample += random.choice(['C','G'])
        else:
            sample += random.choice(['A','T'])

    return sample 

def random_data(length, size):
    samples = []
    for i in range(size):
        samples.append(random_sample(length))

    return samples

def tsne_gen(path,length=128,BATCH_SIZE=64,DIM=40):
    wgan = tf.keras.models.load_model(path)

    selected = []
    while len(selected) < BATCH_SIZE:

        noise = tf.Variable(tf.random.normal(shape=[64,DIM]))

        sequences_init = wgan(noise)

        gen_seqs_init = sequences_init.numpy().astype('float')

        seqs_gen_init = recover_seq(gen_seqs_init, rev_rna_vocab)

        for i in range(len(seqs_gen_init)):
            if len(seqs_gen_init[i]) == length:
                selected.append(seqs_gen_init[i])

    return selected[:BATCH_SIZE]

def generate_data(path,BATCH_SIZE=64,UTR_LEN=128,DIM=40):
    wgan = tf.keras.models.load_model(path)

    gens = []

    while len(gens) < BATCH_SIZE:

        noise = tf.Variable(tf.random.normal(shape=[BATCH_SIZE,DIM]))

        sequences_init = wgan(noise)

        gen_seqs_init = sequences_init.numpy().astype('float')

        seqs_gen_init = recover_seq(gen_seqs_init, rev_rna_vocab)

        for i in range(len(seqs_gen_init)):
            if len(seqs_gen_init[i]) > int(UTR_LEN/2) and len(seqs_gen_init[i])< UTR_LEN+1:
                gens.append(seqs_gen_init[i])
                if len(gens) == BATCH_SIZE:
                    break

    return gens[:BATCH_SIZE]

def gc_percentage(seq):
    count = 0.0
    for char in seq:
        if char == 'C' or char == 'G':
            count +=1

    return float(count/len(seq))

def get_gc_content(data):
    gc_content = []
    for seq in data:
        seq.replace('\n','')
        seq.replace('*','')
        gc = gc_percentage(seq)
        gc_content.append(gc)

    return gc_content

def get_gc_content_many(data):

    collection = []
    gc_contents = []
    for seq in data:
        seq = seq.upper()
        seq.replace('\n','')
        seq.replace('*','')
        gc = gc_percentage(seq)
        gc_contents.append(gc)
    
    return gc_contents

def get_4mers():
    neucs = ['A','C','G','T']
    
    mers = [p for p in itertools.product(neucs, repeat=4)]
    for i in range(len(mers)):
        mers[i] = mers[i][0] + mers[i][1] + mers[i][2] + mers[i][3]

    return mers

def get_4mer_dic(seqs):
    
    _4mers = get_4mers()
    length = 0
    dics = []
    c = 0
    for seq in seqs:
        c += 1
        # dic = {}
        dic = []
        for item in _4mers:
            dic.append(0)

        # Iterate With the Sliding Window
        length = len(seq)
        limit = length - 4
        for i in range(limit):
            mer = seq[i:i+4]
            indx = _4mers.index(mer)
            dic[indx] += 1

        dics.append(dic)

        if c % 2000 == 0:
            print("YUE")
    
    return dics

def euclidean_kmer(item, ref, mers):
    dist = 0
    # diff = np.linalg.norm(item-re)
    for mer in range(len(mers)):
        
        diff = item[mer] - ref[mer]
        dist += diff * diff

    return math.sqrt(dist)

def euclidean_kmer_all(item,refs,mers):
    dists = []
    for ref in refs:
        dist = euclidean_kmer(item,ref,mers)
        if dist > 0:
            dists.append(dist)

    return min(dists)

def kmer_dist_list(source, target, mers):
    dists = []
    for item in source:
        dists.append(euclidean_kmer_all(item, target, mers))

    return dists

def calc_dist_kmer(source,target):
    mers = get_4mers()
    source_dic = get_4mer_dic(source)
    target_dic = get_4mer_dic(target)
    print("Dicts Processed")
    dists = kmer_dist_list(source_dic, target_dic, mers)

    return dists

def hamming_dist(src, target):
    
    dists = []
    for i in range(len(src)):
        smallest = np.inf
        for j in range(len(target)):
            dist = levenshtein(src[i],target[j])
            if dist > 0 and dist < smallest:
                smallest = dist

        dists.append(dist)

    return np.array(dists)








def one_hot_motif(seq,length=128,complementary=False):
    """

    one_hot encoding on sequence

    complementary: encode nucleatide into complementary one

    """
    
    if length == -1:
        length = len(seq)
    
    # seq = str(seq)
    # setting
    seq = list(seq.replace("U","T"))
    seq_len = len(seq)
    complementary = -1 if complementary else 1
    # compose dict
    keys = ['A', 'C', 'G', 'T'][::complementary]
    oh_dict = {keys[i]:i for i in range(4)}
    # array
    oh_array = np.zeros((length,4))
    for i,C in enumerate(seq):
        try:
            oh_array[i,oh_dict[C]]=1
        except:
            continue      # for nucleotide that are not in A C G T   
    return oh_array

def one_hot_all_motif(seqs):
    length = np.max([len(seq) for seq in seqs])
    return [one_hot_motif(seqs[i], length = 128) for i in range(len(seqs))]