Upload 9 files

Tesei-trained_Model/filter_dataset/README.txt

@@ -0,0 +1,28 @@
+Sequence Processing and Filtering Pipeline
+Last updated by Lilianna Houston, Sept. 4th 2024
+
+This pipeline processes a dataset of protein sequences in preparation for Machine Learning. It identifies homologs using length, composition and the NIH Blast tool. Follow the steps below to execute the analysis.
+
+Workflow Overview:
+
+--> Upload Sequence Data
+
+Begin by uploading the data file containing your sequences. This pipeline was originally used on the Tesei 2024 human IDP proteome but can be used on any dataset. The Tesei 2024 set is used an example here and the original datafile is named "LL28k_w2_NC_T37_cs150_out_02.csv".
+
+--> Identify Length-Based Sequence Pairs
+
+Run identify_length_pairs.py to identify pairs of sequences where the difference in length is 15 or less.
+
+--> Calculate Composition Differences
+
+Simultaneously, run full_comp_dif.py to compute the composition differences between every possible pair of sequences. This step will generate two outputs:
+    full_comp_difs.npy: A square NumPy array representing the composition differences across the dataset.
+    LL28k_w2_NC_T37_cs150_out_02_composition.csv: A CSV file that includes the original dataset with additional columns for the counts of each amino acid.
+
+--> Filter Pairs Based on Length and Composition
+
+Execute test_compo.py to filter pairs that meet both the length and composition difference criteria. This will produce indice_pairs_w_compo.csv, which contains the indices of the final pairs along with their composition differences.
+
+--> Remove Homologous and NaN Sequences
+
+Finally, run blast_filter.py to filter out sequences with NaN w2 values and homologous sequences from the original dataset. The output will be LL28k_filtered_nanW2s_noRepeats.csv, a refined dataset excluding the undesired sequences.

Tesei-trained_Model/filter_dataset/blast_filter.py

@@ -0,0 +1,167 @@
+from Bio.Blast.Applications import NcbiblastpCommandline
+import os
+import pandas as pd
+import time
+import networkx as nx
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Load data
+dataset = pd.read_csv("LL28k_w2_NC_T37_cs150_out_02.csv")
+seqs = dataset['fasta']
+pairs = pd.read_csv(r"indice_pairs_w_combos.csv")
+
+#Blast functions
+def validate_protein_sequence(sequence):
+    # Valid amino acid residues (single-letter codes)
+    valid_residues = set("ACDEFGHIKLMNPQRSTVWY")
+    return all(residue in valid_residues for residue in sequence)
+
+def run_blast_with_biopython(query_seq, subject_seq):
+    # Validate sequences
+    if not validate_protein_sequence(query_seq):
+        print("Invalid characters found in query sequence:", query_seq)
+        return
+    if not validate_protein_sequence(subject_seq):
+        print("Invalid characters found in subject sequence:", subject_seq)
+        return
+    
+    # Create temporary files for query and subject sequences
+    with open("query.fasta", "w") as query_file:
+        query_file.write(">query\n" + query_seq)
+    with open("subject.fasta", "w") as subject_file:
+        subject_file.write(">subject\n" + subject_seq)
+    
+    # Use the full path to the blastp.exe executable
+    blastp_cline = NcbiblastpCommandline(
+        cmd=r'C:\Users\lilia\Anaconda3\envs\ml135_env_sp21\lib\site-packages\biopython-1.84.dist-info\ncbi-blast-2.16.0+-x64-win64\ncbi-blast-2.16.0+\bin\blastp.exe',
+        query="query.fasta",
+        subject="subject.fasta",
+        outfmt=6,  # Output format 6 is a tabular format (can be changed if needed)
+        evalue=10  # Set e-value threshold higher to be more inclusive
+    )
+    
+    # Execute BLAST and capture the output
+    stdout, stderr = blastp_cline()
+    
+    # Print the BLAST output and interpret the columns
+    if stdout:
+       # print("BLAST Output:\n")
+        print("BLAST found a match")
+        
+        # print("Query ID, Subject ID, % Identity, Alignment Length, Mismatches, Gap Openings, Query Start, Query End, Subject Start, Subject End, E-value, Bit Score\n")
+        for line in stdout.strip().split("\n"):
+            columns = line.split("\t")
+        return float(columns[2]), int(columns[3])/len(query_seq)*100
+    else:
+        print("No significant matches found.")
+        return 0, 0
+    
+    if stderr:
+        print("BLAST Errors:\n", stderr)
+    
+    # Clean up temporary files
+    os.remove("query.fasta")
+    os.remove("subject.fasta")
+
+# Boolean area, 1 if identified as a pair by Blast, 0 if not
+blast_match = []
+
+for i in range(len(pairs)):
+    iden, cov = run_blast_with_biopython(seqs[pairs["index1"][i]], seqs[pairs["index2"][i]])
+    if iden > 80 and cov > 80:
+        blast_match.append(1)
+    else: blast_match.append(0)
+    print("\n")
+
+pairs["blast_match"] = blast_match
+pairs.to_csv("blasted_pairs.csv", index=False)
+
+# Create a graph of homolog pairs to identify single sequence to keep
+
+# designate x and y arrays (these are the indices of all pairs)
+x_list = list(pairs[pairs["blast_match"] == 1]["index1"])
+y_list = list(pairs[pairs["blast_match"] == 1]["index2"])
+
+# Initialize a graph
+G = nx.Graph()
+
+# Add nodes for each pair
+for i in range(len(x_list)):
+    G.add_node(i, x=x_list[i], y=y_list[i])
+
+# Add edges between nodes with the same x or y
+for i in range(len(x_list)):
+    for j in range(i + 1, len(x_list)):
+        if x_list[i] == x_list[j] or y_list[i] == y_list[j]:
+            G.add_edge(i, j)
+
+# Find connected components
+groups = list(nx.connected_components(G))
+
+# Extract and print the groups
+result = []
+for group in groups:
+    grouped_pairs = [(x_list[i], y_list[i]) for i in group]
+    result.append(grouped_pairs)
+
+# Get number of sequences in each group
+true_lengths = []
+for num in range(len(result)):
+    group = []
+    for i in result[num]:
+        index1 = i[0]
+        index2 = i[1]
+        if index1 not in group:
+            group.append(index1)
+        if index2 not in group:
+            group.append(index2)
+    true_lengths.append(len(group))
+
+print(np.mean(true_lengths))
+plt.hist(true_lengths, bins = 20)
+plt.show()
+
+def process_group(num):
+    group = []
+    lens = []
+    for i in result[num]:
+        index1 = i[0]
+        index2 = i[1]
+        if index1 not in group:
+            group.append(index1)
+            lens.append(len(seqs[index1]))
+        if index2 not in group:
+            group.append(index2)
+            lens.append(len(seqs[index2]))
+    #print(lens)
+    # for i in group:
+    #     print(seqs[i])
+
+    max_value = max(lens)
+    
+    # Find the index of the first occurrence of the maximum value
+    index_of_max = lens.index(max_value)
+    # print(index_of_max)
+    value_to_keep = group[index_of_max]
+    reject_values = [value for i, value in enumerate(group) if i != index_of_max]
+    # print(group)
+    # print(value_to_keep)
+    return(reject_values)
+
+all_reject_vals = []
+for i in range(len(result)):
+    all_reject_vals.extend(process_group(i))
+
+# Create an array to store whether a sequence is a reject due to homology
+reject_column_for_dataset = []
+for i in range(len(dataset)):
+    if i in all_reject_vals:
+        reject_column_for_dataset.append(1)
+    else: reject_column_for_dataset.append(0)
+
+dataset["reject"] = reject_list
+dataset.to_csv("LL28k_w_rejects.csv")
+df_filtered = dataset.dropna(subset=['w2'])   # Drop rows where 'w2' is NaN
+df_filtered = dataset[dataset['reject'] != 1]
+df_filtered.to_csv("LL28k_filtered_nanW2s_noRepeats.csv")

Tesei-trained_Model/filter_dataset/full_comp_dif.py

@@ -0,0 +1,68 @@
+import pandas as pd
+import numpy as np
+import math
+import matplotlib.pyplot as plt
+
+dataset = pd.read_csv("LL28k_w2_NC_T37_cs150_out_02.csv")
+seqs = dataset["fasta"]
+# constants
+amino_acid_data = {
+    "A": 0,
+    "R": 1,
+    "N": 0,
+    "D": -1,
+    "C": 0,
+    "E": -1,
+    "Q": 0,
+    "G": 0,
+    "H": 0,
+    "I": 0,
+    "L": 0,
+    "K": 1,
+    "M": 0,
+    "F": 0,
+    "P": 0,
+    "S": 0,
+    "T": 0,
+    "W": 0,
+    "Y": 0,
+    "V": 0,
+}
+
+aas = amino_acid_data.keys()
+
+column_names = [f'{i}_count' for i in aas]
+for name in column_names:
+    dataset[name] = 0
+
+def count_aa(seq, index):
+    for i in aas:
+        n = seq.count(i)
+        dataset.at[index, f'{i}_count'] = n
+
+for i in range(len(dataset)):
+    count_aa(dataset['fasta'][i], i)
+
+dataset.to_csv("LL28k_w2_NC_T37_cs150_out_02_composition.csv", index=False)
+
+def comp_two_seqs(i, j):
+    test_seq = dataset['fasta'][i]
+    test_seq_N = len(test_seq)
+    
+    comp_seq = dataset['fasta'][j]
+    comp_seq_N = len(comp_seq)
+    
+    total = 0
+    for aa in aas:
+        dif = ( dataset[f'{aa}_count'][i] / test_seq_N - dataset[f'{aa}_count'][j] / comp_seq_N )**2
+        total += dif
+    return total
+
+difs = np.zeros((len(dataset), len(dataset)))
+
+for i in range(len(seqs)):
+    for j in range(0, i):
+        
+        difs[i, j] = comp_two_seqs(i, j)
+
+np.save("full_comp_difs", difs)

Tesei-trained_Model/filter_dataset/identify_length_pairs.py

@@ -0,0 +1,27 @@
+import pandas as pd
+import numpy as np
+import math
+import time
+
+# load data
+dataset = pd.read_csv(r"C:\Users\lilia\Documents\DU\research\Computational_IDP_Research\ML\homologs\LL28k_w2_NC_T37_cs150_out_02.csv")
+seqs = dataset["fasta"]
+
+# compare lengths, save pairs of sequences with < 15 difference in length
+len_match1 = []
+len_match2 = []
+
+start_time = time.time()
+for i in range(len(seqs)):
+    for j in range(i):
+        if abs(len(seqs[i]) -  len(seqs[j])) < 15:
+            len_match1.append(i)
+            len_match2.append(j)
+    if (i % 200 == 0): 
+        print(f"Elapsed time: {(time.time() - start_time):.2f} seconds, index", i)
+end_time = time.time()
+elapsed_time = end_time - start_time
+print(f"Total time: {elapsed_time:.2f} seconds")
+
+#np.save("len_match_indices1.npy", np.asarray(len_match1))
+#np.save("len_match_indices2.npy", np.asarray(len_match2))

Tesei-trained_Model/filter_dataset/test_compo.py

@@ -0,0 +1,30 @@
+import pandas as pd
+import numpy as np
+import math
+import matplotlib.pyplot as plt
+import csv
+import time
+
+full_compos = np.load("full_comp_difs.npy")
+len_match1 = np.load("len_match1.npy")
+len_match2 = np.load("len_match2.npy")
+
+final_pairs1 = []
+final_pairs2 = []
+compos = []
+
+for i in range(len(len_match1)):
+    if full_compos[len_match1[i], len_match2[i]] < .005:
+        final_pairs1.append(len_match1[i])
+        final_pairs2.append(len_match2[i])
+        compos.append(full_compos[len_match1[i], len_match2[i]])
+
+df = pd.DataFrame({
+    'index1': final_pairs1,
+    'index2': final_pairs2,
+    'compo': compos
+})
+
+df.to_csv('indice_pairs_w_compo.csv', index=False)
+
+print(len(compos))