{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "47fa042d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n" ] } ], "source": [ "import requests\n", "import json\n", "import time\n", "import numpy as np\n", "import os\n", "from re import A, L\n", "import numpy as np\n", "import pandas as pd\n", "from tqdm import tqdm\n", "import torch\n", "import tensorflow as tf\n", "import tensorflow.keras.backend as K\n", "from tensorflow.keras import Model\n", "from tensorflow.keras.models import load_model\n", "from util import *\n", "from framepool import *\n", "import sys\n", "from Bio import SeqIO\n", "\n", "tf.compat.v1.enable_eager_execution()\n", "\n", "__file__ = os.getcwd()\n", "\n", "sys.path.append(os.path.dirname(os.path.dirname(__file__)))\n", "\n", "\n", "from models import Modules\n", "import configparser\n", "from sklearn.preprocessing import OneHotEncoder\n", "import logging\n", "import collections\n", "from models.ScheduleOptimizer import ScheduledOptim\n", "\n", "# ====================| some path |=======================\n", "global script_dir\n", "global data_dir\n", "global log_dir\n", "global pth_dir\n", "# global cell_lines\n", "\n", "global egfp_seq\n", "\n", "with open(os.path.join(__file__,\"machine_configure.json\"),'r') as f:\n", " config = json.load(f) \n", "\n", "script_dir = config['script_dir']\n", "data_dir = config['data_dir']\n", "log_dir = config['log_dir']\n", "pth_dir = config['pth_dir']\n", "\n", "\n", "\n", "# =====================| one hot encode |=======================\n", "\n", "class Seq_one_hot(object):\n", " def __init__(self,seq_type='nn',seq_len=100):\n", " \"\"\"\n", " initiate the sequence one hot encoder\n", " \"\"\"\n", " self.seq_len=seq_len\n", " self.seq_type =seq_type\n", " self.enable_encoder()\n", " \n", " def enable_encoder(self):\n", " if self.seq_type == 'nn':\n", " self.encoder = OneHotEncoder(sparse=False)\n", " self.encoder.drop_idx_ = None\n", " self.encoder.categories_ = [np.array(['A', 'C', 'G', 'T'], dtype='>>============== \\t \\t ')\n", " info(\" \\t\"+popen.vae_pth_path)\n", " \n", " return model\n", " \n", "def get_config_cuda(config_file):\n", " with open(config_file,'r') as f:\n", " lines = f.read_lines()\n", " for line in lines:\n", " if \"cuda_id =\" in line:\n", " device = line.split(\"=\")[1].strip()\n", " break\n", " device = int(device) if device.isdigit() else device\n", " return device\n", "\n", "def resume(popen,optimizer,logger):\n", " \"\"\"\n", " for a experiment, check whether it;s a new run, and create dir \n", " \"\"\"\n", " #run_name = model_stype + time.strftime(\"__%Y_%m_%d_%H:%M\"))\n", " \n", " if popen.Resumable:\n", " \n", " checkpoint = torch.load(popen.vae_pth_path, map_location=torch.device('cpu')) # xx-model-best.pth\n", " previous_epoch = checkpoint['epoch']\n", " previous_loss = checkpoint['validation_loss']\n", " previous_acc = checkpoint['validation_acc']\n", " \n", " \n", " # very important\n", " if (type(optimizer) == ScheduledOptim):\n", " optimizer.n_current_steps = popen.n_current_steps\n", " optimizer.delta = popen.delta\n", " \n", " logger.info(\" \\t \\t ========================================================= \\t \\t \")\n", " logger.info(' \\t \\t ==============<<< Resume from checkpoint>>>============== \\t \\t \\n')\n", " logger.info(\" \\t\"+popen.vae_pth_path+'\\n')\n", " logger.info(\" \\t \\t ========================================================= \\t \\t \\n\")\n", " \n", " return previous_epoch,previous_loss,previous_acc\n", " \n", " \n", "egfp_seq = \"atgggcgaattaagtaagggcgaggagctgttcaccggggtggtgcccatcctggtcgagctggacggcgacgtaaacggccacaagttcagcgtgtccggcgagggcgagggcgatgccacctacggcaagctgaccctgaagttcatctgcaccaccggcaagctgcccgtgccctggcccaccctcgtgaccaccctgacctacggcgtgcagtgcttcagccgctaccccgaccacatgaagcagcacgacttcttcaagtccgccatgcccgaaggctacgtccaggagcgcaccatcttct\"\n", "eGFP_seq = egfp_seq.upper()\n", "\n", "class Auto_popen(object):\n", " def __init__(self,config_file):\n", " \"\"\"\n", " read the config_fiel\n", " \"\"\"\n", " # machine config path\n", " self.shuffle = True\n", " self.script_dir = script_dir\n", " self.data_dir = data_dir\n", " self.data_dir = '/mnt/sina/run/ml/gan/motif/MTtrans/test.csv'\n", " self.log_dir = log_dir\n", " self.pth_dir = pth_dir\n", " self.set_attr_as_none(['te_net_l2','loss_fn','modual_to_fix','other_input_columns','pretrain_pth','kfold_index'])\n", " self.split_like = False\n", " self.loss_schema = 'constant'\n", " \n", " # transform to dict and convert to specific data type\n", " self.config = configparser.ConfigParser()\n", " self.config.read(config_file)\n", " self.config_file = config_file\n", " self.config_dict = {item[0]: eval(item[1]) for item in self.config.items('DEFAULT')}\n", " \n", " # assign some attr from config_dict \n", " self.set_attr_from_dict(self.config_dict.keys())\n", " self.check_run_and_setting_name() # check run name\n", " self._dataset = \"_\" + self.dataset if self.dataset != '' else self.dataset\n", " # the saving direction\n", " self.path_category = self.config_file.split('/')[-4]\n", " self.vae_log_path = config_file.replace('.ini','.log')\n", " \n", "\n", " self.Resumable = False\n", "\n", " # covariates for other input\n", " self.n_covar = len(self.other_input_columns) if self.other_input_columns is not None else 0\n", " \n", " # generate self.model_args\n", " self.get_model_config()\n", "\n", " @property\n", " def vae_pth_path(self):\n", " save_to = os.path.join(self.pth_dir,self.model_type+self._dataset,self.setting_name)\n", " if self.kfold_index is None:\n", " pth = os.path.join(save_to, self.run_name + '-model_best.pth')\n", " elif type(self.kfold_index) == int:\n", " k = self.kfold_index\n", " pth = os.path.join(save_to, self.run_name + f'-model_best_cv{k}.pth')\n", " return pth\n", " \n", " @vae_pth_path.setter\n", " def vae_pth_path(self, path):\n", " self._vae_pth_path = path\n", " \n", " def set_attr_from_dict(self,attr_ls):\n", " for attr in attr_ls:\n", " self.__setattr__(attr,self.config_dict[attr])\n", " \n", " def set_attr_as_none(self,attr_ls):\n", " for attr in attr_ls:\n", " self.__setattr__(attr,None)\n", "\n", " def check_run_and_setting_name(self):\n", " file_name = self.config_file.split(\"/\")[-1]\n", " dir_name = self.config_file.split(\"/\")[-2]\n", " self.setting_name = dir_name\n", " assert self.run_name == file_name.split(\".\")[0]\n", " # assert self.run_name == file_name.split(\".\")[0]\")[0]\n", " \n", " def get_model_config(self):\n", " \"\"\"\n", " assert we type in the correct model type and group them into model_args\n", " \"\"\"\n", " self.model_type = 'RL_hard_share'\n", " if self.model_type in dir(Modules):\n", " self.Model_Class = eval(\"Modules.{}\".format(self.model_type))\n", " else:\n", " raise NameError(\"not such model type\")\n", " \n", " # conv_args define the soft-sharing part\n", " conv_args = [\"channel_ls\",\"kernel_size\",\"stride\",\"padding_ls\",\"diliation_ls\",\"pad_to\"]\n", " self.conv_args = tuple([self.__getattribute__(arg) for arg in conv_args])\n", " \n", " # left args dfine the tower part in which the arguments are different among tasks\n", " left_args={# Backbone models\n", " 'RL_regressor':[\"tower_width\",\"dropout_rate\"],\n", " 'RL_clf':[\"n_class\",\"tower_width\",\"dropout_rate\"],\n", " 'RL_gru':[\"tower_width\",\"dropout_rate\"],\n", " 'RL_FACS': [\"tower_width\",\"dropout_rate\"],\n", " 'RL_hard_share':[\"tower_width\",\"dropout_rate\", \"activation\",\"cycle_set\" ],\n", " 'RL_covar_reg':[\"tower_width\",\"dropout_rate\", \"activation\", \"n_covar\", \"cycle_set\" ],\n", " 'RL_covar_intercept':[\"tower_width\",\"dropout_rate\", \"activation\", \"n_covar\", \"cycle_set\" ],\n", " 'RL_mish_gru':[\"tower_width\",\"dropout_rate\"],\n", " # GP models\n", " 'GP_net': ['tower_width', 'dropout_rate', 'global_pooling', 'activation', 'cycle_set'],\n", " 'Frame_GP': ['tower_width', 'dropout_rate', 'activation', 'cycle_set'],\n", " 'RL_Atten': ['qk_dim', 'n_head', 'n_atten_layer', 'tower_width', 'dropout_rate', 'activation', 'cycle_set'],\n", " # Koo net\n", " 'Conf_CNN' : ['pool_size'],\n", " }[self.model_type]\n", " \n", " self.model_args = [self.conv_args] + [self.__getattribute__(arg) for arg in left_args]\n", " \n", " \n", " def check_experiment(self,logger):\n", " \"\"\"\n", " check any unfinished experiment ?\n", " \"\"\"\n", " log_save_dir = os.path.dirname(self.vae_log_path)\n", " pth_save_dir = os.path.join(self.pth_dir,self.model_type+self._dataset,self.setting_name)\n", " # make dirs \n", " if not os.path.exists(log_save_dir):\n", " os.makedirs(log_save_dir)\n", " if not os.path.exists(pth_save_dir):\n", " os.makedirs(pth_save_dir)\n", " \n", " # check resume\n", " if os.path.exists(self.vae_log_path) & os.path.exists(self.vae_pth_path):\n", " self.Resumable = True\n", " logger.info(' \\t \\t ==============<<< Experiment detected >>>============== \\t \\t \\n')\n", " \n", " def update_ini_file(self,E,logger):\n", " \"\"\"\n", " E is the dict contain the things to update\n", " \"\"\"\n", " # update the ini file\n", " self.config_dict.update(E)\n", " strconfig = {K: repr(V) for K,V in self.config_dict.items()}\n", " self.config['DEFAULT'] = strconfig\n", " \n", " with open(self.config_file,'w') as f:\n", " self.config.write(f)\n", " \n", " logger.info(' ini file updated ')\n", " \n", " def chimera_weight_update(self):\n", " # TODO : progressively update the loss weight between tasks\n", " \n", " # TODO : 1. scale the loss into the same magnitude\n", " \n", " # TODO : 2. update the weight by their own learning progress\n", " \n", " return None\n", "\n", "abs_path = './../mrl_te_optimization/log/Backbone/RL_hard_share/3M/small_repective_filed_strides1113.ini'\n", "Configuration = Auto_popen(abs_path)\n", "\n", "np.random.seed(25)\n", "\n", "BATCH_SIZE = 100\n", "N_GENES = 8\n", "LR = 0.001\n", "GPU = '0'\n", "STEPS = 10\n", "\n", "if GPU == '-1':\n", " device = 'cpu'\n", "else:\n", " os.environ['CUDA_VISIBLE_DEVICES'] = GPU\n", " device = 'cuda'\n", "\n", "SEQ_BATCH = N_GENES\n", "UTR_LEN = 128\n", "DIM = 40\n", "gpath = './../../models/checkpoint_3000.h5'\n", "mrl_path = './../../models/utr_model_combined_residual_new.h5'\n", "exp_path = './../../models/humanMedian_trainepoch.11-0.426.h5'\n", "tpath = './../exp_optimization/script/checkpoint/RL_hard_share_MTL/3R/schedule_MTL-model_best_cv1.pth'\n", "LR = np.exp(-int(LR))\n", "\n", "gene_names = [\"MYOC\", \"TIGD4\", \"ATP6V1B2\", \"TAGLN\", \"COX7A2L\", \"IFNGR2\", \"TNFRSF21\", \"SETD6\"]\n", "\n", "target_genes = [\"ANTXR2\", \"NFIL3\", \"UNC13D\", \"DHRS2\", \"RPS13\", \"HBD\", \"METAP1D\", \"NCALD\"]\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3a95c1ae", "metadata": {}, "outputs": [], "source": [ "\n", "\n", "def reverse_complement(sequence):\n", " \"\"\"Compute the reverse complement of a DNA sequence.\"\"\"\n", " complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', \n", " 'a': 't', 't': 'a', 'c': 'g', 'g': 'c', 'N': 'N', 'n': 'N'}\n", " return ''.join(complement.get(base, 'N') for base in reversed(sequence))\n", "\n", "class GeneInfoRetriever:\n", " def __init__(self):\n", " self.base_url = \"https://rest.ensembl.org\"\n", " self.headers = {\"Content-Type\": \"application/json\"}\n", " self.sleep_time = 0.5 # Respect Ensembl API rate limits\n", "\n", " def _make_request(self, endpoint):\n", " \"\"\"Make a request to the Ensembl REST API.\"\"\"\n", " url = self.base_url + endpoint\n", " try:\n", " response = requests.get(url, headers=self.headers)\n", " time.sleep(self.sleep_time)\n", " if response.status_code == 200:\n", " return response.json()\n", " else:\n", " print(f\"Error: {response.status_code} - {response.text}\")\n", " return None\n", " except Exception as e:\n", " print(f\"Request error: {e}\")\n", " return None\n", "\n", " def get_gene_id(self, gene_symbol, species=\"homo_sapiens\"):\n", " \"\"\"Retrieve the Ensembl gene ID for a gene symbol.\"\"\"\n", " endpoint = f\"/lookup/symbol/{species}/{gene_symbol}\"\n", " response = self._make_request(endpoint)\n", " return response.get(\"id\") if response else None\n", "\n", " def get_gene_coordinates(self, gene_id):\n", " \"\"\"Retrieve genomic coordinates for a gene ID.\"\"\"\n", " endpoint = f\"/lookup/id/{gene_id}?expand=1\"\n", " response = self._make_request(endpoint)\n", " if response:\n", " return {\n", " \"chromosome\": response.get(\"seq_region_name\"),\n", " \"start\": response.get(\"start\"),\n", " \"end\": response.get(\"end\"),\n", " \"strand\": response.get(\"strand\")\n", " }\n", " return None\n", "\n", " def get_tss_and_utr(self, gene_id):\n", " \"\"\"Retrieve TSS and 5' UTR coordinates for the canonical transcript.\"\"\"\n", " endpoint = f\"/lookup/id/{gene_id}?expand=1&utr=1\"\n", " response = self._make_request(endpoint)\n", " if not response or \"Transcript\" not in response:\n", " return None\n", "\n", " # Find canonical transcript\n", " canonical_transcript = None\n", " for transcript in response[\"Transcript\"]:\n", " if transcript.get(\"is_canonical\", 0) == 1:\n", " canonical_transcript = transcript\n", " break\n", " if not canonical_transcript:\n", " for transcript in response[\"Transcript\"]:\n", " if transcript.get(\"biotype\") == \"protein_coding\":\n", " canonical_transcript = transcript\n", " break\n", " if not canonical_transcript:\n", " canonical_transcript = response[\"Transcript\"][0] if response[\"Transcript\"] else None\n", "\n", " if not canonical_transcript:\n", " return None\n", "\n", " # Determine TSS and 5' UTR\n", " strand = canonical_transcript.get(\"strand\")\n", " tss = canonical_transcript[\"start\"] if strand == 1 else canonical_transcript[\"end\"]\n", " five_prime_utr = None\n", "\n", " if \"UTR\" in canonical_transcript:\n", " for utr in canonical_transcript[\"UTR\"]:\n", " if utr.get(\"object_type\") == \"five_prime_UTR\":\n", " five_prime_utr = {\n", " \"start\": utr.get(\"start\"),\n", " \"end\": utr.get(\"end\")\n", " }\n", " break\n", "\n", " # Verify TSS matches 5' UTR start\n", " if five_prime_utr:\n", " expected_tss = five_prime_utr[\"start\"] if strand == 1 else five_prime_utr[\"end\"]\n", " if expected_tss != tss:\n", " print(f\"Warning: Adjusting TSS from {tss} to match 5' UTR {'start' if strand == 1 else 'end'} ({expected_tss})\")\n", " tss = expected_tss\n", "\n", " return {\n", " \"tss\": tss,\n", " \"strand\": strand,\n", " \"chromosome\": canonical_transcript.get(\"seq_region_name\"),\n", " \"five_prime_utr\": five_prime_utr,\n", " \"transcript_id\": canonical_transcript.get(\"id\")\n", " }\n", "\n", " def get_promoter_sequence(self, gene_id, upstream=8000, downstream=4000):\n", " \"\"\"Retrieve sequence around TSS (8kb upstream, 4kb downstream).\"\"\"\n", " tss_info = self.get_tss_and_utr(gene_id)\n", " if not tss_info:\n", " return None, None\n", "\n", " chromosome = tss_info[\"chromosome\"]\n", " strand = tss_info[\"strand\"]\n", " tss_position = tss_info[\"tss\"]\n", "\n", " # Calculate region based on strand\n", " if strand == 1:\n", " seq_start = tss_position - upstream\n", " seq_end = tss_position + downstream - 1\n", " else:\n", " seq_start = tss_position - downstream\n", " seq_end = tss_position + upstream - 1\n", "\n", " seq_start = max(1, seq_start)\n", "\n", " # Store sequence coordinates\n", " sequence_coords = {\n", " \"chromosome\": chromosome,\n", " \"start\": seq_start,\n", " \"end\": seq_end,\n", " \"strand\": 1 if strand == 1 else -1\n", " }\n", "\n", " # Validate 5' UTR inclusion\n", " if tss_info[\"five_prime_utr\"]:\n", " utr_start = tss_info[\"five_prime_utr\"][\"start\"]\n", " utr_end = tss_info[\"five_prime_utr\"][\"end\"]\n", " if not (seq_start <= utr_start <= seq_end and seq_start <= utr_end <= seq_end):\n", " print(f\"Warning: 5' UTR ({utr_start}-{utr_end}) not fully within sequence ({seq_start}-{seq_end})\")\n", "\n", " # Get sequence\n", " strand_str = \"1\" if strand == 1 else \"-1\"\n", " endpoint = f\"/sequence/region/human/{chromosome}:{seq_start}..{seq_end}:{strand_str}\"\n", " response = self._make_request(endpoint)\n", " return response.get(\"seq\") if response else None, sequence_coords\n", "\n", " def get_gene_info(self, gene_symbol, species=\"homo_sapiens\", output_json=\"gene_info.json\"):\n", " \n", " if not os.path.exists(os.path.join('./.cache/',f\"{gene_symbol}_info.json\")):\n", "\n", " \"\"\"Retrieve and save promoter sequence, TSS, 5' UTR, and coordinates.\"\"\"\n", " # Get gene ID\n", " gene_id = self.get_gene_id(gene_symbol, species)\n", " if not gene_id:\n", " return {\"error\": f\"Gene {gene_symbol} not found\"}\n", "\n", " # Get TSS and 5' UTR\n", " tss_info = self.get_tss_and_utr(gene_id)\n", " if not tss_info:\n", " return {\"error\": \"Could not retrieve TSS or transcript information\"}\n", "\n", " # Get promoter sequence and coordinates\n", " promoter_sequence, sequence_coords = self.get_promoter_sequence(gene_id)\n", " if not promoter_sequence:\n", " return {\"error\": \"Could not retrieve promoter sequence\"}\n", "\n", " # Compile gene information\n", " gene_info = {\n", " \"gene_symbol\": gene_symbol,\n", " \"gene_id\": gene_id,\n", " \"promoter_sequence\": promoter_sequence,\n", " \"sequence_length\": len(promoter_sequence),\n", " \"sequence_coordinates\": sequence_coords,\n", " \"tss\": {\n", " \"chromosome\": tss_info[\"chromosome\"],\n", " \"position\": tss_info[\"tss\"],\n", " \"strand\": \"+\" if tss_info[\"strand\"] == 1 else \"-\"\n", " },\n", " \"five_prime_utr\": tss_info[\"five_prime_utr\"],\n", " \"transcript_id\": tss_info[\"transcript_id\"]\n", " }\n", "\n", " # Save to JSON\n", " try:\n", " os.makedirs(os.path.dirname('./.cache/'), exist_ok=True)\n", " with open(os.path.join('./.cache/',f\"{gene_symbol}_info.json\"), \"w\") as f:\n", " json.dump(gene_info, f, indent=2)\n", " print(f\"Saved gene information to {output_json}\")\n", " except Exception as e:\n", " print(f\"Error saving JSON: {e}\")\n", "\n", " else:\n", "\n", " with open(os.path.join('./.cache/',f\"{gene_symbol}_info.json\"), \"r\") as f:\n", " gene_info = json.load(f)\n", "\n", " return gene_info\n", "\n", " def reverse_complement(self, sequence):\n", " \"\"\"Compute the reverse complement of a DNA sequence.\"\"\"\n", " complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', \n", " 'a': 't', 't': 'a', 'c': 'g', 'g': 'c', 'N': 'N', 'n': 'N'}\n", " return ''.join(complement.get(base, 'N') for base in reversed(sequence))\n", "\n", " def replace_utr_in_sequence(self, gene_info_file, generated_utrs, target_length=10500, output_prefix=\"modified_sequence\", write_json=False, verbose=False):\n", " \"\"\"\n", " Replace original 5' UTR with generated UTRs, ensuring 10,500nt output.\n", " \n", " Parameters:\n", " gene_info_file (str): Path to JSON file with gene information\n", " generated_utrs (list): List of generated 5' UTR sequences (64-128nt)\n", " target_length (int): Desired output sequence length (default: 10500)\n", " output_prefix (str): Prefix for output JSON files\n", " \n", " Returns:\n", " list: List of modified sequences with metadata\n", " \"\"\"\n", " try:\n", " # Read gene information\n", " with open(gene_info_file, \"r\") as f:\n", " gene_info = json.load(f)\n", "\n", " original_sequence = gene_info[\"promoter_sequence\"]\n", " strand = gene_info[\"tss\"][\"strand\"]\n", " tss_position = gene_info[\"tss\"][\"position\"]\n", " sequence_coords = gene_info[\"sequence_coordinates\"]\n", " seq_start = sequence_coords[\"start\"]\n", " seq_end = sequence_coords[\"end\"]\n", " five_prime_utr = gene_info[\"five_prime_utr\"]\n", " gene_symbol = gene_info[\"gene_symbol\"]\n", " transcript_id = gene_info[\"transcript_id\"]\n", "\n", " if not five_prime_utr:\n", " print(f\"Error: No 5' UTR information available for {gene_symbol}\")\n", " return []\n", "\n", " # Calculate original 5' UTR position in sequence\n", " if strand == \"+\":\n", " utr_start_genomic = five_prime_utr[\"start\"]\n", " utr_end_genomic = five_prime_utr[\"end\"]\n", " utr_start_seq = utr_start_genomic - seq_start\n", " utr_end_seq = utr_end_genomic - seq_start\n", " else:\n", " utr_start_genomic = five_prime_utr[\"end\"] # TSS\n", " utr_end_genomic = five_prime_utr[\"start\"]\n", " utr_start_seq = seq_end - utr_start_genomic\n", " utr_end_seq = seq_end - utr_end_genomic\n", "\n", " # Validate UTR positions\n", " seq_length = len(original_sequence)\n", " if not (0 <= utr_start_seq <= seq_length and 0 <= utr_end_seq <= seq_length):\n", " print(f\"Error: 5' UTR coordinates (seq indices {utr_start_seq}-{utr_end_seq}) out of sequence bounds (0-{seq_length}) for {gene_symbol}\")\n", " return []\n", "\n", " original_utr_length = abs(utr_end_genomic - utr_start_genomic) + 1\n", " if verbose:\n", " print(f\"Original 5' UTR length for {gene_symbol}: {original_utr_length} nt\")\n", "\n", " modified_sequences = []\n", " for i, new_utr in enumerate(generated_utrs):\n", " new_utr_length = len(new_utr)\n", " if not 64 <= new_utr_length <= 128:\n", " if verbose:\n", " print(f\"Warning: Generated UTR {i+1} length ({new_utr_length}) outside 64-128nt range for {gene_symbol}\")\n", " continue\n", "\n", " # Construct new sequence\n", " if strand == \"+\":\n", " new_sequence = (\n", " original_sequence[:utr_start_seq] +\n", " new_utr +\n", " original_sequence[utr_end_seq + 1:]\n", " )\n", " new_utr_start_genomic = utr_start_genomic\n", " new_utr_end_genomic = utr_start_genomic + new_utr_length - 1\n", " if len(new_sequence) > target_length:\n", " new_sequence = new_sequence[:target_length]\n", " sequence_coords[\"end\"] = seq_start + target_length - 1\n", " elif len(new_sequence) < target_length:\n", " if verbose:\n", " print(f\"Error: Sequence too short ({len(new_sequence)} nt) after UTR replacement for {gene_symbol}\")\n", " continue\n", " else:\n", " new_utr_rc = reverse_complement(new_utr)\n", " new_sequence = (\n", " original_sequence[:min(utr_start_seq, utr_end_seq)] +\n", " new_utr_rc +\n", " original_sequence[max(utr_start_seq, utr_end_seq) + 1:]\n", " )\n", " new_utr_start_genomic = utr_start_genomic\n", " new_utr_end_genomic = utr_start_genomic - new_utr_length + 1\n", " if len(new_sequence) > target_length:\n", " trim_amount = len(new_sequence) - target_length\n", " new_sequence = new_sequence[trim_amount:]\n", " sequence_coords[\"start\"] = seq_start + trim_amount\n", " elif len(new_sequence) < target_length:\n", " if verbose:\n", " print(f\"Error: Sequence too short ({len(new_sequence)} nt) after UTR replacement for {gene_symbol}\")\n", " continue\n", "\n", " # Store modified sequence and metadata\n", " modified_info = {\n", " \"gene_symbol\": gene_symbol,\n", " \"transcript_id\": transcript_id,\n", " \"modified_sequence\": new_sequence,\n", " \"sequence_length\": len(new_sequence),\n", " \"sequence_coordinates\": sequence_coords.copy(),\n", " \"tss\": gene_info[\"tss\"],\n", " \"five_prime_utr\": {\n", " \"start\": new_utr_start_genomic,\n", " \"end\": new_utr_end_genomic,\n", " \"sequence\": new_utr if strand == \"+\" else new_utr_rc\n", " },\n", " \"original_utr_length\": original_utr_length,\n", " \"new_utr_length\": new_utr_length,\n", " \"utr_index\": i + 1\n", " }\n", "\n", " # Save to JSON\n", " if write_json:\n", " output_file = f\"{output_prefix}_{gene_symbol}_utr_{i+1}.json\"\n", " try:\n", " os.makedirs(os.path.dirname(output_file), exist_ok=True)\n", " with open(output_file, \"w\") as f:\n", " json.dump(modified_info, f, indent=2)\n", " print(f\"Saved modified sequence {i+1} for {gene_symbol} to {output_file}\")\n", " except Exception as e:\n", " print(f\"Error saving modified sequence {i+1} for {gene_symbol}: {e}\")\n", "\n", " modified_sequences.append(modified_info[\"modified_sequence\"])\n", "\n", " return modified_sequences\n", "\n", " except Exception as e:\n", " print(f\"Error processing UTR replacement for {gene_info.get('gene_symbol', 'unknown')}: {e}\")\n", " return []\n", "\n", "\n", " def replace_utr_in_multiple_sequences(self, gene_symbols, generated_utrs, target_length=10500, cache_dir=\"./.cache\", output_prefix=\"modified_sequence\", verbose=False):\n", " \"\"\"\n", " Replace 5' UTRs for multiple genes with generated UTRs.\n", " \n", " Parameters:\n", " gene_symbols (list): List of gene names\n", " generated_utrs (list): List of generated 5' UTR sequences (64-128nt)\n", " target_length (int): Desired output sequence length (default: 10500)\n", " cache_dir (str): Directory containing cached gene info JSON files\n", " output_prefix (str): Prefix for output JSON files\n", " \n", " Returns:\n", " list: List of n_utrs * n_genes modified sequences with metadata\n", " \"\"\"\n", " all_modified_sequences = []\n", " n_utrs = len(generated_utrs)\n", " n_genes = len(gene_symbols)\n", "\n", " for gene_symbol in gene_symbols:\n", " json_file = os.path.join(cache_dir, f\"{gene_symbol}_info.json\")\n", " if not os.path.exists(json_file):\n", " print(f\"Error: Gene info file {json_file} not found\")\n", " continue\n", " \n", " if verbose:\n", " print(f\"\\nProcessing gene: {gene_symbol}\")\n", " modified_sequences = self.replace_utr_in_sequence(\n", " gene_info_file=json_file,\n", " generated_utrs=generated_utrs,\n", " target_length=target_length,\n", " output_prefix=os.path.join(cache_dir, output_prefix)\n", " )\n", "\n", " if modified_sequences:\n", " all_modified_sequences.extend(modified_sequences)\n", " else:\n", " if verbose:\n", " print(f\"No modified sequences generated for {gene_symbol}\")\n", "\n", " expected_count = n_utrs * n_genes\n", "\n", " if verbose:\n", " print(f\"\\nGenerated {n_utrs * n_genes} modified sequences (expected: {expected_count})\")\n", "\n", " return all_modified_sequences\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "def convert_model(model_:Model):\n", "\n", " input_ = tf.keras.layers.Input(shape=( 10500, 4))\n", " input = input_\n", " for i in range(len(model_.layers)-1):\n", "\n", " \n", " if isinstance(model_.layers[i+1],tf.keras.layers.Concatenate):\n", " paddings = tf.constant([[0,0],[0,6]])\n", " output = tf.pad(input, paddings, 'CONSTANT')\n", " input = output\n", " else:\n", " if not isinstance(model_.layers[i+1],tf.keras.layers.InputLayer):\n", " output = model_.layers[i+1](input)\n", " input = output\n", "\n", " if isinstance(model_.layers[i+1],tf.keras.layers.Conv1D):\n", " pass\n", "\n", " model = tf.keras.Model(inputs=input_, outputs=output)\n", " model.compile(loss=\"mse\", optimizer=\"adam\")\n", " return model\n", "\n", "def one_hot(seq):\n", " convert = True\n", " if isinstance(seq, tf.Tensor):\n", " seq = seq.numpy().astype(str)\n", " convert = True\n", "\n", " num_seqs = len(seq)\n", " seq_len = len(seq[0])\n", " seqindex = {'A':0, 'C':1, 'G':2, 'T':3, 'a':0, 'c':1, 'g':2, 't':3}\n", " seq_vec = np.zeros((num_seqs,seq_len,4), dtype='bool')\n", " for i in range(num_seqs):\n", " thisseq = seq[i]\n", " for j in range(seq_len):\n", " try:\n", " seq_vec[i,j,seqindex[thisseq[j]]] = 1\n", " except:\n", " pass\n", " \n", " if convert:\n", " seq_vec = tf.convert_to_tensor(seq_vec,dtype=tf.float32)\n", "\n", "\n", " return seq_vec\n", "\n", "\n", "def select_best(scores, seqs, gc_control=False, GC=-1, per_gene=False):\n", " selected_scores = []\n", " selected_seqs = []\n", " if per_gene: \n", "\n", " scores = np.asarray(scores)\n", " seqs = np.asarray(seqs)\n", " \n", " A, B, C = np.shape(scores)\n", " selected_scores = []\n", " selected_seqs = []\n", " \n", " for b in range(B):\n", "\n", " best_score = np.max(scores[0, b, :]) \n", " best_seq = seqs[0, :] \n", " \n", " for a in range(1, A):\n", " current_score = np.max(scores[a, b, :]) \n", " \n", " if current_score > best_score:\n", " if gc_control:\n", "\n", " gc_content = get_gc_content(seqs[a, :])\n", " if gc_content < GC:\n", " best_score = current_score\n", " best_seq = seqs[a, :]\n", " best_a = a\n", " else:\n", " best_score = current_score\n", " best_seq = seqs[a, :]\n", " best_a = a\n", " \n", " selected_scores.append(best_score)\n", " selected_seqs.append(best_seq)\n", " \n", "\n", " selected_scores = np.array(selected_scores) \n", " selected_seqs = np.array(selected_seqs) \n", " else:\n", " for i in range(len(scores[0])):\n", " best = scores[1][i]\n", " best_seq = seqs[1][i]\n", " for j in range(len(scores)-1):\n", " if scores[j+1][i] > best:\n", " if gc_control:\n", " if get_gc_content(seqs[j][i]) < GC:\n", " best = scores[j+1][i]\n", " best_seq = seqs[j+1][i]\n", " else:\n", " best = scores[j+1][i]\n", " best_seq = seqs[j+1][i]\n", "\n", " selected_scores.append(best)\n", " selected_seqs.append(best_seq)\n", "\n", " return selected_seqs, selected_scores" ] }, { "cell_type": "code", "execution_count": 7, "id": "9dcefb89", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n", "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [00:42<00:00, 4.27s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "(8,)\n", "\n", "Evaluation of Optimization on Original Genes (Log TPM):\n", "\n", "Expression Levels (Log TPM):\n", " Average Initial Log TPM: -0.1834 (TPM: 0.6555)\n", " Average Optimized Log TPM: -0.0183 (TPM: 0.9587)\n", " Log TPM Difference: 0.1651\n", " TPM Improvement: 0.3032 (+46.25% (increase))\n", "Genes:\n", "['MYOC', 'TIGD4', 'ATP6V1B2', 'TAGLN', 'COX7A2L', 'IFNGR2', 'TNFRSF21', 'SETD6']\n", "Average Initial Expression: -0.1834263348579407\n", "Best Expression: -0.018321754410862923\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "Error processing UTR replacement for DHRS2: ufunc 'add' did not contain a loop with signature matching types (dtype(' None\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "Error processing UTR replacement for METAP1D: ufunc 'add' did not contain a loop with signature matching types (dtype(' None\n", "/Users/sbarazan/Documents/bilkent/utrgan/bioinformatics advances/code/UTRGAN/src/exp_optimization\n", "\n", "Evaluation of Optimization on Target Genes (Log TPM):\n", "Original Genes: ['MYOC', 'TIGD4', 'ATP6V1B2', 'TAGLN', 'COX7A2L', 'IFNGR2', 'TNFRSF21', 'SETD6']\n", "Target Genes: ['ANTXR2', 'NFIL3', 'UNC13D', 'DHRS2', 'RPS13', 'HBD', 'METAP1D', 'NCALD']\n", "\n", "Expression Levels (Log TPM):\n", " Average Initial Log TPM: -0.5353 (TPM: 0.2915)\n", " Average Optimized Log TPM: -0.4488 (TPM: 0.3558)\n", " Log TPM Difference: 0.0865\n", " TPM Improvement: 0.0643 (+22.05% (increase))\n", "\n", "Per-Gene Expression Levels (Log TPM):\n", " ANTXR2: Initial Log TPM = -0.1230 (TPM: 0.7534), Optimized Log TPM = -0.5100 (TPM: 0.3091), TPM Improvement = -0.4443 (-58.98% (decrease))\n", " NFIL3: Initial Log TPM = 0.7915 (TPM: 6.1874), Optimized Log TPM = 0.3522 (TPM: 2.2503), TPM Improvement = -3.9371 (-63.63% (decrease))\n", " UNC13D: Initial Log TPM = -0.4673 (TPM: 0.3409), Optimized Log TPM = 0.3383 (TPM: 2.1793), TPM Improvement = 1.8384 (+539.22% (increase))\n", " DHRS2: Initial Log TPM = -1.1142 (TPM: 0.0769), Optimized Log TPM = -0.5517 (TPM: 0.2807), TPM Improvement = 0.2038 (+265.13% (increase))\n", " RPS13: Initial Log TPM = 0.1343 (TPM: 1.3622), Optimized Log TPM = 0.0449 (TPM: 1.1090), TPM Improvement = -0.2533 (-18.59% (decrease))\n", " HBD: Initial Log TPM = -1.3306 (TPM: 0.0467), Optimized Log TPM = -0.8894 (TPM: 0.1290), TPM Improvement = 0.0823 (+176.14% (increase))\n", " METAP1D: Initial Log TPM = -1.1724 (TPM: 0.0672), Optimized Log TPM = -1.2720 (TPM: 0.0535), TPM Improvement = -0.0138 (-20.50% (decrease))\n", " NCALD: Initial Log TPM = -1.0010 (TPM: 0.0998), Optimized Log TPM = -1.1027 (TPM: 0.0789), TPM Improvement = -0.0208 (-20.88% (decrease))\n" ] } ], "source": [ "\n", "model = tf.keras.models.load_model(exp_path)\n", "\n", "model = convert_model(model)\n", "\n", "wgan = tf.keras.models.load_model(gpath)\n", "\n", "\"\"\"\n", "Data:\n", "\"\"\"\n", "\n", "\n", "noise = tf.Variable(tf.random.normal(shape=[BATCH_SIZE,40]))\n", "\n", "tf.random.set_seed(25)\n", "\n", "diffs = []\n", "init_exps = []\n", "\n", "opt_exps = []\n", "\n", "orig_vals = []\n", "\n", "noise = tf.Variable(tf.random.normal(shape=[BATCH_SIZE,40]))\n", "noise_small = tf.random.normal(shape=[BATCH_SIZE,40],stddev=1e-5)\n", "\n", "optimizer = tf.keras.optimizers.Adam(learning_rate=LR)\n", "\n", "'''\n", "Optimization takes place here.\n", "'''\n", "\n", "bind_scores_list = []\n", "bind_scores_means = []\n", "sequences_list = []\n", "\n", "means = []\n", "maxes = []\n", "\n", "iters_ = []\n", "\n", "OPTIMIZE = True\n", "\n", "DNA_SEL = False\n", "\n", "retriever = GeneInfoRetriever()\n", "refs = []\n", "for i in range(len(gene_names)):\n", " output_json = f\"{gene_names[i]}_info.json\"\n", "\n", " if not os.path.exists(os.path.join('./.cache/',output_json)):\n", "\n", " # Retrieve gene information\n", " gene_info = retriever.get_gene_info(gene_names[i], output_json=output_json)\n", "\n", " if \"error\" in gene_info:\n", " print(f\"Error: {gene_info['error']}\")\n", " else:\n", " refs.append(gene_info[\"promoter_sequence\"]) \n", " else:\n", " with open(os.path.join('./.cache/',output_json), \"r\") as f:\n", " gene_info = json.load(f)\n", " refs.append(gene_info[\"promoter_sequence\"])\n", "\n", "sequences_init = wgan(noise)\n", "\n", "gen_seqs_init = sequences_init.numpy().astype('float')\n", "\n", "seqs_gen_init = recover_seq(gen_seqs_init, rev_rna_vocab)\n", "\n", "seqs_init = retriever.replace_utr_in_multiple_sequences(gene_names, seqs_gen_init, target_length=10500, cache_dir=\"./.cache\", output_prefix=\"modified_sequence\")\n", "\n", "seqs_init = one_hot(seqs_init)\n", "\n", "pred_init = model(seqs_init) \n", "\n", "pred_init = tf.reshape(pred_init,(SEQ_BATCH,-1))\n", "\n", "average_initial_prediction = tf.reduce_mean(pred_init,axis=0).numpy().astype('float')\n", "\n", "# %%\n", "\n", "seqs_collection = []\n", "scores_collection = []\n", "scores_collection_genes = []\n", "if OPTIMIZE:\n", "\n", " iter_ = 0\n", " for opt_iter in tqdm(range(STEPS)):\n", " \n", " with tf.GradientTape() as gtape:\n", " gtape.watch(noise)\n", " \n", " sequences = wgan(noise)\n", "\n", " seqs_gen = recover_seq(sequences, rev_rna_vocab)\n", " seqs_collection.append(seqs_gen)\n", "\n", " g1_ = tf.zeros_like(sequences)\n", "\n", " scores_collection_temp = []\n", "\n", " for gene in gene_names:\n", "\n", " seqs_dna = retriever.replace_utr_in_sequence(f\"./.cache/{gene}_info.json\", seqs_gen, target_length=10500, output_prefix=\"modified_sequence\") \n", " \n", " seqs = one_hot(seqs_dna)\n", " \n", " with tf.GradientTape() as ptape:\n", " ptape.watch(seqs)\n", "\n", " pred = model(seqs)\n", " t = tf.reshape(pred,(-1))\n", " scores_collection_temp.append(t.numpy().astype('float'))\n", " nt = t.numpy().astype('float')\n", "\n", " g1 = ptape.gradient(pred,seqs)\n", " g1 = tf.math.scalar_mul(-1.0, g1)\n", " g1 = tf.slice(g1,[0,7000,0],[-1,128,-1])\n", "\n", " tmp_g = g1.numpy().astype('float')\n", " tmp_seqs = seqs_gen\n", "\n", " # Initialize tmp_lst with correct size\n", " batch_size = min(len(tmp_seqs), tmp_g.shape[0])\n", " tmp_lst = np.zeros(shape=(batch_size, 128, 5))\n", "\n", " # Loop on the batch size and update the UTR only\n", " for i in range(batch_size):\n", " len_ = min(len(tmp_seqs[i]), tmp_g.shape[1]) # Prevent exceeding tmp_g's dimensions\n", " edited_g = tmp_g[i][:len_, :]\n", " edited_g = np.pad(edited_g, ((0, 128-len_), (0, 1)), 'constant')\n", " tmp_lst[i] = edited_g\n", "\n", " g1 = tf.convert_to_tensor(tmp_lst, dtype=tf.float32)\n", "\n", " g1_ = tf.math.add(g1, g1_)\n", "\n", " scores_collection.append(np.mean(scores_collection_temp,axis=0))\n", " scores_collection_genes.append(scores_collection_temp)\n", " g2 = gtape.gradient(sequences,noise,output_gradients=g1_)\n", "\n", "\n", " a1 = g2 + noise_small\n", " change = [(a1,noise)]\n", "\n", " optimizer.apply_gradients(change)\n", "\n", " iters_.append(iter_)\n", " iter_ += 1\n", "\n", " sequences_opt = wgan(noise)\n", "\n", " gen_seqs_opt = sequences_opt.numpy().astype('float')\n", "\n", " seqs_gen_opt = recover_seq(gen_seqs_opt, rev_rna_vocab)\n", "\n", " seqs_opt = retriever.replace_utr_in_multiple_sequences(gene_names, seqs_gen_opt, target_length=10500, cache_dir=\"./.cache\", output_prefix=\"modified_sequence\")\n", "\n", " seqs_opt = one_hot(seqs_opt)\n", "\n", " pred_opt = model(seqs_opt)\n", "\n", " pred_opt = tf.reshape(pred_opt,(SEQ_BATCH,-1))\n", "\n", "\n", " average_optimized_prediction = tf.reduce_mean(pred_opt,axis=0).numpy().astype('float')\n", "\n", "\n", "best_seqs, best_scores = select_best(scores_collection_genes, seqs_collection, per_gene=True)\n", "\n", "\n", "# %%\n", "print(np.shape(best_scores))\n", "\n", "# %%\n", "\n", "with open('./outputs/mul_init_exps.txt', 'w') as f:\n", " for item in average_initial_prediction:\n", " f.write(f'{item}\\n')\n", "\n", "with open('./outputs/mul_best_exps.txt', 'w') as f:\n", " for item in best_scores:\n", " f.write(f'{item}\\n')\n", "\n", "with open('./outputs/mul_opt_exps.txt', 'w') as f:\n", " for item in average_optimized_prediction:\n", " f.write(f'{item}\\n')\n", "\n", "with open('./outputs/mul_best_seqs.txt', 'w') as f:\n", " for item in best_seqs:\n", " f.write(f'{item}\\n')\n", "\n", "with open('./outputs/mul_init_seqs.txt', 'w') as f:\n", " for item in seqs_gen_init:\n", " f.write(f'{item}\\n')\n", "\n", "# Compute average Log TPM per gene\n", "init_log_tpm_target = tf.reduce_mean(pred_init, axis=1).numpy().astype('float')\n", "opt_log_tpm_target = tf.reduce_mean(pred_opt, axis=1).numpy().astype('float')\n", "opt_log_tpm_target = best_scores\n", "\n", "# Compute overall average Log TPM across target genes\n", "avg_init_log_tpm = np.average(init_log_tpm_target)\n", "avg_opt_log_tpm = np.average(opt_log_tpm_target)\n", "\n", "# Convert Log TPM to TPM for percentage improvement\n", "# Assuming Log TPM is base-10 (common for TPM), TPM = 10^LogTPM\n", "avg_init_tpm = np.power(10, avg_init_log_tpm)\n", "avg_opt_tpm = np.power(10, avg_opt_log_tpm)\n", "\n", "# Compute improvement\n", "log_tpm_diff = avg_opt_log_tpm - avg_init_log_tpm\n", "tpm_improvement = avg_opt_tpm - avg_init_tpm\n", "# Percentage improvement based on TPM: ((opt - init) / init) * 100\n", "if avg_init_tpm != 0: # Avoid division by zero\n", " tpm_percent_change = (tpm_improvement / avg_init_tpm) * 100\n", "else:\n", " tpm_percent_change = float('inf') if tpm_improvement > 0 else 0.0\n", "\n", "# Handle negative and positive percentages\n", "percent_str = f\"{tpm_percent_change:.2f}%\"\n", "if tpm_percent_change < 0:\n", " percent_str = f\"{tpm_percent_change:.2f}% (decrease)\"\n", "elif tpm_percent_change > 0:\n", " percent_str = f\"+{tpm_percent_change:.2f}% (increase)\"\n", "\n", "# Print evaluation results\n", "print(\"\\nEvaluation of Optimization on Original Genes (Log TPM):\")\n", "print(\"\\nExpression Levels (Log TPM):\")\n", "print(f\" Average Initial Log TPM: {avg_init_log_tpm:.4f} (TPM: {avg_init_tpm:.4f})\")\n", "print(f\" Average Optimized Log TPM: {avg_opt_log_tpm:.4f} (TPM: {avg_opt_tpm:.4f})\")\n", "print(f\" Log TPM Difference: {log_tpm_diff:.4f}\")\n", "print(f\" TPM Improvement: {tpm_improvement:.4f} ({percent_str})\")\n", "\n", "\n", "print(\"Genes:\")\n", "print(gene_names)\n", "print(f\"Average Initial Expression: {np.average(average_initial_prediction)}\")\n", "print(f\"Best Expression: {np.average(best_scores)}\")\n", "\n", "\n", "target_refs = []\n", "for gene in target_genes:\n", " output_json = f\"{gene}_info.json\"\n", " cache_path = os.path.join('./.cache/', output_json)\n", " \n", " if not os.path.exists(cache_path):\n", " # Retrieve gene information\n", " gene_info = retriever.get_gene_info(gene, output_json=output_json)\n", " if \"error\" in gene_info:\n", " print(f\"Error retrieving info for {gene}: {gene_info['error']}\")\n", " target_refs.append(None) # Handle errors gracefully\n", " else:\n", " target_refs.append(gene_info[\"promoter_sequence\"])\n", " else:\n", " with open(cache_path, \"r\") as f:\n", " gene_info = json.load(f)\n", " target_refs.append(gene_info[\"promoter_sequence\"])\n", "\n", "\n", "valid_indices = [i for i, ref in enumerate(target_refs) if ref is not None]\n", "target_genes = [target_genes[i] for i in valid_indices]\n", "target_refs = [target_refs[i] for i in valid_indices]\n", "\n", "if not target_genes:\n", " print(\"No valid target genes retrieved. Exiting evaluation.\")\n", "else:\n", "\n", " seqs_gen_init = seqs_gen_init \n", " seqs_gen_opt = best_seqs \n", "\n", "\n", " seqs_init_target = retriever.replace_utr_in_multiple_sequences(\n", " target_genes, seqs_gen_init, target_length=10500, cache_dir=\"./.cache\", output_prefix=\"target_modified_sequence\"\n", " )\n", " seqs_opt_target = retriever.replace_utr_in_multiple_sequences(\n", " target_genes, seqs_gen_opt, target_length=10500, cache_dir=\"./.cache\", output_prefix=\"target_modified_sequence\"\n", " )\n", "\n", "\n", " seqs_init_target = one_hot(seqs_init_target)\n", " seqs_opt_target = one_hot(seqs_opt_target)\n", "\n", "\n", " pred_init_target = model(seqs_init_target)\n", " pred_opt_target = model(seqs_opt_target)\n", "\n", " pred_init_target = tf.reshape(pred_init_target, (len(target_genes), -1))\n", " pred_opt_target = tf.reshape(pred_opt_target, (len(target_genes), -1))\n", "\n", " # Compute average Log TPM per gene\n", " init_log_tpm_target = tf.reduce_mean(pred_init_target, axis=1).numpy().astype('float')\n", " opt_log_tpm_target = tf.reduce_mean(pred_opt_target, axis=1).numpy().astype('float')\n", "\n", " # Compute overall average Log TPM across target genes\n", " avg_init_log_tpm = np.average(init_log_tpm_target)\n", " avg_opt_log_tpm = np.average(opt_log_tpm_target)\n", "\n", " # Convert Log TPM to TPM for percentage improvement\n", " avg_init_tpm = np.power(10, avg_init_log_tpm)\n", " avg_opt_tpm = np.power(10, avg_opt_log_tpm)\n", "\n", " # Compute improvement\n", " log_tpm_diff = avg_opt_log_tpm - avg_init_log_tpm\n", " tpm_improvement = avg_opt_tpm - avg_init_tpm\n", " # Percentage improvement based on TPM: ((opt - init) / init) * 100\n", " if avg_init_tpm != 0: # Avoid division by zero\n", " tpm_percent_change = (tpm_improvement / avg_init_tpm) * 100\n", " else:\n", " tpm_percent_change = float('inf') if tpm_improvement > 0 else 0.0\n", "\n", " # Handle negative and positive percentages\n", " percent_str = f\"{tpm_percent_change:.2f}%\"\n", " if tpm_percent_change < 0:\n", " percent_str = f\"{tpm_percent_change:.2f}% (decrease)\"\n", " elif tpm_percent_change > 0:\n", " percent_str = f\"+{tpm_percent_change:.2f}% (increase)\"\n", "\n", " # Print evaluation results\n", " print(\"\\nEvaluation of Optimization on Target Genes (Log TPM):\")\n", " print(f\"Original Genes: {gene_names}\")\n", " print(f\"Target Genes: {target_genes}\")\n", " print(\"\\nExpression Levels (Log TPM):\")\n", " print(f\" Average Initial Log TPM: {avg_init_log_tpm:.4f} (TPM: {avg_init_tpm:.4f})\")\n", " print(f\" Average Optimized Log TPM: {avg_opt_log_tpm:.4f} (TPM: {avg_opt_tpm:.4f})\")\n", " print(f\" Log TPM Difference: {log_tpm_diff:.4f}\")\n", " print(f\" TPM Improvement: {tpm_improvement:.4f} ({percent_str})\")\n", "\n", " # Save evaluation results to a file\n", " with open('./outputs/target_genes_evaluation.txt', 'w') as f:\n", " f.write(\"Evaluation of Optimization on Target Genes (Log TPM)\\n\")\n", " f.write(f\"Original Genes: {gene_names}\\n\")\n", " f.write(f\"Target Genes: {target_genes}\\n\\n\")\n", " f.write(\"Expression Levels (Log TPM):\\n\")\n", " f.write(f\" Average Initial Log TPM: {avg_init_log_tpm:.4f} (TPM: {avg_init_tpm:.4f})\\n\")\n", " f.write(f\" Average Optimized Log TPM: {avg_opt_log_tpm:.4f} (TPM: {avg_opt_tpm:.4f})\\n\")\n", " f.write(f\" Log TPM Difference: {log_tpm_diff:.4f}\\n\")\n", " f.write(f\" TPM Improvement: {tpm_improvement:.4f} ({percent_str})\\n\")\n", "\n", " # Optional: Per-gene breakdown\n", " print(\"\\nPer-Gene Expression Levels (Log TPM):\")\n", " for gene, init_log, opt_log in zip(target_genes, init_log_tpm_target, opt_log_tpm_target):\n", " init_tpm = np.power(10, init_log)\n", " opt_tpm = np.power(10, opt_log)\n", " tpm_diff = opt_tpm - init_tpm\n", " if init_tpm != 0:\n", " gene_percent = (tpm_diff / init_tpm) * 100\n", " else:\n", " gene_percent = float('inf') if tpm_diff > 0 else 0.0\n", " gene_percent_str = f\"{gene_percent:.2f}%\"\n", " if gene_percent < 0:\n", " gene_percent_str = f\"{gene_percent:.2f}% (decrease)\"\n", " elif gene_percent > 0:\n", " gene_percent_str = f\"+{gene_percent:.2f}% (increase)\"\n", " print(f\" {gene}: Initial Log TPM = {init_log:.4f} (TPM: {init_tpm:.4f}), \"\n", " f\"Optimized Log TPM = {opt_log:.4f} (TPM: {opt_tpm:.4f}), \"\n", " f\"TPM Improvement = {tpm_diff:.4f} ({gene_percent_str})\")\n", "\n", "\n", "\n" ] } ], "metadata": { "kernelspec": { "display_name": "utrgan", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" } }, "nbformat": 4, "nbformat_minor": 5 }