diff --git "a/dpacman/make_splits.ipynb" "b/dpacman/make_splits.ipynb"
new file mode 100644--- /dev/null
+++ "b/dpacman/make_splits.ipynb"
@@ -0,0 +1,2887 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "82a7f2d0",
+   "metadata": {},
+   "source": [
+    "# Scrap notebook for figuring out how to make better splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "9004776a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>tr_seqid</th>\n",
+       "      <th>dna_seqid</th>\n",
+       "      <th>peak_seqid</th>\n",
+       "      <th>chrpeak_id</th>\n",
+       "      <th>tr_name</th>\n",
+       "      <th>chipscore</th>\n",
+       "      <th>total_jaspar_hits</th>\n",
+       "      <th>dna_sequence</th>\n",
+       "      <th>tr_sequence</th>\n",
+       "      <th>scores</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>trseq26_dnaseq49877</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49877</td>\n",
+       "      <td>peakseq24686</td>\n",
+       "      <td>chr12_peak1150</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>trseq26_dnaseq75052</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq75052</td>\n",
+       "      <td>peakseq12129</td>\n",
+       "      <td>chr9_peak512</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>trseq26_dnaseq14843</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq14843</td>\n",
+       "      <td>peakseq12863</td>\n",
+       "      <td>chr1_peak1335</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>trseq26_dnaseq39522</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq39522</td>\n",
+       "      <td>peakseq5250</td>\n",
+       "      <td>chr5_peak280</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>trseq26_dnaseq49215</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49215</td>\n",
+       "      <td>peakseq4451</td>\n",
+       "      <td>chr4_peak201</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>138.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71152</th>\n",
+       "      <td>trseq9_dnaseq4066</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq4066</td>\n",
+       "      <td>peakseq12188</td>\n",
+       "      <td>chr18_peak213</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AATAATATATCTATTTCTTTATCTTTGTCTTCCCTACTGGACTAGC...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71153</th>\n",
+       "      <td>trseq9_dnaseq62118</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq62118</td>\n",
+       "      <td>peakseq46829</td>\n",
+       "      <td>chr18_peak812</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>TAAATATGTATTTTAGTAAAGTGTTATGATACACTGTGATGGGGGT...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71154</th>\n",
+       "      <td>trseq9_dnaseq41538</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq41538</td>\n",
+       "      <td>peakseq3957</td>\n",
+       "      <td>chrY_peak4</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>267.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GACAGGAGTTGTGTACGAATGTGTGTGAATGTGGGAGCCTAACTAG...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71155</th>\n",
+       "      <td>trseq9_dnaseq38134</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq38134</td>\n",
+       "      <td>peakseq40502</td>\n",
+       "      <td>chr5_peak1955</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>CTGGGCGGGTAGGTGAGAGGACAGGAGGGCGAAGTGGAGAGGAGGG...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71156</th>\n",
+       "      <td>trseq9_dnaseq5106</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq5106</td>\n",
+       "      <td>peakseq37888</td>\n",
+       "      <td>chr1_peak4019</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>ACAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAA...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>71157 rows × 11 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        ID tr_seqid    dna_seqid    peak_seqid  \\\n",
+       "0      trseq26_dnaseq49877  trseq26  dnaseq49877  peakseq24686   \n",
+       "1      trseq26_dnaseq75052  trseq26  dnaseq75052  peakseq12129   \n",
+       "2      trseq26_dnaseq14843  trseq26  dnaseq14843  peakseq12863   \n",
+       "3      trseq26_dnaseq39522  trseq26  dnaseq39522   peakseq5250   \n",
+       "4      trseq26_dnaseq49215  trseq26  dnaseq49215   peakseq4451   \n",
+       "...                    ...      ...          ...           ...   \n",
+       "71152    trseq9_dnaseq4066   trseq9   dnaseq4066  peakseq12188   \n",
+       "71153   trseq9_dnaseq62118   trseq9  dnaseq62118  peakseq46829   \n",
+       "71154   trseq9_dnaseq41538   trseq9  dnaseq41538   peakseq3957   \n",
+       "71155   trseq9_dnaseq38134   trseq9  dnaseq38134  peakseq40502   \n",
+       "71156    trseq9_dnaseq5106   trseq9   dnaseq5106  peakseq37888   \n",
+       "\n",
+       "           chrpeak_id tr_name  chipscore  total_jaspar_hits  \\\n",
+       "0      chr12_peak1150    NFYB        6.0                  1   \n",
+       "1        chr9_peak512    NFYB        5.0                  1   \n",
+       "2       chr1_peak1335    NFYB        6.0                  3   \n",
+       "3        chr5_peak280    NFYB        4.0                  1   \n",
+       "4        chr4_peak201    NFYB      138.0                  2   \n",
+       "...               ...     ...        ...                ...   \n",
+       "71152   chr18_peak213    CTCF     1000.0                  1   \n",
+       "71153   chr18_peak812    CTCF     1000.0                  1   \n",
+       "71154      chrY_peak4    CTCF      267.0                  1   \n",
+       "71155   chr5_peak1955    CTCF     1000.0                  2   \n",
+       "71156   chr1_peak4019    CTCF       14.0                  2   \n",
+       "\n",
+       "                                            dna_sequence  \\\n",
+       "0      GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...   \n",
+       "1      TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...   \n",
+       "2      AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...   \n",
+       "3      CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...   \n",
+       "4      GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...   \n",
+       "...                                                  ...   \n",
+       "71152  AATAATATATCTATTTCTTTATCTTTGTCTTCCCTACTGGACTAGC...   \n",
+       "71153  TAAATATGTATTTTAGTAAAGTGTTATGATACACTGTGATGGGGGT...   \n",
+       "71154  GACAGGAGTTGTGTACGAATGTGTGTGAATGTGGGAGCCTAACTAG...   \n",
+       "71155  CTGGGCGGGTAGGTGAGAGGACAGGAGGGCGAAGTGGAGAGGAGGG...   \n",
+       "71156  ACAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAA...   \n",
+       "\n",
+       "                                             tr_sequence  \\\n",
+       "0      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "1      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "2      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "3      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "4      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "...                                                  ...   \n",
+       "71152  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71153  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71154  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71155  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71156  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "\n",
+       "                                                  scores  \n",
+       "0      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "1      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "2      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "3      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "4      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "...                                                  ...  \n",
+       "71152  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "71153  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "71154  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "71155  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "71156  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...  \n",
+       "\n",
+       "[71157 rows x 11 columns]"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "protein_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/protein/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "protein_clusters.columns=[\"tr_cluster_rep\",\"tr_cluster_member\"]\n",
+    "protein_clusters.head()\n",
+    "\n",
+    "dna_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/dna_full/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "dna_clusters.columns=[\"dna_cluster_rep\",\"dna_cluster_member\"]\n",
+    "dna_clusters.head()\n",
+    "\n",
+    "all_data = pd.read_parquet(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/remap2022_crm_fimo_output_q_processed_seed0.parquet\")\n",
+    "all_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce5cf600",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "29\n",
+      "71134\n",
+      "0\n",
+      "0\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>tr_seqid</th>\n",
+       "      <th>dna_seqid</th>\n",
+       "      <th>peak_seqid</th>\n",
+       "      <th>chrpeak_id</th>\n",
+       "      <th>tr_name</th>\n",
+       "      <th>chipscore</th>\n",
+       "      <th>total_jaspar_hits</th>\n",
+       "      <th>dna_sequence</th>\n",
+       "      <th>tr_sequence</th>\n",
+       "      <th>scores</th>\n",
+       "      <th>tr_cluster_rep</th>\n",
+       "      <th>dna_cluster_rep</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>trseq26_dnaseq49877</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49877</td>\n",
+       "      <td>peakseq24686</td>\n",
+       "      <td>chr12_peak1150</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq58018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>trseq26_dnaseq75052</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq75052</td>\n",
+       "      <td>peakseq12129</td>\n",
+       "      <td>chr9_peak512</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq75052</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>trseq26_dnaseq14843</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq14843</td>\n",
+       "      <td>peakseq12863</td>\n",
+       "      <td>chr1_peak1335</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq14843</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>trseq26_dnaseq39522</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq39522</td>\n",
+       "      <td>peakseq5250</td>\n",
+       "      <td>chr5_peak280</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq3280</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>trseq26_dnaseq49215</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49215</td>\n",
+       "      <td>peakseq4451</td>\n",
+       "      <td>chr4_peak201</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>138.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq57257</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    ID tr_seqid    dna_seqid    peak_seqid      chrpeak_id  \\\n",
+       "0  trseq26_dnaseq49877  trseq26  dnaseq49877  peakseq24686  chr12_peak1150   \n",
+       "1  trseq26_dnaseq75052  trseq26  dnaseq75052  peakseq12129    chr9_peak512   \n",
+       "2  trseq26_dnaseq14843  trseq26  dnaseq14843  peakseq12863   chr1_peak1335   \n",
+       "3  trseq26_dnaseq39522  trseq26  dnaseq39522   peakseq5250    chr5_peak280   \n",
+       "4  trseq26_dnaseq49215  trseq26  dnaseq49215   peakseq4451    chr4_peak201   \n",
+       "\n",
+       "  tr_name  chipscore  total_jaspar_hits  \\\n",
+       "0    NFYB        6.0                  1   \n",
+       "1    NFYB        5.0                  1   \n",
+       "2    NFYB        6.0                  3   \n",
+       "3    NFYB        4.0                  1   \n",
+       "4    NFYB      138.0                  2   \n",
+       "\n",
+       "                                        dna_sequence  \\\n",
+       "0  GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...   \n",
+       "1  TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...   \n",
+       "2  AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...   \n",
+       "3  CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...   \n",
+       "4  GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...   \n",
+       "\n",
+       "                                         tr_sequence  \\\n",
+       "0  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "1  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "2  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "3  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "4  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "\n",
+       "                                              scores tr_cluster_rep  \\\n",
+       "0  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "1  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "2  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "3  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "4  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "\n",
+       "  dna_cluster_rep  \n",
+       "0     dnaseq58018  \n",
+       "1     dnaseq75052  \n",
+       "2     dnaseq14843  \n",
+       "3      dnaseq3280  \n",
+       "4     dnaseq57257  "
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "protein_cluster_map = dict(zip(protein_clusters[\"tr_cluster_member\"],protein_clusters[\"tr_cluster_rep\"]))\n",
+    "dna_cluster_map = dict(zip(dna_clusters[\"dna_cluster_member\"],dna_clusters[\"dna_cluster_rep\"]))\n",
+    "print(len(protein_cluster_map))\n",
+    "print(len(dna_cluster_map))\n",
+    "all_data[\"tr_cluster_rep\"] = all_data[\"tr_seqid\"].map(protein_cluster_map)\n",
+    "all_data[\"dna_cluster_rep\"] = all_data[\"dna_seqid\"].map(dna_cluster_map)\n",
+    "print(len(all_data[all_data[\"tr_cluster_rep\"].isna()]))\n",
+    "print(len(all_data[all_data[\"dna_cluster_rep\"].isna()]))\n",
+    "all_data.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2f09e399",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>From</th>\n",
+       "      <th>Entry</th>\n",
+       "      <th>Reviewed</th>\n",
+       "      <th>Entry Name</th>\n",
+       "      <th>Protein names</th>\n",
+       "      <th>Gene Names</th>\n",
+       "      <th>Organism</th>\n",
+       "      <th>Length</th>\n",
+       "      <th>Sequence</th>\n",
+       "      <th>InterPro</th>\n",
+       "      <th>Pfam</th>\n",
+       "      <th>Motif</th>\n",
+       "      <th>Zinc finger</th>\n",
+       "      <th>Protein families</th>\n",
+       "      <th>Binding site</th>\n",
+       "      <th>Site</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>E2F8</td>\n",
+       "      <td>A0AVK6</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>E2F8_HUMAN</td>\n",
+       "      <td>Transcription factor E2F8 (E2F-8)</td>\n",
+       "      <td>E2F8</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>867</td>\n",
+       "      <td>MENEKENLFCEPHKRGLMKTPLKESTTANIVLAEIQPDFGPLTTPT...</td>\n",
+       "      <td>IPR015633;IPR003316;IPR036388;IPR036390;</td>\n",
+       "      <td>PF02319;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>E2F/DP family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>FEZF1</td>\n",
+       "      <td>A0PJY2</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>FEZF1_HUMAN</td>\n",
+       "      <td>Fez family zinc finger protein 1 (Zinc finger ...</td>\n",
+       "      <td>FEZF1 FEZ ZNF312B</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>475</td>\n",
+       "      <td>MDSSCHNATTKMLATAPARGNMMSTSKPLAFSIERIMARTPEPKAL...</td>\n",
+       "      <td>IPR036236;IPR013087;</td>\n",
+       "      <td>PF00096;PF13912;</td>\n",
+       "      <td>MOTIF 28..43; /note=\"Engrailed homology 1 repr...</td>\n",
+       "      <td>ZN_FING 260..282; /note=\"C2H2-type 1\"; /eviden...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>ZNF320</td>\n",
+       "      <td>A2RRD8</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>ZN320_HUMAN</td>\n",
+       "      <td>Zinc finger protein 320</td>\n",
+       "      <td>ZNF320</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>509</td>\n",
+       "      <td>MALSQGLLTFRDVAIEFSQEEWKCLDPAQRTLYRDVMLENYRNLVS...</td>\n",
+       "      <td>IPR050589;IPR001909;IPR036051;IPR036236;IPR013...</td>\n",
+       "      <td>PF01352;PF00096;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>ZN_FING 161..183; /note=\"C2H2-type 1\"; /eviden...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>BDP1</td>\n",
+       "      <td>A6H8Y1</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>BDP1_HUMAN</td>\n",
+       "      <td>Transcription factor TFIIIB component B'' homo...</td>\n",
+       "      <td>BDP1 KIAA1241 KIAA1689 TFNR</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>2624</td>\n",
+       "      <td>MFRRARLSVKPNVRPGVGARGSTASNPQRGRESPRPPDPATDSASK...</td>\n",
+       "      <td>IPR009057;IPR001005;IPR039467;</td>\n",
+       "      <td>PF15963;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>ZNF316</td>\n",
+       "      <td>A6NFI3</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>ZN316_HUMAN</td>\n",
+       "      <td>Zinc finger protein 316</td>\n",
+       "      <td>ZNF316</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>1004</td>\n",
+       "      <td>MAALHTTPDSPAAQLERAEDGSECDPDQEEEEEEEEKGEEVQEVEE...</td>\n",
+       "      <td>IPR001909;IPR036051;IPR036236;IPR013087;</td>\n",
+       "      <td>PF01352;PF00096;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>ZN_FING 345..367; /note=\"C2H2-type 1\"; /eviden...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1177</th>\n",
+       "      <td>1177</td>\n",
+       "      <td>GMEB1</td>\n",
+       "      <td>Q9Y692</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>GMEB1_HUMAN</td>\n",
+       "      <td>Glucocorticoid modulatory element-binding prot...</td>\n",
+       "      <td>GMEB1</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>573</td>\n",
+       "      <td>MANAEVSVPVGDVVVVPTEGNEGENPEDTKTQVILQLQPVQQGLFI...</td>\n",
+       "      <td>IPR010919;IPR000770;</td>\n",
+       "      <td>PF01342;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>BINDING 113; /ligand=\"Zn(2+)\"; /ligand_id=\"ChE...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1178</th>\n",
+       "      <td>1178</td>\n",
+       "      <td>ZFP37</td>\n",
+       "      <td>Q9Y6Q3</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>ZFP37_HUMAN</td>\n",
+       "      <td>Zinc finger protein 37 homolog (Zfp-37)</td>\n",
+       "      <td>ZFP37</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>630</td>\n",
+       "      <td>MSVSSGVQILTKPETVDRRRSAETTKEAGRPLEMAVSEPEASAAEW...</td>\n",
+       "      <td>IPR001909;IPR036051;IPR050826;IPR056436;IPR036...</td>\n",
+       "      <td>PF01352;PF00096;PF23561;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>ZN_FING 293..315; /note=\"C2H2-type 1\"; /eviden...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1179</th>\n",
+       "      <td>1179</td>\n",
+       "      <td>NCOA3</td>\n",
+       "      <td>Q9Y6Q9</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>NCOA3_HUMAN</td>\n",
+       "      <td>Nuclear receptor coactivator 3 (NCoA-3) (EC 2....</td>\n",
+       "      <td>NCOA3 AIB1 BHLHE42 RAC3 TRAM1</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>1424</td>\n",
+       "      <td>MSGLGENLDPLASDSRKRKLPCDTPGQGLTCSGEKRRREQESKYIE...</td>\n",
+       "      <td>IPR011598;IPR056193;IPR036638;IPR010011;IPR032...</td>\n",
+       "      <td>PF23172;PF07469;PF16279;PF16665;PF08815;PF0098...</td>\n",
+       "      <td>MOTIF 685..689; /note=\"LXXLL motif 1\"; MOTIF 7...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>SRC/p160 nuclear receptor coactivator family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1180</th>\n",
+       "      <td>1180</td>\n",
+       "      <td>ZHX2</td>\n",
+       "      <td>Q9Y6X8</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>ZHX2_HUMAN</td>\n",
+       "      <td>Zinc fingers and homeoboxes protein 2 (Alpha-f...</td>\n",
+       "      <td>ZHX2 AFR1 KIAA0854 RAF</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>837</td>\n",
+       "      <td>MASKRKSTTPCMVRTSQVVEQDVPEEVDRAKEKGIGTPQPDVAKDS...</td>\n",
+       "      <td>IPR001356;IPR009057;IPR041057;IPR036236;IPR013...</td>\n",
+       "      <td>PF00046;PF18387;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>ZN_FING 78..101; /note=\"C2H2-type 1\"; /evidenc...</td>\n",
+       "      <td>ZHX family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1181</th>\n",
+       "      <td>1181</td>\n",
+       "      <td>MORC2</td>\n",
+       "      <td>Q9Y6X9</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>MORC2_HUMAN</td>\n",
+       "      <td>ATPase MORC2 (EC 3.6.1.-) (MORC family CW-type...</td>\n",
+       "      <td>MORC2 KIAA0852 ZCWCC1</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>1032</td>\n",
+       "      <td>MAFTNYSSLNRAQLTFEYLHTNSTTHEFLFGALAELVDNARDADAT...</td>\n",
+       "      <td>IPR056360;IPR036890;IPR041006;IPR011124;</td>\n",
+       "      <td>PF23327;PF13589;PF17942;PF07496;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>ZN_FING 490..544; /note=\"CW-type\"; /evidence=\"...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>BINDING 39; /ligand=\"ATP\"; /ligand_id=\"ChEBI:C...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1182 rows × 17 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      Unnamed: 0    From   Entry  Reviewed   Entry Name  \\\n",
+       "0              0    E2F8  A0AVK6  reviewed   E2F8_HUMAN   \n",
+       "1              1   FEZF1  A0PJY2  reviewed  FEZF1_HUMAN   \n",
+       "2              2  ZNF320  A2RRD8  reviewed  ZN320_HUMAN   \n",
+       "3              3    BDP1  A6H8Y1  reviewed   BDP1_HUMAN   \n",
+       "4              4  ZNF316  A6NFI3  reviewed  ZN316_HUMAN   \n",
+       "...          ...     ...     ...       ...          ...   \n",
+       "1177        1177   GMEB1  Q9Y692  reviewed  GMEB1_HUMAN   \n",
+       "1178        1178   ZFP37  Q9Y6Q3  reviewed  ZFP37_HUMAN   \n",
+       "1179        1179   NCOA3  Q9Y6Q9  reviewed  NCOA3_HUMAN   \n",
+       "1180        1180    ZHX2  Q9Y6X8  reviewed   ZHX2_HUMAN   \n",
+       "1181        1181   MORC2  Q9Y6X9  reviewed  MORC2_HUMAN   \n",
+       "\n",
+       "                                          Protein names  \\\n",
+       "0                     Transcription factor E2F8 (E2F-8)   \n",
+       "1     Fez family zinc finger protein 1 (Zinc finger ...   \n",
+       "2                               Zinc finger protein 320   \n",
+       "3     Transcription factor TFIIIB component B'' homo...   \n",
+       "4                               Zinc finger protein 316   \n",
+       "...                                                 ...   \n",
+       "1177  Glucocorticoid modulatory element-binding prot...   \n",
+       "1178            Zinc finger protein 37 homolog (Zfp-37)   \n",
+       "1179  Nuclear receptor coactivator 3 (NCoA-3) (EC 2....   \n",
+       "1180  Zinc fingers and homeoboxes protein 2 (Alpha-f...   \n",
+       "1181  ATPase MORC2 (EC 3.6.1.-) (MORC family CW-type...   \n",
+       "\n",
+       "                         Gene Names              Organism  Length  \\\n",
+       "0                              E2F8  Homo sapiens (Human)     867   \n",
+       "1                 FEZF1 FEZ ZNF312B  Homo sapiens (Human)     475   \n",
+       "2                            ZNF320  Homo sapiens (Human)     509   \n",
+       "3       BDP1 KIAA1241 KIAA1689 TFNR  Homo sapiens (Human)    2624   \n",
+       "4                            ZNF316  Homo sapiens (Human)    1004   \n",
+       "...                             ...                   ...     ...   \n",
+       "1177                          GMEB1  Homo sapiens (Human)     573   \n",
+       "1178                          ZFP37  Homo sapiens (Human)     630   \n",
+       "1179  NCOA3 AIB1 BHLHE42 RAC3 TRAM1  Homo sapiens (Human)    1424   \n",
+       "1180         ZHX2 AFR1 KIAA0854 RAF  Homo sapiens (Human)     837   \n",
+       "1181          MORC2 KIAA0852 ZCWCC1  Homo sapiens (Human)    1032   \n",
+       "\n",
+       "                                               Sequence  \\\n",
+       "0     MENEKENLFCEPHKRGLMKTPLKESTTANIVLAEIQPDFGPLTTPT...   \n",
+       "1     MDSSCHNATTKMLATAPARGNMMSTSKPLAFSIERIMARTPEPKAL...   \n",
+       "2     MALSQGLLTFRDVAIEFSQEEWKCLDPAQRTLYRDVMLENYRNLVS...   \n",
+       "3     MFRRARLSVKPNVRPGVGARGSTASNPQRGRESPRPPDPATDSASK...   \n",
+       "4     MAALHTTPDSPAAQLERAEDGSECDPDQEEEEEEEEKGEEVQEVEE...   \n",
+       "...                                                 ...   \n",
+       "1177  MANAEVSVPVGDVVVVPTEGNEGENPEDTKTQVILQLQPVQQGLFI...   \n",
+       "1178  MSVSSGVQILTKPETVDRRRSAETTKEAGRPLEMAVSEPEASAAEW...   \n",
+       "1179  MSGLGENLDPLASDSRKRKLPCDTPGQGLTCSGEKRRREQESKYIE...   \n",
+       "1180  MASKRKSTTPCMVRTSQVVEQDVPEEVDRAKEKGIGTPQPDVAKDS...   \n",
+       "1181  MAFTNYSSLNRAQLTFEYLHTNSTTHEFLFGALAELVDNARDADAT...   \n",
+       "\n",
+       "                                               InterPro  \\\n",
+       "0              IPR015633;IPR003316;IPR036388;IPR036390;   \n",
+       "1                                  IPR036236;IPR013087;   \n",
+       "2     IPR050589;IPR001909;IPR036051;IPR036236;IPR013...   \n",
+       "3                        IPR009057;IPR001005;IPR039467;   \n",
+       "4              IPR001909;IPR036051;IPR036236;IPR013087;   \n",
+       "...                                                 ...   \n",
+       "1177                               IPR010919;IPR000770;   \n",
+       "1178  IPR001909;IPR036051;IPR050826;IPR056436;IPR036...   \n",
+       "1179  IPR011598;IPR056193;IPR036638;IPR010011;IPR032...   \n",
+       "1180  IPR001356;IPR009057;IPR041057;IPR036236;IPR013...   \n",
+       "1181           IPR056360;IPR036890;IPR041006;IPR011124;   \n",
+       "\n",
+       "                                                   Pfam  \\\n",
+       "0                                              PF02319;   \n",
+       "1                                      PF00096;PF13912;   \n",
+       "2                                      PF01352;PF00096;   \n",
+       "3                                              PF15963;   \n",
+       "4                                      PF01352;PF00096;   \n",
+       "...                                                 ...   \n",
+       "1177                                           PF01342;   \n",
+       "1178                           PF01352;PF00096;PF23561;   \n",
+       "1179  PF23172;PF07469;PF16279;PF16665;PF08815;PF0098...   \n",
+       "1180                                   PF00046;PF18387;   \n",
+       "1181                   PF23327;PF13589;PF17942;PF07496;   \n",
+       "\n",
+       "                                                  Motif  \\\n",
+       "0                                                   NaN   \n",
+       "1     MOTIF 28..43; /note=\"Engrailed homology 1 repr...   \n",
+       "2                                                   NaN   \n",
+       "3                                                   NaN   \n",
+       "4                                                   NaN   \n",
+       "...                                                 ...   \n",
+       "1177                                                NaN   \n",
+       "1178                                                NaN   \n",
+       "1179  MOTIF 685..689; /note=\"LXXLL motif 1\"; MOTIF 7...   \n",
+       "1180                                                NaN   \n",
+       "1181                                                NaN   \n",
+       "\n",
+       "                                            Zinc finger  \\\n",
+       "0                                                   NaN   \n",
+       "1     ZN_FING 260..282; /note=\"C2H2-type 1\"; /eviden...   \n",
+       "2     ZN_FING 161..183; /note=\"C2H2-type 1\"; /eviden...   \n",
+       "3                                                   NaN   \n",
+       "4     ZN_FING 345..367; /note=\"C2H2-type 1\"; /eviden...   \n",
+       "...                                                 ...   \n",
+       "1177                                                NaN   \n",
+       "1178  ZN_FING 293..315; /note=\"C2H2-type 1\"; /eviden...   \n",
+       "1179                                                NaN   \n",
+       "1180  ZN_FING 78..101; /note=\"C2H2-type 1\"; /evidenc...   \n",
+       "1181  ZN_FING 490..544; /note=\"CW-type\"; /evidence=\"...   \n",
+       "\n",
+       "                                   Protein families  \\\n",
+       "0                                     E2F/DP family   \n",
+       "1     Krueppel C2H2-type zinc-finger protein family   \n",
+       "2     Krueppel C2H2-type zinc-finger protein family   \n",
+       "3                                               NaN   \n",
+       "4     Krueppel C2H2-type zinc-finger protein family   \n",
+       "...                                             ...   \n",
+       "1177                                            NaN   \n",
+       "1178  Krueppel C2H2-type zinc-finger protein family   \n",
+       "1179   SRC/p160 nuclear receptor coactivator family   \n",
+       "1180                                     ZHX family   \n",
+       "1181                                            NaN   \n",
+       "\n",
+       "                                           Binding site Site  \n",
+       "0                                                   NaN  NaN  \n",
+       "1                                                   NaN  NaN  \n",
+       "2                                                   NaN  NaN  \n",
+       "3                                                   NaN  NaN  \n",
+       "4                                                   NaN  NaN  \n",
+       "...                                                 ...  ...  \n",
+       "1177  BINDING 113; /ligand=\"Zn(2+)\"; /ligand_id=\"ChE...  NaN  \n",
+       "1178                                                NaN  NaN  \n",
+       "1179                                                NaN  NaN  \n",
+       "1180                                                NaN  NaN  \n",
+       "1181  BINDING 39; /ligand=\"ATP\"; /ligand_id=\"ChEBI:C...  NaN  \n",
+       "\n",
+       "[1182 rows x 17 columns]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "idmap = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/remap/idmapping_reviewed_true_processed_2025_08_11.tsv\", sep=\"\\t\")\n",
+    "idmap"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "94f26da4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'trseq13': 'MGRKKIQITRIMDERNRQVTFTKRKFGLMKKAYELSVLCDCEIALIIFNSSNKLFQYASTDMDKVLLKYTEYNEPHESRTNSDIVEALNKKEHRGCDSPDPDTSYVLTPHTEEKYKKINEEFDNMMRNHKIAPGLPPQNFSMSVTVPVTSPNALSYTNPGSSLVSPSLAASSTLTDSSMLSPPQTTLHRNVSPGAPQRPPSTGNAGGMLSTTDLTVPNGAGSSPVGNGFVNSRASPNLIGATGANSLGKVMPTKSPPPPGGGNLGMNSRKPDLRVVIPPSSKGMMPPLSEEEELELNTQRISSSQATQPLATPVVSVTTPSLPPQGLVYSAMPTAYNTDYSLTSADLSALQGFNSPGMLSLGQVSAWQQHHLGQAALSSLVAGGQLSQGSNLSINTNQNISIKSEPISPPRDRMTPSGFQQQQQQQQQQQPPPPPQPQPQPPQPQPRQEMGRSPVDSLSSSSSSYDGSDREDPRGDFHSPIVLGRPPNTEDRESPSVKRMRMDAWVT',\n",
+       " 'trseq8': 'MEEPQSDPSVEPPLSQETFSDLWKLLPENNVLSPLPSQAMDDLMLSPDDIEQWFTEDPGPDEAPRMPEAAPPVAPAPAAPTPAAPAPAPSWPLSSSVPSQKTYQGSYGFRLGFLHSGTAKSVTCTYSPALNKMFCQLAKTCPVQLWVDSTPPPGTRVRAMAIYKQSQHMTEVVRRCPHHERCSDSDGLAPPQHLIRVEGNLRVEYLDDRNTFRHSVVVPYEPPEVGSDCTTIHYNYMCNSSCMGGMNRRPILTIITLEDSSGNLLGRNSFEVRVCACPGRDRRTEEENLRKKGEPHHELPPGSTKRALPNNTSSSPQPKKKPLDGEYFTLQIRGRERFEMFRELNEALELKDAQAGKEPGGSRAHSSHLKSKKGQSTSRHKKLMFKTEGPDSD',\n",
+       " 'trseq29': 'MTSSSPAGLEGSDLSSINTMMSAVMSVGKVTENGGSPQGIKSPSKPPGPNRIGRRNQETKEEKSSYNCPLCEKICTTQHQLTMHIRQHNTDTGGADHSCSICGKSLSSASSLDRHMLVHSGERPYKCTVCGQSFTTNGNMHRHMKIHEKDPNSATATAPPSPLKRRRLSSKRKLSHDAESEREDPAPAKKMVEDGQSGDLEKKADEVFHCPVCFKEFVCKYGLETHMETHSDNPLRCDICCVTFRTHRGLLRHNALVHKQLPRDAMGRPFIQNNPSIPAGFHDLGFTDFSCRKFPRISQAWCETNLRRCISEQHRFVCDTCDKAFPMLCSLALHKQTHVAADQGQEKPQATPLPGDALDQKGFLALLGLQHTKDVRPAPAEEPLPDDNQAIQLQTLKCQLPQDPGCTNLLSLSPFEAASLGGSLTVLPATKDSIKHLSLQPFQKGFIIQPDSSIVVKPISGESAIELADIQQILKMAASAPPQISLPPFSKAPAAPLQAIFKHMPPLKPKPLVTPRTVVATSTPPPLINAQQASPGCISPSLPPPPLKLLKGSVEAASNAHLLQSKSGTQPHAATRLSLQQPRAELPGQPEMKTQLEQDSIIEALLPLSMEAKIKQEITEGELKAFMTAPGGKKTPAMRKVLYPCRFCNQVFAFSGVLRAHVRSHLGISPYQCNICDYIAADKAALIRHLRTHSGERPYICKICHYPFTVKANCERHLRKKHLKATRKDIEKNIEYVSSSAAELVDAFCAPDTVCRLCGEDLKHYRALRIHMRTHCGRGLGGGHKGRKPFECKECSAAFAAKRNCIHHILKQHLHVPEQDIESYVLAADGLGPAEAPAAEASGRGEDSGCAALGDCKPLTAFLEPQNGFLHRGPTQPPPPHVSIKLEPASSFAVDFNEPLDFSQKGLALVQVKQENISFLSPSSLVPYDCSMEPIDLSIPKNFRKGDKDLATPSEAKKPEEEAGSSEQPSPCPAPGPSLPVTLGPSGILESPMAPAPAATPEPPAQPLQGPVQLAVPIYSSALVSSPPLVGSSALLSGTALLRPLRPKPPLLLPKPPVTEELPPLASIAQIISSVSSAPTLLKTKVADPGPASTGSNTTASDSLGGSVPKAATTATPAATTSPKESSEPPAPASSPEAASPTEQGPAGTSKKRGRKRGMRSRPRANSGGVDLDSSGEFASIEKMLATTDTNKFSPFLQTAEDNTQDEVAGAPADHHGPSDEEQGSPPEDKLLRAKRNSYTNCLQKITCPHCPRVFPWASSLQRHMLTHTDSQSDAETAAAAGEVLDLTSRDREQPSEGATELRQVAGDAPVEQATAETASPVHREEHGRGESHEPEEEHGTEESTGDADGAEEDASSNQSLDLDFATKLMDFKLAEGDGEAGAGGAASQEQKLACDTCGKSFKFLGTLSRHRKAHGRQEPKDEKGDGASTAEEGPQPAPEQEEKPPETPAEVVESAPGAGEAPAEKLAEETEGPSDGESAAEKRSSEKSDDDKKPKTDSPKSVASKADKRKKVCSVCNKRFWSLQDLTRHMRSHTGERPYKCQTCERTFTLKHSLVRHQRIHQKARHAKHHGKDSDKEERGEEDSENESTHSGNNAVSENEAELAPNASNHMAVTRSRKEGLASATKDCSHREEKVTAGWPSEPGQGDLNPESPAALGQDLLEPRSKRPAHPILATADGASQLVGME',\n",
+       " 'trseq20': 'MPVERMRMRPWLEEQINSNTIPGLKWLNKEKKIFQIPWMHAARHGWDVEKDAPLFRNWAIHTGKHQPGVDKPDPKTWKANFRCAMNSLPDIEEVKDKSIKKGNNAFRVYRMLPLSERPSKKGKKPKTEKEDKVKHIKQEPVESSLGLSNGVSDLSPEYAVLTSTIKNEVDSTVNIIVVGQSHLDSNIENQEIVTNPPDICQVVEVTTESDEQPVSMSELYPLQISPVSSYAESETTDSVPSDEESAEGRPHWRKRNIEGKQYLSNMGTRGSYLLPGMASFVTSNKPDLQVTIKEESNPVPYNSSWPPFQDLPLSSSMTPASSSSRPDRETRASVIKKTSDITQARVKSC',\n",
+       " 'trseq17': 'MNFETSRCATLQYCPDPYIQRFVETPAHFSWKESYYRSTMSQSTQTNEFLSPEVFQHIWDFLEQPICSVQPIDLNFVDEPSEDGATNKIEISMDCIRMQDSDLSDPMWPQYTNLGLLNSMDQQIQNGSSSTSPYNTDHAQNSVTAPSPYAQPSSTFDALSPSPAIPSNTDYPGPHSFDVSFQQSSTAKSATWTYSTELKKLYCQIAKTCPIQIKVMTPPPQGAVIRAMPVYKKAEHVTEVVKRCPNHELSREFNEGQIAPPSHLIRVEGNSHAQYVEDPITGRQSVLVPYEPPQVGTEFTTVLYNFMCNSSCVGGMNRRPILIIVTLETRDGQVLGRRCFEARICACPGRDRKADEDSIRKQQVSDSTKNGDGTKRPFRQNTHGIQMTSIKKRRSPDDELLYLPVRGRETYEMLLKIKESLELMQYLPQHTIETYRQQQQQQHQHLLQKQTSIQSPSSYGNSSPPLNKMNSMNKLPSVSQLINPQQRNALTPTTIPDGMGANIPMMGTHMPMAGDMNGLSPTQALPPPLSMPSTSHCTPPPPYPTDCSIVSFLARLGCSSCLDYFTTQGLTTIYQIEHYSMDDLASLKIPEQFRHAIWKGILDHRQLHEFSSPSHLLRTPSSASTVSVGSSETRGERVIDAVRFTLRQTISFPPRDEWNDFNFDMDARRNKQQRIKEEGE',\n",
+       " 'trseq28': 'MTSPSPRIQIISTDSAVASPQRIQIVTDQQTGQKIQIVTAVDASGSPKQQFILTSPDGAGTGKVILASPETSSAKQLIFTTSDNLVPGRIQIVTDSASVERLLGKTDVQRPQVVEYCVVCGDKASGRHYGAVSCEGCKGFFKRSVRKNLTYSCRSNQDCIINKHHRNRCQFCRLKKCLEMGMKMESVQSERKPFDVQREKPSNCAASTEKIYIRKDLRSPLIATPTFVADKDGARQTGLLDPGMLVNIQQPLIREDGTVLLATDSKAETSQGALGTLANVVTSLANLSESLNNGDTSEIQPEDQSASEITRAFDTLAKALNTTDSSSSPSLADGIDTSGGGSIHVISRDQSTPIIEVEGPLLSDTHVTFKLTMPSPMPEYLNVHYICESASRLLFLSMHWARSIPAFQALGQDCNTSLVRACWNELFTLGLAQCAQVMSLSTILAAIVNHLQNSIQEDKLSGDRIKQVMEHIWKLQEFCNSMAKLDIDGYEYAYLKAIVLFSPDHPGLTSTSQIEKFQEKAQMELQDYVQKTYSEDTYRLARILVRLPALRLMSSNITEELFFTGLIGNVSIDSIIPYILKMETAEYNGQITGASL',\n",
+       " 'trseq27': 'MTMTLHTKASGMALLHQIQGNELEPLNRPQLKIPLERPLGEVYLDSSKPAVYNYPEGAAYEFNAAAAANAQVYGQTGLPYGPGSEAAAFGSNGLGGFPPLNSVSPSPLMLLHPPPQLSPFLQPHGQQVPYYLENEPSGYTVREAGPPAFYRPNSDNRRQGGRERLASTNDKGSMAMESAKETRYCAVCNDYASGYHYGVWSCEGCKAFFKRSIQGHNDYMCPATNQCTIDKNRRKSCQACRLRKCYEVGMMKGGIRKDRRGGRMLKHKRQRDDGEGRGEVGSAGDMRAANLWPSPLMIKRSKKNSLALSLTADQMVSALLDAEPPILYSEYDPTRPFSEASMMGLLTNLADRELVHMINWAKRVPGFVDLTLHDQVHLLECAWLEILMIGLVWRSMEHPGKLLFAPNLLLDRNQGKCVEGMVEIFDMLLATSSRFRMMNLQGEEFVCLKSIILLNSGVYTFLSSTLKSLEEKDHIHRVLDKITDTLIHLMAKAGLTLQQQHQRLAQLLLILSHIRHMSNKGMEHLYSMKCKNVVPLYDLLLEMLDAHRLHAPTSRGGASVEETDQSHLATAGSTSSHSLQKYYITGEAEGFPATV',\n",
+       " 'trseq15': 'MLWKITDNVKYEEDCEDRHDGSSNGNPRVPHLSSAGQHLYSPAPPLSHTGVAEYQPPPYFPPPYQQLAYSQSADPYSHLGEAYAAAINPLHQPAPTGSQQQAWPGRQSQEGAGLPSHHGRPAGLLPHLSGLEAGAVSARRDAYRRSDLLLPHAHALDAAGLAENLGLHDMPHQMDEVQNVDDQHLLLHDQTVIRKGPISMTKNPLNLPCQKELVGAVMNPTEVFCSVPGRLSLLSSTSKYKVTVAEVQRRLSPPECLNASLLGGVLRRAKSKNGGRSLREKLDKIGLNLPAGRRKAAHVTLLTSLVEGEAVHLARDFAYVCEAEFPSKPVAEYLTRPHLGGRNEMAARKNMLLAAQQLCKEFTELLSQDRTPHGTSRLAPVLETNIQNCLSHFSLITHGFGSQAICAAVSALQNYIKEALIVIDKSYMNPGDQSPADSNKTLEKMEKHRK',\n",
+       " 'trseq4': 'MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRRFRFQEAAGPREALSRLQELCHGWLRPEMRTKEQILELLVLEQFLTILPQEIQSRVQELHPESGEEAVTLVEDMQRELGRLRQQVTNHGRGTEVLLEEPLPLETARESPSFKLEPMETERSPGPRLQELLGPSPQRDPQAVKERALSAPWLSLFPPEGNMEDKEMTGPQLPESLEDVAMYISQEEWGHQDPSKRALSRDTVQESYENVDSLESHIPSQEVPGTQVGQGGKLWDPSVQSCKEGLSPRGPAPGEEKFENLEGVPSVCSENIHPQVLLPDQARGEVPWSPELGRPHDRSQGDWAPPPEGGMEQALAGASSGRELGRPKELQPKKLHLCPLCGKNFSNNSNLIRHQRIHAAERLCMGVDCTEIFGGNPRFLSLHRAHLGEEAHKCLECGKCFSQNTHLTRHQRTHTGEKPYQCNICGKCFSCNSNLHRHQRTHTGEKPYKCPECGEIFAHSSNLLRHQRIHTGERPYKCPECGKSFSRSSHLVIHERTHERERLYPFSECGEAVSDSTPFLTNHGAHKAEKKLFECLTCGKSFRQGMHLTRHQRTHTGEKPYKCTLCGENFSHRSNLIRHQRIHTGEKPYTCHECGDSFSHSSNRIRHLRTHTGERPYKCSECGESFSRSSRLMSHQRTHTG',\n",
+       " 'trseq18': 'MNTTDNGVNCLCAICGDRATGKHYGASSCDGCKGFFRRSIRKSHVYSCRFSRQCVVDKDKRNQCRYCRLRKCFRAGMKKEAVQNERDRISTRRSTFDGSNIPSINTLAQAEVRSRQISVSSPGSSTDINVKKIASIGDVCESMKQQLLVLVEWAKYIPAFCELPLDDQVALLRAHAGEHLLLGATKRSMMYKDILLLGNNYVIHRNSCEVEISRVANRVLDELVRPFQEIQIDDNEYACLKAIVFFDPDAKGLSDPVKIKNMRFQVQIGLEDYINDRQYDSRGRFGELLLLLPTLQSITWQMIEQIQFVKLFGMVKIDNLLQEMLLGGASNDGSHLHHPMHPHLSQDPLTGQTILLGPMSTLVHADQISTPETPLPSPPQGSGQEQYKIAANQASVISHQHLSKQKQL',\n",
+       " 'trseq26': 'MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTNGSKESFREQDIYLPIANVARIMKNAIPQTGKIAKDAKECVQECVSEFISFITSEASERCHQEKRKTINGEDILFAMSTLGFDSYVEPLKLYLQKFREAMKGEKGIGGAVTATDGLSEELTEEAFTNQLPAGLITTDGQQQNVMVYTTSYQQISGVQQIQFS',\n",
+       " 'trseq25': 'MSVDPLSSKALKIKRELSENTPHLSDEALMGLSVRELNRHLRGLSAEEVTRLKQRRRTLKNRGYAASCRVKRVCQKEELQKQKSELEREVDKLARENAAMRLELDALRGKCEALQGFARSVAAARGPATLVAPASVITIVKSTPGSGSGPAHGPDPAHGPASCS',\n",
+       " 'trseq16': 'MMQESGTETKSNGSAIQNGSGGSNHLLECGGLREGRSNGETPAVDIGAADLAHAQQQQQQALQVARQLLLQQQQQQQVSGLKSPKRNDKQPALQVPVSVAMMTPQVITPQQMQQILQQQVLSPQQLQVLLQQQQALMLQQQQLQEFYKKQQEQLQLQLLQQQHAGKQPKEQQQVATQQLAFQQQLLQMQQLQQQHLLSLQRQGLLTIQPGQPALPLQPLAQGMIPTELQQLWKEVTSAHTAEETTGNNHSSLDLTTTCVSSSAPSKTSLIMNPHASTNGQLSVHTPKRESLSHEEHPHSHPLYGHGVCKWPGCEAVCEDFQSFLKHLNSEHALDDRSTAQCRVQMQVVQQLELQLAKDKERLQAMMTHLHVKSTEPKAAPQPLNLVSSVTLSKSASEASPQSLPHTPTTPTAPLTPVTQGPSVITTTSMHTVGPIRRRYSDKYNVPISSADIAQNQEFYKNAEVRPPFTYASLIRQAILESPEKQLTLNEIYNWFTRMFAYFRRNAATWKNAVRHNLSLHKCFVRVENVKGAVWTVDEVEFQKRRPQKISGNPSLIKNMQSSHAYCTPLNAALQASMAENSIPLYTTASMGNPTLGNLASAIREELNGAMEHTNSNESDSSPGRSPMQAVHPVHVKEEPLDPEEAEGPLSLVTTANHSPDFDHDRDYEDEPVNEDME',\n",
+       " 'trseq10': 'MEQYTANSNSSTEQIVVQAGQIQQQQQGGVTAVQLQTEAQVASASGQQVQTLQVVQGQPLMVQVSGGQLITSTGQPIMVQAVPGGQGQTIMQVPVSGTQGLQQIQLVPPGQIQIQGGQAVQVQGQQGQTQQIIIQQPQTAVTAGQTQTQQQIAVQGQQVAQTAEGQTIVYQPVNADGTILQQVTVPVSGMITIPAASLAGAQIVQTGANTNTTSSGQGTVTVTLPVAGNVVNSGGMVMMVPGAGSVPAIQRIPLPGAEMLEEEPLYVNAKQYHRILKRRQARAKLEAEGKIPKERRKYLHESRHRHAMARKRGEGGRFFSPKEKDSPHMQDPNQADEEAMTQIIRVS',\n",
+       " 'trseq23': 'MSDPQTSMAATAAVSPSDYLQPAASTTQDSQPSPLALLAATCSKIGPPAVEAAVTPPAPPQPTPRKLVPIKPAPLPLSPGKNSFGILSSKGNILQIQGSQLSASYPGGQLVFAIQNPTMINKGTRSNANIQYQAVPQIQASNSQTIQVQPNLTNQIQIIPGTNQAIITPSPSSHKPVPIKPAPIQKSSTTTTPVQSGANVVKLTGGGGNVTLTLPVNNLVNASDTGAPTQLLTESPPTPLSKTNKKARKKSLPASQPPVAVAEQVETVLIETTADNIIQAGNNLLIVQSPGGGQPAVVQQVQVVPPKAEQQQVVQIPQQALRVVQAASATLPTVPQKPSQNFQIQAAEPTPTQVYIRTPSGEVQTVLVQDSPPATAAATSNTTCSSPASRAPHLSGTSKKHSAAILRKERPLPKIAPAGSIISLNAAQLAAAAQAMQTININGVQVQGVPVTITNTGGQQQLTVQNVSGNNLTISGLSPTQIQLQMEQALAGETQPGEKRRRMACTCPNCKDGEKRSGEQGKKKHVCHIPDCGKTFRKTSLLRAHVRLHTGERPFVCNWFFCGKRFTRSDELQRHARTHTGDKRFECAQCQKRFMRSDHLTKHYKTHLVTKNL',\n",
+       " 'trseq11': 'MESRKLISATDIQYSGSLLNSLNEQRGHGLFCDVTVIVEDRKFRAHKNILSASSTYFHQLFSVAGQVVELSFIRAEIFAEILNYIYSSKIVRVRSDLLDELIKSGQLLGVKFIAELGVPLSQVKSISGTAQDGNTEPLPPDSGDKNLVIQKSKDEAQDNGATIMPIITESFSLSAEDYEMKKIIVTDSDDDDDDVIFCSEILPTKETLPSNNTVAQVQSNPGPVAISDVAPSASNNSPPLTNITPTQKLPTPVNQATLSQTQGSEKLLVSSAPTHLTPNIILLNQTPLSTPPNVSSSLPNHMPSSINLLVQNQQTPNSAILTGNKANEEEEEEIIDDDDDTISSSPDSAVSNTSLVPQADTSQNTSFDGSLIQKMQIPTLLQEPLSNSLKISDIITRNTNDPGVGSKHLMEGQKIITLDTATEIEGLSTGCKVYANIGEDTYDIVIPVKDDPDEGEARLENEIPKTSGSEMANKRMKVKHDDHYELIVDGRVYYICIVCKRSYVCLTSLRRHFNIHSWEKKYPCRYCEKVFPLAEYRTKHEIHHTGERRYQCLACGKSFINYQFMSSHIKSVHSQDPSGDSKLYRLHPCRSLQIRQYAYLSDRSSTIPAMKDDGIGYKVDTGKEPPVGTTTSTQNKPMTWEDIFIQQENDSIFKQNVTDGSTEFEFIIPESY',\n",
+       " 'trseq3': 'MASGDTLYIATDGSEMPAEIVELHEIEVETIPVETIETTVVGEEEEEDDDDEDGGGGDHGGGGGHGHAGHHHHHHHHHHHPPMIALQPLVTDDPTQVHHHQEVILVQTREEVVGGDDSDGLRAEDGFEDQILIPVPAPAGGDDDYIEQTLVTVAAAGKSGGGGSSSSGGGRVKKGGGKKSGKKSYLSGGAGAAGGGGADPGNKKWEQKQVQIKTLEGEFSVTMWSSDEKKDIDHETVVEEQIIGENSPPDYSEYMTGKKLPPGGIPGIDLSDPKQLAEFARMKPRKIKEDDAPRTIACPHKGCTKMFRDNSAMRKHLHTHGPRVHVCAECGKAFVESSKLKRHQLVHTGEKPFQCTFEGCGKRFSLDFNLRTHVRIHTGDRPYVCPFDGCNKKFAQSTNLKSHILTHAKAKNNQ',\n",
+       " 'trseq22': 'MRKGIQPALEQYLVTAGGGEGAAVVAAAAAASMDKRALLASPGFAAAAAAAAAPGAYIQILTTNTSTTSCSSSLQSGAVAAGPLLPSAPGAEQTAGSLLYTTPHGPSSRAGLLQQPPALGRGGSGGGGGPPAKRRLELGESGHQYLSDGLKTPKGKGRAALRSPDSPKTPKSPSEKTRYDTSLGLLTKKFIQLLSQSPDGVLDLNKAAEVLKVQKRRIYDITNVLEGIHLIKKKSKNNVQWMGCSLSEDGGMLAQCQGLSKEVTELSQEEKKLDELIQSCTLDLKLLTEDSENQRLAYVTYQDIRKISGLKDQTVIVVKAPPETRLEVPDSIESLQIHLASTQGPIEVYLCPEETETHSPMKTNNQDHNGNIPKPASKDLASTNSGHSDCSVSMGNLSPLASPANLLQQTEDQIPSNLEGPFVNLLPPLLQEDYLLSLGEEEGISDLFDAYDLEKLPLVEDFMCS',\n",
+       " 'trseq7': 'MDSKESLTPGREENPSSVLAQERGDVMDFYKTLRGGATVKVSASSPSLAVASQSDSKQRRLLVDFPKGSVSNAQQPDLSKAVSLSMGLYMGETETKVMGNDLGFPQQGQISLSSGETDLKLLEESIANLNRSTSVPENPKSSASTAVSAAPTEKEFPKTHSDVSSEQQHLKGQTGTNGGNVKLYTTDQSTFDILQDLEFSSGSPGKETNESPWRSDLLIDENCLLSPLAGEDDSFLLEGNSNEDCKPLILPDTKPKIKDNGDLVLSSPSNVTLPQVKTEKEDFIELCTPGVIKQEKLGTVYCQASFPGANIIGNKMSAISVHGVSTSGGQMYHYDMNTASLSQQQDQKPIFNVIPPIPVGSENWNRCQGSGDDNLTSLGTLNFPGRTVFSNGYSSPSMRPDVSSPPSSSSTATTGPPPKLCLVCSDEASGCHYGVLTCGSCKVFFKRAVEGQHNYLCAGRNDCIIDKIRRKNCPACRYRKCLQAGMNLEARKTKKKIKGIQQATTGVSQETSENPGNKTIVPATLPQLTPTLVSLLEVIEPEVLYAGYDSSVPDSTWRIMTTLNMLGGRQVIAAVKWAKAIPGFRNLHLDDQMTLLQYSWMFLMAFALGWRSYRQSSANLLCFAPDLIINEQRMTLPCMYDQCKHMLYVSSELHRLQVSYEEYLCMKTLLLLSSVPKDGLKSQELFDEIRMTYIKELGKAIVKREGNSSQNWQRFYQLTKLLDSMHEVVENLLNYCFQTFLDKTMSIEFPEMLAEIITNQIPKYSNGNIKKLLFHQK',\n",
+       " 'trseq19': 'MPITRMRMRPWLEMQINSNQIPGLIWINKEEMIFQIPWKHAAKHGWDINKDACLFRSWAIHTGRYKAGEKEPDPKTWKANFRCAMNSLPDIEEVKDQSRNKGSSAVRVYRMLPPLTKNQRKERKSKSSRDAKSKAKRKSCGDSSPDTFSDGLSSSTLPDDHSSYTVPGYMQDLEVEQALTPALSPCAVSSTLPDWHIPVEVVPDSTSDLYNFQVSPMPSTSEATTDEDEEGKLPEDIMKLLEQSEWQPTNVDGKGYLLNEPGVQPTSVYGDFSCKEEPEIDSPGGDIGLSLQRVFTDLKNMDATWLDSLLTPVRLPSIQAIPCAP',\n",
+       " 'trseq5': 'MATQVMGQSSGGGGLFTSSGNIGMALPNDMYDLHDLSKAELAAPQLIMLANVALTGEVNGSCCDYLVGEERQMAELMPVGDNNFSDSEEGEGLEESADIKGEPHGLENMELRSLELSVVEPQPVFEASGAPDIYSSNKDLPPETPGAEDKGKSSKTKPFRCKPCQYEAESEEQFVHHIRVHSAKKFFVEESAEKQAKARESGSSTAEEGDFSKGPIRCDRCGYNTNRYDHYTAHLKHHTRAGDNERVYKCIICTYTTVSEYHWRKHLRNHFPRKVYTCGKCNYFSDRKNNYVQHVRTHTGERPYKCELCPYSSSQKTHLTRHMRTHSGEKPFKCDQCSYVASNQHEVTRHARQVHNGPKPLNCPHCDYKTADRSNFKKHVELHVNPRQFNCPVCDYAASKKCNLQYHFKSKHPTCPNKTMDVSKVKLKKTKKREADLPDNITNEKTEIEQTKIKGDVAGKKNEKSVKAEKRDVSKEKKPSNNVSVIQVTTRTRKSVTEVKEMDVHTGSNSEKFSKTKKSKRKLEVDSHSLHGPVNDEESSTKKKKKVESKSKNNSQEVPKGDSKVEENKKQNTCMKKSTKKKTLKNKSSKKSSKPPQKEPVEKGSAQMDPPQMGPAPTEAVQKGPVQVEPPPPMEHAQMEGAQIRPAPDEPVQMEVVQEGPAQKELLPPVEPAQMVGAQIVLAHMELPPPMETAQTEVAQMGPAPMEPAQMEVAQVESAPMQVVQKEPVQMELSPPMEVVQKEPVQIELSPPMEVVQKEPVKIELSPPIEVVQKEPVQMELSPPMGVVQKEPAQREPPPPREPPLHMEPISKKPPLRKDKKEKSNMQSERARKEQVLIEVGLVPVKDSWLLKESVSTEDLSPPSPPLPKENLREEASGDQKLLNTGEGNKEAPLQKVGAEEADESLPGLAANINESTHISSSGQNLNTPEGETLNGKHQTDSIVCEMKMDTDQNTRENLTGINSTVEEPVSPMLPPSAVEEREAVSKTALASPPATMAANESQEIDEDEGIHSHEGSDLSDNMSEGSDDSGLHGARPVPQESSRKNAKEALAVKAAKGDFVCIFCDRSFRKGKDYSKHLNRHLVNVYYLEEAAQGQE',\n",
+       " 'trseq2': 'MAAVVQQNDLVFEFASNVMEDERQLGDPAIFPAVIVEHVPGADILNSYAGLACVEEPNDMITESSLDVAEEEIIDDDDDDITLTVEASCHDGDETIETIEAAEALLNMDSPGPMLDEKRINNNIFSSPEDDMVVAPVTHVSVTLDGIPEVMETQQVQEKYADSPGASSPEQPKRKKGRKTKPPRPDSPATTPNISVKKKNKDGKGNTIYLWEFLLALLQDKATCPKYIKWTQREKGIFKLVDSKAVSRLWGKHKNKPDMNYETMGRALRYYYQRGILAKVEGQRLVYQFKEMPKDLIYINDEDPSSSIESSDPSLSSSATSNRNQTSRSRVSSSPGVKGGATTVLKPGNSKAAKPKDPVEVAQPSEVLRTVQPTQSPYPTQLFRTVHVVQPVQAVPEGEAARTSTMQDETLNSSVQSIRTIQAPTQVPVVVSPRNQQLHTVTLQTVPLTTVIASTDPSAGTGSQKFILQAIPSSQPMTVLKENVMLQSQKAGSPPSIVLGPAQVQQVLTSNVQTICNGTVSVASSPSFSATAPVVTFSPRSSQLVAHPPGTVITSVIKTQETKTLTQEVEKKESEDHLKENTEKTEQQPQPYVMVVSSSNGFTSQVAMKQNELLEPNSF',\n",
+       " 'trseq21': 'MQNSEGGADSPASVALRPSAAAPPVPASPQRVLVQAASSNPKGAQMQPISLPRVQQVPQQVQPVQHVYPAQVQYVEGGDAVYTNGAIRTAYTYNPEPQMYAPSSTASYFEAPGGAQVTVAASSPPAVPSHSMVGITMDVGGSPIVSSAGAYLIHGGMDSTRHSLAHTSRSSPATLEMAIENLQKSEGITSHKSGLLNSHLQWLLDNYETAEGVSLPRSSLYNHYLRHCQEHKLDPVNAASFGKLIRSVFMGLRTRRLGTRGNSKYHYYGIRLKPDSPLNRLQEDTQYMAMRQQPMHQKPRYRPAQKTDSLGDSGSHSGLHSTPEQTMAVQSQHHQQYIDVSHVFPEFPAPDLGSFLLQDGVTLHDVKALQLVYRRHCEATVDVVMNLQFHYIEKLWLSFWNSKASSSDGPTSLPASDEDPEGAVLPKDKLISLCQCDPILRWMRSCDHILYQALVEILIPDVLRPVPSTLTQAIRNFAKSLEGWLTNAMSDFPQQVIQTKVGVVSAFAQTLRRYTSLNHLAQAARAVLQNTSQINQMLSDLNRVDFANVQEQASWVCQCEESVVQRLEQDFKLTLQQQSSLDQWASWLDSVVTQVLKQHAGSPSFPKAARQFLLKWSFYSSMVIRDLTLRSAASFGSFHLIRLLYDEYMFYLVEHRVAEATGETPIAVMGEFNDLASLSLTLLDKDDMGDEQRGSEAGPDARSLGEPLVKRERSDPNHSLQGI',\n",
+       " 'trseq12': 'MEVQLGLGRVYPRPPSKTYRGAFQNLFQSVREVIQNPGPRHPEAASAAPPGASLLLLQQQQQQQQQQQQQQQQQQQQQQQETSPRQQQQQQGEDGSPQAHRRGPTGYLVLDEEQQPSQPQSALECHPERGCVPEPGAAVAASKGLPQQLPAPPDEDDSAAPSTLSLLGPTFPGLSSCSADLKDILSEASTMQLLQQQQQEAVSEGSSSGRAREASGAPTSSKDNYLGGTSTISDNAKELCKAVSVSMGLGVEALEHLSPGEQLRGDCMYAPLLGVPPAVRPTPCAPLAECKGSLLDDSAGKSTEDTAEYSPFKGGYTKGLEGESLGCSGSAAAGSSGTLELPSTLSLYKSGALDEAAAYQSRDYYNFPLALAGPPPPPPPPHPHARIKLENPLDYGSAWAAAAAQCRYGDLASLHGAGAAGPGSGSPSAAASSSWHTLFTAEEGQLYGPCGGGGGGGGGGGGGGGGGGGGGGGEAGAVAPYGYTRPPQGLAGQESDFTAPDVWYPGGMVSRVPYPSPTCVKSEMGPWMDSYSGPYGDMRLETARDHVLPIDYYFPPQKTCLICGDEASGCHYGALTCGSCKVFFKRAAEGKQKYLCASRNDCTIDKFRRKNCPSCRLRKCYEAGMTLGARKLKKLGNLKLQEEGEASSTTSPTEETTQKLTVSHIEGYECQPIFLNVLEAIEPGVVCAGHDNNQPDSFAALLSSLNELGERQLVHVVKWAKALPGFRNLHVDDQMAVIQYSWMGLMVFAMGWRSFTNVNSRMLYFAPDLVFNEYRMHKSRMYSQCVRMRHLSQEFGWLQITPQEFLCMKALLLFSIIPVDGLKNQKFFDELRMNYIKELDRIIACKRKNPTSCSRRFYQLTKLLDSVQPIARELHQFTFDLLIKSHMVSVDFPEMMAEIISVQVPKILSGKVKPIYFHTQ',\n",
+       " 'trseq1': 'MAAAKAEMQLMSPLQISDPFGSFPHSPTMDNYPKLEEMMLLSNGAPQFLGAAGAPEGSGSNSSSSSSGGGGGGGGGSNSSSSSSTFNPQADTGEQPYEHLTAESFPDISLNNEKVLVETSYPSQTTRLPPITYTGRFSLEPAPNSGNTLWPEPLFSLVSGLVSMTNPPASSSSAPSPAASSASASQSPPLSCAVPSNDSSPIYSAAPTFPTPNTDIFPEPQSQAFPGSAGTALQYPPPAYPAAKGGFQVPMIPDYLFPQQQGDLGLGTPDQKPFQGLESRTQQPSLTPLSTIKAFATQSGSQDLKALNTSYQSQLIKPSRMRKYPNRPSKTPPHERPYACPVESCDRRFSRSDELTRHIRIHTGQKPFQCRICMRNFSRSDHLTTHIRTHTGEKPFACDICGRKFARSDERKRHTKIHLRQKDKKADKSVVASSATSSLSSYPSPVATSYPSPVTTSYPSPATTSYPSPVPTSFSSPGSSTYPSPVHSGFPSPSVATTYSSVPPAFPAQVSSFPSSAVTNSFSASTGLSDMTATFSPRTIEIC',\n",
+       " 'trseq6': 'MDLPVGPGAAGPSNVPAFLTKLWTLVSDPDTDALICWSPSGNSFHVFDQGQFAKEVLPKYFKHNNMASFVRQLNMYGFRKVVHIEQGGLVKPERDDTEFQHPCFLRGQEQLLENIKRKVTSVSTLKSEDIKIRQDSVTKLLTDVQLMKGKQECMDSKLLAMKHENEALWREVASLRQKHAQQQKVVNKLIQFLISLVQSNRILGVKRKIPLMLNDSGSAHSMPKYSRQFSLEHVHGSGPYSAPSPAYSSSSLYAPDAVASSGPIISDITELAPASPMASPGGSIDERPLSSSPLVRVKEEPPSPPQSPRVEEASPGRPSSVDTLLSPTALIDSILRESEPAPASVTALTDARGHTDTEGRPPSPPPTSTPEKCLSVACLDKNELSDHLDAMDSNLDNLQTMLSSHGFSVDTSALLDLFSPSVTVPDMSLPDLDSSLASIQELLSPQEPPRPPEAENSSPDSGKQLVHYTAQPLFLLDPGSVDTGSNDLPVLFELGEGSYFSEGDGFAEDPTISLLTGSEPPKAKDPTVS',\n",
+       " 'trseq24': 'MSDQDHSMDEMTAVVKIEKGVGGNNGGNGNGGGAFSQARSSSTGSSSSTGGGGQESQPSPLALLAATCSRIESPNENSNNSQGPSQSGGTGELDLTATQLSQGANGWQIISSSSGATPTSKEQSGSSTNGSNGSESSKNRTVSGGQYVVAAAPNLQNQQVLTGLPGVMPNIQYQVIPQFQTVDGQQLQFAATGAQVQQDGSGQIQIIPGANQQIITNRGSGGNIIAAMPNLLQQAVPLQGLANNVLSGQTQYVTNVPVALNGNITLLPVNSVSAATLTPSSQAVTISSSGSQESGSQPVTSGTTISSASLVSSQASSSSFFTNANSYSTTTTTSNMGIMNFTTSGSSGTNSQGQTPQRVSGLQGSDALNIQQNQTSGGSLQAGQQKEGEQNQQTQQQQILIQPQLVQGGQALQALQAAPLSGQTFTTQAISQETLQNLQLQAVPNSGPIIIRTPTVGPNGQVSWQTLQLQNLQVQNPQAQTITLAPMQGVSLGQTSSSNTTLTPIASAASIPAGTVTVNAAQLSSMPGLQTINLSALGTSGIQVHPIQGLPLAIANAPGDHGAQLGLHGAGGDGIHDDTAGGEEGENSPDAQPQAGRRTRREACTCPYCKDSEGRGSGDPGKKKQHICHIQGCGKVYGKTSHLRAHLRWHTGERPFMCTWSYCGKRFTRSDELQRHKRTHTGEKKFACPECPKRFMRSDHLSKHIKTHQNKKGGPGVALSVGTLPLDSGAGSEGSGTATPSALITTNMVAMEAICPEGIARLANSGINVMQVADLQSINISGNGF',\n",
+       " 'trseq9': 'MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTDGGEVVQDVNSSVQMVMMEQLDPTLLQMKTEVMEGTVAPEAEAAVDDTQIITLQVVNMEEQPINIGELQLVQVPVPVTVPVATTSVEELQGAYENEVSKEGLAESEPMICHTLPLPEGFQVVKVGANGEVETLEQGELPPQEDPSWQKDPDYQPPAKKTKKTKKSKLRYTEEGKDVDVSVYDFEEEQQEGLLSEVNAEKVVGNMKPPKPTKIKKKGVKKTFQCELCSYTCPRRSNLDRHMKSHTDERPHKCHLCGRAFRTVTLLRNHLNTHTGTRPHKCPDCDMAFVTSGELVRHRRYKHTHEKPFKCSMCDYASVEVSKLKRHIRSHTGERPFQCSLCSYASRDTYKLKRHMRTHSGEKPYECYICHARFTQSGTMKMHILQKHTENVAKFHCPHCDTVIARKSDLGVHLRKQHSYIEQGKKCRYCDAVFHERYALIQHQKSHKNEKRFKCDQCDYACRQERHMIMHKRTHTGEKPYACSHCDKTFRQKQLLDMHFKRYHDPNFVPAAFVCSKCGKTFTRRNTMARHADNCAGPDGVEGENGGETKKSKRGRKRKMRSKKEDSSDSENAEPDLDDNEDEEEPAVEIEPEPEPQPVTPAPPPAKKRRGRPPGRTNQPKQNQPTAIIQVEDQNTGAIENIIVEVKKEPDAEPAEGEEEEAQPAATDAPNGDLTPEMILSMMDR',\n",
+       " 'trseq14': 'MLPTQAGAAAALGRGSALGGSLNRTPTGRPGGGGGTRGANGGRVPGNGAGLGPGRLEREAAAAAATTPAPTAGALYSGSEGDSESGEEEELGAERRGLKRSLSEMEIGMVVGGPEASAAATGGYGPVSGAVSGAKPGKKTRGRVKIKMEFIDNKLRRYTTFSKRKTGIMKKAYELSTLTGTQVLLLVASETGHVYTFATRKLQPMITSETGKALIQTCLNSPDSPPRSDPTTDQRMSATGFEETDLTYQVSESDSSGETKDTLKPAFTVTNLPGTTSTIQTAPSTSTTMQVSSGPSFPITNYLAPVSASVSPSAVSSANGTVLKSTGSGPVSSGGLMQLPTSFTLMPGGAVAQQVPVQAIQVHQAPQQASPSRDSSTDLTQTSSSGTVTLPATIMTSSVPTTVGGHMMYPSPHAVMYAPTSGLGDGSLTVLNAFSQAPSTMQVSHSQVQEPGGVPQVFLTASSGTVQIPVSAVQLHQMAVIGQQAGSSSNLTELQVVNLDTAHSTKSE'}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "with open(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/maps/tr_seqid_to_tr_sequence.json\", \"r\") as f:\n",
+    "    import json\n",
+    "    tr_map = json.load(f)\n",
+    "\n",
+    "tr_map"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "9ad16929",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>tr_seqid</th>\n",
+       "      <th>dna_seqid</th>\n",
+       "      <th>peak_seqid</th>\n",
+       "      <th>chrpeak_id</th>\n",
+       "      <th>tr_name</th>\n",
+       "      <th>chipscore</th>\n",
+       "      <th>total_jaspar_hits</th>\n",
+       "      <th>dna_sequence</th>\n",
+       "      <th>tr_sequence</th>\n",
+       "      <th>scores</th>\n",
+       "      <th>tr_cluster_rep</th>\n",
+       "      <th>dna_cluster_rep</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>trseq26_dnaseq49877</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49877</td>\n",
+       "      <td>peakseq24686</td>\n",
+       "      <td>chr12_peak1150</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq58018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>trseq26_dnaseq75052</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq75052</td>\n",
+       "      <td>peakseq12129</td>\n",
+       "      <td>chr9_peak512</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq75052</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>trseq26_dnaseq14843</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq14843</td>\n",
+       "      <td>peakseq12863</td>\n",
+       "      <td>chr1_peak1335</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq14843</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>trseq26_dnaseq39522</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq39522</td>\n",
+       "      <td>peakseq5250</td>\n",
+       "      <td>chr5_peak280</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq3280</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>trseq26_dnaseq49215</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq49215</td>\n",
+       "      <td>peakseq4451</td>\n",
+       "      <td>chr4_peak201</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>138.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>dnaseq57257</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71152</th>\n",
+       "      <td>trseq9_dnaseq4066</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq4066</td>\n",
+       "      <td>peakseq12188</td>\n",
+       "      <td>chr18_peak213</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AATAATATATCTATTTCTTTATCTTTGTCTTCCCTACTGGACTAGC...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq48336</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71153</th>\n",
+       "      <td>trseq9_dnaseq62118</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq62118</td>\n",
+       "      <td>peakseq46829</td>\n",
+       "      <td>chr18_peak812</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>TAAATATGTATTTTAGTAAAGTGTTATGATACACTGTGATGGGGGT...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq62118</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71154</th>\n",
+       "      <td>trseq9_dnaseq41538</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq41538</td>\n",
+       "      <td>peakseq3957</td>\n",
+       "      <td>chrY_peak4</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>267.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GACAGGAGTTGTGTACGAATGTGTGTGAATGTGGGAGCCTAACTAG...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq41538</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71155</th>\n",
+       "      <td>trseq9_dnaseq38134</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq38134</td>\n",
+       "      <td>peakseq40502</td>\n",
+       "      <td>chr5_peak1955</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>CTGGGCGGGTAGGTGAGAGGACAGGAGGGCGAAGTGGAGAGGAGGG...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq73435</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>71156</th>\n",
+       "      <td>trseq9_dnaseq5106</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq5106</td>\n",
+       "      <td>peakseq37888</td>\n",
+       "      <td>chr1_peak4019</td>\n",
+       "      <td>CTCF</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>ACAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAA...</td>\n",
+       "      <td>MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>dnaseq5106</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>71157 rows × 13 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        ID tr_seqid    dna_seqid    peak_seqid  \\\n",
+       "0      trseq26_dnaseq49877  trseq26  dnaseq49877  peakseq24686   \n",
+       "1      trseq26_dnaseq75052  trseq26  dnaseq75052  peakseq12129   \n",
+       "2      trseq26_dnaseq14843  trseq26  dnaseq14843  peakseq12863   \n",
+       "3      trseq26_dnaseq39522  trseq26  dnaseq39522   peakseq5250   \n",
+       "4      trseq26_dnaseq49215  trseq26  dnaseq49215   peakseq4451   \n",
+       "...                    ...      ...          ...           ...   \n",
+       "71152    trseq9_dnaseq4066   trseq9   dnaseq4066  peakseq12188   \n",
+       "71153   trseq9_dnaseq62118   trseq9  dnaseq62118  peakseq46829   \n",
+       "71154   trseq9_dnaseq41538   trseq9  dnaseq41538   peakseq3957   \n",
+       "71155   trseq9_dnaseq38134   trseq9  dnaseq38134  peakseq40502   \n",
+       "71156    trseq9_dnaseq5106   trseq9   dnaseq5106  peakseq37888   \n",
+       "\n",
+       "           chrpeak_id tr_name  chipscore  total_jaspar_hits  \\\n",
+       "0      chr12_peak1150    NFYB        6.0                  1   \n",
+       "1        chr9_peak512    NFYB        5.0                  1   \n",
+       "2       chr1_peak1335    NFYB        6.0                  3   \n",
+       "3        chr5_peak280    NFYB        4.0                  1   \n",
+       "4        chr4_peak201    NFYB      138.0                  2   \n",
+       "...               ...     ...        ...                ...   \n",
+       "71152   chr18_peak213    CTCF     1000.0                  1   \n",
+       "71153   chr18_peak812    CTCF     1000.0                  1   \n",
+       "71154      chrY_peak4    CTCF      267.0                  1   \n",
+       "71155   chr5_peak1955    CTCF     1000.0                  2   \n",
+       "71156   chr1_peak4019    CTCF       14.0                  2   \n",
+       "\n",
+       "                                            dna_sequence  \\\n",
+       "0      GCTCTTAAAGATGGTGTGTCCAGAGTTTGTTCCTTCAGATGTTCAG...   \n",
+       "1      TGTTGGTCTCGCTGACCTCAAGAACGGAGCCGTGGACCCTCGCGGT...   \n",
+       "2      AGTTTGGGTACTCAAATATGGTACCAGCAACCAGATGGTGAGTTGC...   \n",
+       "3      CTTGGAGAACCTTTATGTCTAGCTAAGGGATTGTAAATACACCAAT...   \n",
+       "4      GCGGTGACTGTTACAGTTCTTAAAGGCGGCGTGTCTGGAGTTTGTT...   \n",
+       "...                                                  ...   \n",
+       "71152  AATAATATATCTATTTCTTTATCTTTGTCTTCCCTACTGGACTAGC...   \n",
+       "71153  TAAATATGTATTTTAGTAAAGTGTTATGATACACTGTGATGGGGGT...   \n",
+       "71154  GACAGGAGTTGTGTACGAATGTGTGTGAATGTGGGAGCCTAACTAG...   \n",
+       "71155  CTGGGCGGGTAGGTGAGAGGACAGGAGGGCGAAGTGGAGAGGAGGG...   \n",
+       "71156  ACAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAA...   \n",
+       "\n",
+       "                                             tr_sequence  \\\n",
+       "0      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "1      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "2      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "3      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "4      MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "...                                                  ...   \n",
+       "71152  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71153  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71154  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71155  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "71156  MEGDAVEAIVEESETFIKGKERKTYQRRREGGQEEDACHLPQNQTD...   \n",
+       "\n",
+       "                                                  scores tr_cluster_rep  \\\n",
+       "0      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "1      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "2      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "3      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "4      0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...        trseq26   \n",
+       "...                                                  ...            ...   \n",
+       "71152  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq9   \n",
+       "71153  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq9   \n",
+       "71154  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq9   \n",
+       "71155  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq9   \n",
+       "71156  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq9   \n",
+       "\n",
+       "      dna_cluster_rep  \n",
+       "0         dnaseq58018  \n",
+       "1         dnaseq75052  \n",
+       "2         dnaseq14843  \n",
+       "3          dnaseq3280  \n",
+       "4         dnaseq57257  \n",
+       "...               ...  \n",
+       "71152     dnaseq48336  \n",
+       "71153     dnaseq62118  \n",
+       "71154     dnaseq41538  \n",
+       "71155     dnaseq73435  \n",
+       "71156      dnaseq5106  \n",
+       "\n",
+       "[71157 rows x 13 columns]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa60c388",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>tr_cluster_rep</th>\n",
+       "      <th>count</th>\n",
+       "      <th>Protein names</th>\n",
+       "      <th>Protein families</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>46445</td>\n",
+       "      <td>Zinc finger protein 263 (Zinc finger protein F...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>trseq1</td>\n",
+       "      <td>8962</td>\n",
+       "      <td>Early growth response protein 1 (EGR-1) (AT225...</td>\n",
+       "      <td>EGR C2H2-type zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>trseq23</td>\n",
+       "      <td>6407</td>\n",
+       "      <td>Transcription factor Sp2</td>\n",
+       "      <td>Sp1 C2H2-type zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>trseq9</td>\n",
+       "      <td>2641</td>\n",
+       "      <td>Transcriptional repressor CTCF (11-zinc finger...</td>\n",
+       "      <td>CTCF zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>trseq5</td>\n",
+       "      <td>1839</td>\n",
+       "      <td>RE1-silencing transcription factor (Neural-res...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>trseq19</td>\n",
+       "      <td>1820</td>\n",
+       "      <td>Interferon regulatory factor 1 (IRF-1)</td>\n",
+       "      <td>IRF family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>873</td>\n",
+       "      <td>Nuclear transcription factor Y subunit beta (C...</td>\n",
+       "      <td>NFYB/HAP3 subunit family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>trseq24</td>\n",
+       "      <td>873</td>\n",
+       "      <td>Transcription factor Sp1</td>\n",
+       "      <td>Sp1 C2H2-type zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>trseq10</td>\n",
+       "      <td>528</td>\n",
+       "      <td>Nuclear transcription factor Y subunit alpha (...</td>\n",
+       "      <td>NFYA/HAP2 subunit family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>trseq16</td>\n",
+       "      <td>487</td>\n",
+       "      <td>Forkhead box protein P1 (Mac-1-regulated forkh...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>trseq29</td>\n",
+       "      <td>71</td>\n",
+       "      <td>Ras-responsive element-binding protein 1 (RREB...</td>\n",
+       "      <td>Krueppel C2H2-type zinc-finger protein family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>trseq14</td>\n",
+       "      <td>62</td>\n",
+       "      <td>Serum response factor (SRF)</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>trseq21</td>\n",
+       "      <td>34</td>\n",
+       "      <td>DNA-binding protein RFX2 (Regulatory factor X 2)</td>\n",
+       "      <td>RFX family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>trseq17</td>\n",
+       "      <td>30</td>\n",
+       "      <td>Tumor protein 63 (p63) (Chronic ulcerative sto...</td>\n",
+       "      <td>P53 family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>trseq28</td>\n",
+       "      <td>29</td>\n",
+       "      <td>Nuclear receptor subfamily 2 group C member 2 ...</td>\n",
+       "      <td>Nuclear hormone receptor family, NR2 subfamily</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>trseq22</td>\n",
+       "      <td>16</td>\n",
+       "      <td>Transcription factor E2F3 (E2F-3)</td>\n",
+       "      <td>E2F/DP family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>trseq15</td>\n",
+       "      <td>9</td>\n",
+       "      <td>Transcription factor AP-2 gamma (AP2-gamma) (A...</td>\n",
+       "      <td>AP-2 family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>trseq2</td>\n",
+       "      <td>9</td>\n",
+       "      <td>ETS-related transcription factor Elf-1 (E74-li...</td>\n",
+       "      <td>ETS family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>trseq20</td>\n",
+       "      <td>3</td>\n",
+       "      <td>Interferon regulatory factor 2 (IRF-2)</td>\n",
+       "      <td>IRF family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>trseq8</td>\n",
+       "      <td>3</td>\n",
+       "      <td>Cellular tumor antigen p53 (Antigen NY-CO-13) ...</td>\n",
+       "      <td>P53 family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>trseq27</td>\n",
+       "      <td>3</td>\n",
+       "      <td>Estrogen receptor (ER) (ER-alpha) (Estradiol r...</td>\n",
+       "      <td>Nuclear hormone receptor family, NR3 subfamily</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>trseq11</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Transcriptional regulator Kaiso (Zinc finger a...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>trseq25</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Transcription factor MafF (U-Maf) (V-maf muscu...</td>\n",
+       "      <td>BZIP family, Maf subfamily</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>trseq6</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Heat shock factor protein 1 (HSF 1) (Heat shoc...</td>\n",
+       "      <td>HSF family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>trseq3</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Transcriptional repressor protein YY1 (Delta t...</td>\n",
+       "      <td>YY transcription factor family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>trseq18</td>\n",
+       "      <td>2</td>\n",
+       "      <td>Hepatocyte nuclear factor 4-gamma (HNF-4-gamma...</td>\n",
+       "      <td>Nuclear hormone receptor family, NR2 subfamily</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>trseq13</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Myocyte-specific enhancer factor 2A (Serum res...</td>\n",
+       "      <td>MEF2 family</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>trseq7</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Glucocorticoid receptor (GR) (Nuclear receptor...</td>\n",
+       "      <td>Nuclear hormone receptor family, NR3 subfamily</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>trseq12</td>\n",
+       "      <td>1</td>\n",
+       "      <td>Androgen receptor (Dihydrotestosterone recepto...</td>\n",
+       "      <td>Nuclear hormone receptor family, NR3 subfamily</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   tr_cluster_rep  count                                      Protein names  \\\n",
+       "0          trseq4  46445  Zinc finger protein 263 (Zinc finger protein F...   \n",
+       "1          trseq1   8962  Early growth response protein 1 (EGR-1) (AT225...   \n",
+       "2         trseq23   6407                           Transcription factor Sp2   \n",
+       "3          trseq9   2641  Transcriptional repressor CTCF (11-zinc finger...   \n",
+       "4          trseq5   1839  RE1-silencing transcription factor (Neural-res...   \n",
+       "5         trseq19   1820             Interferon regulatory factor 1 (IRF-1)   \n",
+       "6         trseq26    873  Nuclear transcription factor Y subunit beta (C...   \n",
+       "7         trseq24    873                           Transcription factor Sp1   \n",
+       "8         trseq10    528  Nuclear transcription factor Y subunit alpha (...   \n",
+       "9         trseq16    487  Forkhead box protein P1 (Mac-1-regulated forkh...   \n",
+       "10        trseq29     71  Ras-responsive element-binding protein 1 (RREB...   \n",
+       "11        trseq14     62                        Serum response factor (SRF)   \n",
+       "12        trseq21     34   DNA-binding protein RFX2 (Regulatory factor X 2)   \n",
+       "13        trseq17     30  Tumor protein 63 (p63) (Chronic ulcerative sto...   \n",
+       "14        trseq28     29  Nuclear receptor subfamily 2 group C member 2 ...   \n",
+       "15        trseq22     16                  Transcription factor E2F3 (E2F-3)   \n",
+       "16        trseq15      9  Transcription factor AP-2 gamma (AP2-gamma) (A...   \n",
+       "17         trseq2      9  ETS-related transcription factor Elf-1 (E74-li...   \n",
+       "18        trseq20      3             Interferon regulatory factor 2 (IRF-2)   \n",
+       "19         trseq8      3  Cellular tumor antigen p53 (Antigen NY-CO-13) ...   \n",
+       "20        trseq27      3  Estrogen receptor (ER) (ER-alpha) (Estradiol r...   \n",
+       "21        trseq11      2  Transcriptional regulator Kaiso (Zinc finger a...   \n",
+       "22        trseq25      2  Transcription factor MafF (U-Maf) (V-maf muscu...   \n",
+       "23         trseq6      2  Heat shock factor protein 1 (HSF 1) (Heat shoc...   \n",
+       "24         trseq3      2  Transcriptional repressor protein YY1 (Delta t...   \n",
+       "25        trseq18      2  Hepatocyte nuclear factor 4-gamma (HNF-4-gamma...   \n",
+       "26        trseq13      1  Myocyte-specific enhancer factor 2A (Serum res...   \n",
+       "27         trseq7      1  Glucocorticoid receptor (GR) (Nuclear receptor...   \n",
+       "28        trseq12      1  Androgen receptor (Dihydrotestosterone recepto...   \n",
+       "\n",
+       "                                  Protein families  \n",
+       "0    Krueppel C2H2-type zinc-finger protein family  \n",
+       "1         EGR C2H2-type zinc-finger protein family  \n",
+       "2         Sp1 C2H2-type zinc-finger protein family  \n",
+       "3                  CTCF zinc-finger protein family  \n",
+       "4                                              NaN  \n",
+       "5                                       IRF family  \n",
+       "6                         NFYB/HAP3 subunit family  \n",
+       "7         Sp1 C2H2-type zinc-finger protein family  \n",
+       "8                         NFYA/HAP2 subunit family  \n",
+       "9                                              NaN  \n",
+       "10   Krueppel C2H2-type zinc-finger protein family  \n",
+       "11                                             NaN  \n",
+       "12                                      RFX family  \n",
+       "13                                      P53 family  \n",
+       "14  Nuclear hormone receptor family, NR2 subfamily  \n",
+       "15                                   E2F/DP family  \n",
+       "16                                     AP-2 family  \n",
+       "17                                      ETS family  \n",
+       "18                                      IRF family  \n",
+       "19                                      P53 family  \n",
+       "20  Nuclear hormone receptor family, NR3 subfamily  \n",
+       "21                                             NaN  \n",
+       "22                      BZIP family, Maf subfamily  \n",
+       "23                                      HSF family  \n",
+       "24                  YY transcription factor family  \n",
+       "25  Nuclear hormone receptor family, NR2 subfamily  \n",
+       "26                                     MEF2 family  \n",
+       "27  Nuclear hormone receptor family, NR3 subfamily  \n",
+       "28  Nuclear hormone receptor family, NR3 subfamily  "
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vcs = all_data[\"tr_cluster_rep\"].value_counts().reset_index()\n",
+    "vcs[\"sequence\"] = vcs[\"tr_cluster_rep\"].map(tr_map)\n",
+    "vcs = pd.merge(vcs, idmap.rename(columns={\"Sequence\":\"sequence\"}),on=\"sequence\")\n",
+    "vcs[[\"tr_cluster_rep\",\"count\",\"Protein names\",\"Protein families\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "3628496c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "29\n",
+      "71134\n",
+      "0\n",
+      "0\n",
+      "Rows in test: 7310\n",
+      "Rows to be split between train and val: 59068\n",
+      "Total rows: 66378. Test percentage: 11.01%\n",
+      "Lost rows: 4779\n",
+      "Rows in val: 4462\n",
+      "Rows left for train: 54002\n",
+      "Total rows: 58464. Test percentage: 7.63%\n",
+      "Lost rows: 5383\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Full pipeline\n",
+    "import pandas as pd\n",
+    "protein_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/protein/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "protein_clusters.columns=[\"tr_cluster_rep\",\"tr_cluster_member\"]\n",
+    "protein_clusters.head()\n",
+    "\n",
+    "dna_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/dna_full/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "dna_clusters.columns=[\"dna_cluster_rep\",\"dna_cluster_member\"]\n",
+    "dna_clusters.head()\n",
+    "\n",
+    "all_data = pd.read_parquet(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/remap2022_crm_fimo_output_q_processed_seed0.parquet\")\n",
+    "all_data\n",
+    "\n",
+    "protein_cluster_map = dict(zip(protein_clusters[\"tr_cluster_member\"],protein_clusters[\"tr_cluster_rep\"]))\n",
+    "dna_cluster_map = dict(zip(dna_clusters[\"dna_cluster_member\"],dna_clusters[\"dna_cluster_rep\"]))\n",
+    "print(len(protein_cluster_map))\n",
+    "print(len(dna_cluster_map))\n",
+    "all_data[\"tr_cluster_rep\"] = all_data[\"tr_seqid\"].map(protein_cluster_map)\n",
+    "all_data[\"dna_cluster_rep\"] = all_data[\"dna_seqid\"].map(dna_cluster_map)\n",
+    "print(len(all_data[all_data[\"tr_cluster_rep\"].isna()]))\n",
+    "print(len(all_data[all_data[\"dna_cluster_rep\"].isna()]))\n",
+    "all_data.head()\n",
+    "\n",
+    "\n",
+    "### handpick test\n",
+    "handpicked_test_trs = [\"trseq23\",\"trseq26\",\"trseq17\"]\n",
+    "handpicked_test = all_data.loc[\n",
+    "    all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_test[\"dna_cluster_rep\"].unique().tolist()\n",
+    "remaining = all_data.loc[\n",
+    "    (~all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)) & \n",
+    "    (~all_data[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "test_ids = handpicked_test[\"ID\"].unique().tolist()\n",
+    "remaining_ids = remaining[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(remaining_ids))\n",
+    "]\n",
+    "print(f\"Rows in test: {len(handpicked_test)}\")\n",
+    "print(f\"Rows to be split between train and val: {len(remaining)}\")\n",
+    "total_rows = len(handpicked_test) + len(remaining)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_test)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")\n",
+    "\n",
+    "### handpick val\n",
+    "handpicked_val_trs = [\"trseq9\", \"trseq5\", \"trseq28\"]\n",
+    "\n",
+    "handpicked_val = remaining.loc[\n",
+    "    remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_val[\"dna_cluster_rep\"].unique().tolist()\n",
+    "train_remain = remaining.loc[\n",
+    "    (~remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)) & \n",
+    "    (~remaining[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "val_ids = handpicked_val[\"ID\"].unique().tolist()\n",
+    "train_remain_ids = train_remain[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(val_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(train_remain_ids))\n",
+    "]\n",
+    "print(f\"Rows in val: {len(handpicked_val)}\")\n",
+    "print(f\"Rows left for train: {len(train_remain)}\")\n",
+    "total_rows = len(handpicked_val) + len(train_remain)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_val)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d92891a1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "29\n",
+      "71134\n",
+      "0\n",
+      "0\n",
+      "Rows in test: 7310\n",
+      "Rows to be split between train and val: 59068\n",
+      "Total rows: 66378. Test percentage: 11.01%\n",
+      "Lost rows: 4779\n",
+      "Rows in val: 4462\n",
+      "Rows left for train: 54002\n",
+      "Total rows: 58464. Test percentage: 7.63%\n",
+      "Lost rows: 5383\n",
+      "Original total: 71157\n",
+      "New, exclusive total: 65774\n",
+      "Lost rows: 5383\n",
+      "Length train: 54002/65774 (82.10%)\n",
+      "Length val: 4462/65774 (6.78%)\n",
+      "Length test: 7310/65774 (11.11%)\n",
+      "Pass! No overlap in IDs\n",
+      "Train-Val TR intersection: 0\n",
+      "Train-Test TR intersection: 0\n",
+      "Val-Test TR intersection: 0\n",
+      "Train-Val TR Cluster Rep intersection: 0\n",
+      "Train-Test TR Cluster Rep intersection: 0\n",
+      "Val-Test TR Cluster Rep intersection: 0\n",
+      "Train-Val DNA intersection: 0\n",
+      "Train-Test DNA intersection: 0\n",
+      "Val-Test DNA intersection: 0\n",
+      "Train-Val DNA Cluster Rep intersection: 0\n",
+      "Train-Test DNA Cluster Rep intersection: 0\n",
+      "Val-Test DNA Cluster Rep intersection: 0\n",
+      "Added reverse complement sequences to train_exclusive, val_exclusive, and test_exclusive (and leaky test_exclusive)\n",
+      "Pass! No overlap in IDs\n",
+      "Train-Val TR intersection: 0\n",
+      "Train-Test TR intersection: 0\n",
+      "Val-Test TR intersection: 0\n",
+      "Train-Val TR Cluster Rep intersection: 0\n",
+      "Train-Test TR Cluster Rep intersection: 0\n",
+      "Val-Test TR Cluster Rep intersection: 0\n",
+      "Train-Val DNA intersection: 0\n",
+      "Train-Test DNA intersection: 0\n",
+      "Val-Test DNA intersection: 0\n",
+      "Train-Val DNA Cluster Rep intersection: 0\n",
+      "Train-Test DNA Cluster Rep intersection: 0\n",
+      "Val-Test DNA Cluster Rep intersection: 0\n",
+      "Length of train_exclusive dataset: 108004 (75.89%)\n",
+      "Length of val_exclusive dataset: 8924 (6.27%)\n",
+      "Length of test_exclusive dataset: 14620 (10.27%)\n",
+      "Length of leaky_test dataset: 10766 (7.56%)\n",
+      "Total sequences = 142314. Same as edges size*2? True\n",
+      "Saved all splits to /home/a03-svincoff/DPACMAN/dpacman/data_files/processed/splits/handpicked_val_test\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Full pipeline\n",
+    "import pandas as pd\n",
+    "protein_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/protein/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "protein_clusters.columns=[\"tr_cluster_rep\",\"tr_cluster_member\"]\n",
+    "protein_clusters.head()\n",
+    "\n",
+    "dna_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/dna_full/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "dna_clusters.columns=[\"dna_cluster_rep\",\"dna_cluster_member\"]\n",
+    "dna_clusters.head()\n",
+    "\n",
+    "all_data = pd.read_parquet(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/remap2022_crm_fimo_output_q_processed_seed0.parquet\")\n",
+    "all_data\n",
+    "\n",
+    "protein_cluster_map = dict(zip(protein_clusters[\"tr_cluster_member\"],protein_clusters[\"tr_cluster_rep\"]))\n",
+    "dna_cluster_map = dict(zip(dna_clusters[\"dna_cluster_member\"],dna_clusters[\"dna_cluster_rep\"]))\n",
+    "print(len(protein_cluster_map))\n",
+    "print(len(dna_cluster_map))\n",
+    "all_data[\"tr_cluster_rep\"] = all_data[\"tr_seqid\"].map(protein_cluster_map)\n",
+    "all_data[\"dna_cluster_rep\"] = all_data[\"dna_seqid\"].map(dna_cluster_map)\n",
+    "print(len(all_data[all_data[\"tr_cluster_rep\"].isna()]))\n",
+    "print(len(all_data[all_data[\"dna_cluster_rep\"].isna()]))\n",
+    "all_data.head()\n",
+    "\n",
+    "\n",
+    "### handpick test\n",
+    "handpicked_test_trs = [\"trseq23\",\"trseq26\",\"trseq17\"]\n",
+    "handpicked_test = all_data.loc[\n",
+    "    all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_test[\"dna_cluster_rep\"].unique().tolist()\n",
+    "remaining = all_data.loc[\n",
+    "    (~all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)) & \n",
+    "    (~all_data[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "test_ids = handpicked_test[\"ID\"].unique().tolist()\n",
+    "remaining_ids = remaining[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(remaining_ids))\n",
+    "]\n",
+    "print(f\"Rows in test: {len(handpicked_test)}\")\n",
+    "print(f\"Rows to be split between train and val: {len(remaining)}\")\n",
+    "total_rows = len(handpicked_test) + len(remaining)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_test)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")\n",
+    "\n",
+    "### handpick val\n",
+    "handpicked_val_trs = [\"trseq9\", \"trseq5\", \"trseq28\"]\n",
+    "\n",
+    "handpicked_val = remaining.loc[\n",
+    "    remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_val[\"dna_cluster_rep\"].unique().tolist()\n",
+    "train_remain = remaining.loc[\n",
+    "    (~remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)) & \n",
+    "    (~remaining[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "val_ids = handpicked_val[\"ID\"].unique().tolist()\n",
+    "train_remain_ids = train_remain[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(val_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(train_remain_ids))\n",
+    "]\n",
+    "print(f\"Rows in val: {len(handpicked_val)}\")\n",
+    "print(f\"Rows left for train: {len(train_remain)}\")\n",
+    "total_rows = len(handpicked_val) + len(train_remain)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_val)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")\n",
+    "\n",
+    "train_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(train_remain_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "val_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(val_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "test_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(test_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "leaky_test = all_data.loc[\n",
+    "    ~(all_data[\"ID\"].isin(train_exclusive[\"ID\"].tolist())) & \n",
+    "    ~(all_data[\"ID\"].isin(val_exclusive[\"ID\"].tolist())) & \n",
+    "    ~(all_data[\"ID\"].isin(test_exclusive[\"ID\"].tolist()))\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "print(f\"Original total: {len(all_data)}\")\n",
+    "retained_total = len(train_exclusive)+len(val_exclusive)+len(test_exclusive)\n",
+    "print(f\"New, exclusive total: {retained_total}\")\n",
+    "print(f\"Lost rows: {len(all_data)-retained_total}\")\n",
+    "print(f\"Length train: {len(train_exclusive)}/{retained_total} ({100*len(train_exclusive)/retained_total:.2f}%)\")\n",
+    "print(f\"Length val: {len(val_exclusive)}/{retained_total} ({100*len(val_exclusive)/retained_total:.2f}%)\")\n",
+    "print(f\"Length test: {len(test_exclusive)}/{retained_total} ({100*len(test_exclusive)/retained_total:.2f}%)\")\n",
+    "\n",
+    "def check_validity(train_exclusive, val_exclusive, test_exclusive):\n",
+    "    train_exclusive_ids = set(train_exclusive[\"ID\"].unique().tolist())\n",
+    "    val_exclusive_ids = set(val_exclusive[\"ID\"].unique().tolist())\n",
+    "    test_exclusive_ids = set(test_exclusive[\"ID\"].unique().tolist())\n",
+    "\n",
+    "    assert len(train_exclusive_ids.intersection(val_exclusive_ids)) == 0\n",
+    "    assert len(train_exclusive_ids.intersection(test_exclusive_ids)) == 0\n",
+    "    assert len(val_exclusive_ids.intersection(test_exclusive_ids)) == 0\n",
+    "    print(f\"Pass! No overlap in IDs\")\n",
+    "\n",
+    "    # Investigate TR intersection. No assertions unless we are explicitly splitting on this. \n",
+    "    train_exclusive_tr_seqs = set(train_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "    val_exclusive_tr_seqs = set(val_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "    test_exclusive_tr_seqs = set(test_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "\n",
+    "    train_exclusive_tr_reps = set(train_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "    val_exclusive_tr_reps = set(val_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "    test_exclusive_tr_reps = set(test_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "\n",
+    "    print(f\"Train-Val TR intersection: {len(train_exclusive_tr_seqs.intersection(val_exclusive_tr_seqs))}\")\n",
+    "    print(f\"Train-Test TR intersection: {len(train_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}\")\n",
+    "    print(f\"Val-Test TR intersection: {len(val_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}\")\n",
+    "\n",
+    "    print(f\"Train-Val TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(val_exclusive_tr_reps))}\")\n",
+    "    print(f\"Train-Test TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}\")\n",
+    "    print(f\"Val-Test TR Cluster Rep intersection: {len(val_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}\")\n",
+    "\n",
+    "    # Investigate DNA intersection. No assertions unless we are explicitly splitting on this. \n",
+    "    train_exclusive_dna_seqs = set(train_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "    val_exclusive_dna_seqs = set(val_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "    test_exclusive_dna_seqs = set(test_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "\n",
+    "    train_exclusive_dna_reps = set(train_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "    val_exclusive_dna_reps = set(val_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "    test_exclusive_dna_reps = set(test_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "\n",
+    "    print(f\"Train-Val DNA intersection: {len(train_exclusive_dna_seqs.intersection(val_exclusive_dna_seqs))}\")\n",
+    "    print(f\"Train-Test DNA intersection: {len(train_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}\")\n",
+    "    print(f\"Val-Test DNA intersection: {len(val_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}\")\n",
+    "\n",
+    "    print(f\"Train-Val DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(val_exclusive_dna_reps))}\")\n",
+    "    print(f\"Train-Test DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}\")\n",
+    "    print(f\"Val-Test DNA Cluster Rep intersection: {len(val_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}\")\n",
+    "\n",
+    "def get_reverse_complement(s):\n",
+    "    \"\"\"\n",
+    "    Returns 5' to 3' sequence of the reverse complement\n",
+    "    \"\"\"\n",
+    "    chars = list(s)\n",
+    "    recon = []\n",
+    "    rev_map = {\n",
+    "        \"a\": \"t\",\n",
+    "        \"c\": \"g\",\n",
+    "        \"t\": \"a\",\n",
+    "        \"g\": \"c\",\n",
+    "        \"A\": \"T\",\n",
+    "        \"C\": \"G\",\n",
+    "        \"T\": \"A\",\n",
+    "        \"G\": \"C\",\n",
+    "        \"n\": \"n\",\n",
+    "        \"N\": \"N\",\n",
+    "    }\n",
+    "    for c in chars:\n",
+    "        recon += [rev_map[c]]\n",
+    "\n",
+    "    recon = \"\".join(recon)\n",
+    "    return recon[::-1]\n",
+    "\n",
+    "# now make reverse complements\n",
+    "def augment_rc(df):\n",
+    "    \"\"\"\n",
+    "    Get the reverse complement and add it as a datapoint, effectively doubling the dataset.\n",
+    "    Also flip the orientation of the scores\n",
+    "\n",
+    "    columns = [\"ID\",\"dna_sequence\",\"tr_sequence\",\"tr_cluster_rep\",\"dna_cluster_rep\", \"scores\",\"split\"]\n",
+    "    \"\"\"\n",
+    "    df_rc = df.copy(deep=True)\n",
+    "\n",
+    "    df_rc[\"dna_sequence\"] = df_rc[\"dna_sequence\"].apply(\n",
+    "        lambda x: get_reverse_complement(x)\n",
+    "    )\n",
+    "    df_rc[\"ID\"] = df_rc[\"ID\"] + \"_rc\"\n",
+    "    df_rc[\"scores\"] = df_rc[\"scores\"].apply(lambda s: \",\".join(s.split(\",\")[::-1]))\n",
+    "\n",
+    "    final_df = pd.concat([df, df_rc]).reset_index(drop=True)\n",
+    "\n",
+    "    return final_df\n",
+    "\n",
+    "def convert_scores(scores, mode=1):\n",
+    "    \"\"\"\n",
+    "    Two modes: 1 means FIMO peaks get 1. 0 means FIMO peaks get their max score\n",
+    "    \"\"\"\n",
+    "    svec = [int(x) for x in scores.split(\",\")]\n",
+    "    max_score = max(svec)\n",
+    "    if mode ==1:\n",
+    "        binary_svec = [0 if x<max_score else 1 for x in svec]\n",
+    "        assert(svec.count(max_score)==binary_svec.count(1))\n",
+    "    else:\n",
+    "        binary_svec = [0 if x<max_score else max_score for x in svec]\n",
+    "        assert(svec.count(max_score)==binary_svec.count(max_score))\n",
+    "    binary_svec = \",\".join([str(x) for x in binary_svec])\n",
+    "    return binary_svec\n",
+    "\n",
+    "check_validity(train_exclusive, val_exclusive, test_exclusive)\n",
+    "\n",
+    "train_exclusive = augment_rc(train_exclusive)\n",
+    "val_exclusive = augment_rc(val_exclusive)\n",
+    "test_exclusive = augment_rc(test_exclusive)\n",
+    "leaky_test = augment_rc(leaky_test)\n",
+    "\n",
+    "print(f\"Added reverse complement sequences to train_exclusive, val_exclusive, and test_exclusive (and leaky test_exclusive)\")\n",
+    "\n",
+    "check_validity(train_exclusive, val_exclusive, test_exclusive)\n",
+    "\n",
+    "total = sum([len(train_exclusive), len(val_exclusive), len(test_exclusive), len(leaky_test)])\n",
+    "print(\n",
+    "    f\"Length of train_exclusive dataset: {len(train_exclusive)} ({100*len(train_exclusive)/total:.2f}%)\"\n",
+    ")\n",
+    "print(f\"Length of val_exclusive dataset: {len(val_exclusive)} ({100*len(val_exclusive)/total:.2f}%)\")\n",
+    "print(f\"Length of test_exclusive dataset: {len(test_exclusive)} ({100*len(test_exclusive)/total:.2f}%)\")\n",
+    "print(f\"Length of leaky_test dataset: {len(leaky_test)} ({100*len(leaky_test)/total:.2f}%)\")\n",
+    "print(\n",
+    "    f\"Total sequences = {total}. Same as edges size*2? {total==len(all_data)*2}\"\n",
+    ")\n",
+    "\n",
+    "# since we've added all these new DNA sequences, we do need a new mapping of seq id to dna sequence\n",
+    "all_data = pd.concat([train_exclusive, val_exclusive, test_exclusive, leaky_test])\n",
+    "all_data[\"dna_seqid\"] = all_data[\"ID\"].str.split(\"_\", n=1, expand=True)[1]\n",
+    "dna_dict = dict(zip(all_data[\"dna_seqid\"], all_data[\"dna_sequence\"]))\n",
+    "assert len(dna_dict) == len(all_data.drop_duplicates([\"dna_sequence\"]))\n",
+    "\n",
+    "# create the output dir\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "split_out_dir = Path(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/splits/handpicked_val_test\")\n",
+    "os.makedirs(split_out_dir, exist_ok=True)\n",
+    "    \n",
+    "# add binary_scores to allow other training modes\n",
+    "train_exclusive[\"binary_scores\"] = train_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "val_exclusive[\"binary_scores\"] = val_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "test_exclusive[\"binary_scores\"] = test_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "leaky_test[\"binary_scores\"] = leaky_test[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "\n",
+    "train_exclusive[\"split\"] = [\"train\"]*len(train_exclusive)\n",
+    "val_exclusive[\"split\"] = [\"val\"]*len(val_exclusive)\n",
+    "test_exclusive[\"split\"] = [\"test\"]*len(test_exclusive)\n",
+    "leaky_test[\"split\"] = [\"leakytest\"]*len(leaky_test)\n",
+    "\n",
+    "# slect final cols and save\n",
+    "split_final_cols = [\"ID\", \"dna_sequence\", \"tr_sequence\", \"scores\", \"binary_scores\", \"split\"]\n",
+    "train_exclusive[split_final_cols].to_csv(split_out_dir / \"train.csv\", index=False)\n",
+    "val_exclusive[split_final_cols].to_csv(split_out_dir / \"val.csv\", index=False)\n",
+    "test_exclusive[split_final_cols].to_csv(split_out_dir / \"test.csv\", index=False)\n",
+    "leaky_test[split_final_cols].to_csv(split_out_dir / \"leakytest.csv\", index=False)\n",
+    "print(f\"Saved all splits to {split_out_dir}\")\n",
+    "\n",
+    "# make baby versions too\n",
+    "train_exclusive[split_final_cols].sample(400, random_state=42).to_csv(split_out_dir / \"babytrain.csv\", index=False)\n",
+    "val_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babyval.csv\", index=False)\n",
+    "test_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babytest.csv\", index=False)\n",
+    "leaky_test[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babyleakytest.csv\", index=False)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "642afa4e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ID</th>\n",
+       "      <th>tr_seqid</th>\n",
+       "      <th>dna_seqid</th>\n",
+       "      <th>peak_seqid</th>\n",
+       "      <th>chrpeak_id</th>\n",
+       "      <th>tr_name</th>\n",
+       "      <th>chipscore</th>\n",
+       "      <th>total_jaspar_hits</th>\n",
+       "      <th>dna_sequence</th>\n",
+       "      <th>tr_sequence</th>\n",
+       "      <th>scores</th>\n",
+       "      <th>tr_cluster_rep</th>\n",
+       "      <th>dna_cluster_rep</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>28719</th>\n",
+       "      <td>trseq4_dnaseq38058</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq38058</td>\n",
+       "      <td>peakseq59667</td>\n",
+       "      <td>chr8_peak2619</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>CTGGGAGTCTCCCAGTGAATCCTCTCCCTCCAGGAAGCATTCAGGG...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq38058</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30710</th>\n",
+       "      <td>trseq4_dnaseq58107</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq58107</td>\n",
+       "      <td>peakseq38026</td>\n",
+       "      <td>chr22_peak949</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>909.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GTAACGATGCCTTCCTAGGCACTGGCGTTACCGCCTGACCAAGGAG...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq58107</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8736</th>\n",
+       "      <td>trseq4_dnaseq18629</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq18629</td>\n",
+       "      <td>peakseq39291</td>\n",
+       "      <td>chr6_peak2197</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>61.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>CAAAGAAAGAAAATCTACTTTATTACAAGGAACAAAAACATAATAG...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq18629</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5747</th>\n",
+       "      <td>trseq4_dnaseq4928</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq4928</td>\n",
+       "      <td>peakseq39282</td>\n",
+       "      <td>chr1_peak4157</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>394.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AATTTCTATCAACTGAGGCAAAAGTCTTAAGTTCCCCCAAACCAAT...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq4928</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28098</th>\n",
+       "      <td>trseq4_dnaseq17297</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq17297</td>\n",
+       "      <td>peakseq46179</td>\n",
+       "      <td>chr1_peak4932</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>ATGTGGGAGTAGAGATAAAGAAATCAGTGCAGTTAAGGAGGGTAGA...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq17297</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6990</th>\n",
+       "      <td>trseq4_dnaseq45969</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq45969</td>\n",
+       "      <td>peakseq3403</td>\n",
+       "      <td>chr12_peak164</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>GCAGGAGAGGTCACAGACCCACAGAATCGTCCAATCCCTGCCCCAG...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq45969</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29914</th>\n",
+       "      <td>trseq4_dnaseq70215</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq70215</td>\n",
+       "      <td>peakseq29873</td>\n",
+       "      <td>chr10_peak1270</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>99.0</td>\n",
+       "      <td>3</td>\n",
+       "      <td>TGAGAAAAATAAACCCTGGGATATACAAAGGGACATCTGTCCACAG...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,99,99,99,99,99,9...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq70215</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>52579</th>\n",
+       "      <td>trseq4_dnaseq3768</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq3768</td>\n",
+       "      <td>peakseq43489</td>\n",
+       "      <td>chr22_peak1089</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>65.0</td>\n",
+       "      <td>9</td>\n",
+       "      <td>AAGTAGCTGGGATTACAGACGTACACCACCACGCCTGGCTAATTTT...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq3768</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6423</th>\n",
+       "      <td>trseq4_dnaseq19395</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq19395</td>\n",
+       "      <td>peakseq60837</td>\n",
+       "      <td>chr20_peak1976</td>\n",
+       "      <td>ZNF263</td>\n",
+       "      <td>76.0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>CAATACTCAATTTCCCCCTATTTATTTCTAGTGCAGGTTTCACAGC...</td>\n",
+       "      <td>MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq4</td>\n",
+       "      <td>dnaseq19395</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>49313</th>\n",
+       "      <td>trseq1_dnaseq5894</td>\n",
+       "      <td>trseq1</td>\n",
+       "      <td>dnaseq5894</td>\n",
+       "      <td>peakseq60338</td>\n",
+       "      <td>chr17_peak3283</td>\n",
+       "      <td>EGR1</td>\n",
+       "      <td>1000.0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>ACAGGGCAACAGCGGGATTGAGAGATGGAGGGATCCCCGCATCTGA...</td>\n",
+       "      <td>MAAAKAEMQLMSPLQISDPFGSFPHSPTMDNYPKLEEMMLLSNGAP...</td>\n",
+       "      <td>0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...</td>\n",
+       "      <td>trseq1</td>\n",
+       "      <td>dnaseq5894</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       ID tr_seqid    dna_seqid    peak_seqid      chrpeak_id  \\\n",
+       "28719  trseq4_dnaseq38058   trseq4  dnaseq38058  peakseq59667   chr8_peak2619   \n",
+       "30710  trseq4_dnaseq58107   trseq4  dnaseq58107  peakseq38026   chr22_peak949   \n",
+       "8736   trseq4_dnaseq18629   trseq4  dnaseq18629  peakseq39291   chr6_peak2197   \n",
+       "5747    trseq4_dnaseq4928   trseq4   dnaseq4928  peakseq39282   chr1_peak4157   \n",
+       "28098  trseq4_dnaseq17297   trseq4  dnaseq17297  peakseq46179   chr1_peak4932   \n",
+       "6990   trseq4_dnaseq45969   trseq4  dnaseq45969   peakseq3403   chr12_peak164   \n",
+       "29914  trseq4_dnaseq70215   trseq4  dnaseq70215  peakseq29873  chr10_peak1270   \n",
+       "52579   trseq4_dnaseq3768   trseq4   dnaseq3768  peakseq43489  chr22_peak1089   \n",
+       "6423   trseq4_dnaseq19395   trseq4  dnaseq19395  peakseq60837  chr20_peak1976   \n",
+       "49313   trseq1_dnaseq5894   trseq1   dnaseq5894  peakseq60338  chr17_peak3283   \n",
+       "\n",
+       "      tr_name  chipscore  total_jaspar_hits  \\\n",
+       "28719  ZNF263        5.0                  2   \n",
+       "30710  ZNF263      909.0                  1   \n",
+       "8736   ZNF263       61.0                  1   \n",
+       "5747   ZNF263      394.0                  1   \n",
+       "28098  ZNF263       10.0                  1   \n",
+       "6990   ZNF263     1000.0                  1   \n",
+       "29914  ZNF263       99.0                  3   \n",
+       "52579  ZNF263       65.0                  9   \n",
+       "6423   ZNF263       76.0                  2   \n",
+       "49313    EGR1     1000.0                  1   \n",
+       "\n",
+       "                                            dna_sequence  \\\n",
+       "28719  CTGGGAGTCTCCCAGTGAATCCTCTCCCTCCAGGAAGCATTCAGGG...   \n",
+       "30710  GTAACGATGCCTTCCTAGGCACTGGCGTTACCGCCTGACCAAGGAG...   \n",
+       "8736   CAAAGAAAGAAAATCTACTTTATTACAAGGAACAAAAACATAATAG...   \n",
+       "5747   AATTTCTATCAACTGAGGCAAAAGTCTTAAGTTCCCCCAAACCAAT...   \n",
+       "28098  ATGTGGGAGTAGAGATAAAGAAATCAGTGCAGTTAAGGAGGGTAGA...   \n",
+       "6990   GCAGGAGAGGTCACAGACCCACAGAATCGTCCAATCCCTGCCCCAG...   \n",
+       "29914  TGAGAAAAATAAACCCTGGGATATACAAAGGGACATCTGTCCACAG...   \n",
+       "52579  AAGTAGCTGGGATTACAGACGTACACCACCACGCCTGGCTAATTTT...   \n",
+       "6423   CAATACTCAATTTCCCCCTATTTATTTCTAGTGCAGGTTTCACAGC...   \n",
+       "49313  ACAGGGCAACAGCGGGATTGAGAGATGGAGGGATCCCCGCATCTGA...   \n",
+       "\n",
+       "                                             tr_sequence  \\\n",
+       "28719  MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "30710  MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "8736   MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "5747   MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "28098  MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "6990   MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "29914  MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "52579  MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "6423   MASGPGSQEREGLLIVKLEEDCAWSQELPPPDPGPSPEASHLRFRR...   \n",
+       "49313  MAAAKAEMQLMSPLQISDPFGSFPHSPTMDNYPKLEEMMLLSNGAP...   \n",
+       "\n",
+       "                                                  scores tr_cluster_rep  \\\n",
+       "28719  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "30710  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "8736   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "5747   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "28098  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "6990   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "29914  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,99,99,99,99,99,9...         trseq4   \n",
+       "52579  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "6423   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq4   \n",
+       "49313  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...         trseq1   \n",
+       "\n",
+       "      dna_cluster_rep  \n",
+       "28719     dnaseq38058  \n",
+       "30710     dnaseq58107  \n",
+       "8736      dnaseq18629  \n",
+       "5747       dnaseq4928  \n",
+       "28098     dnaseq17297  \n",
+       "6990      dnaseq45969  \n",
+       "29914     dnaseq70215  \n",
+       "52579      dnaseq3768  \n",
+       "6423      dnaseq19395  \n",
+       "49313      dnaseq5894  "
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_exclusive.sample(10, random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "id": "9e422e5b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.9"
+      ]
+     },
+     "execution_count": 67,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "round((0.8*total_rows)/len(remaining), 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "id": "e7163e49",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>tr_cluster_rep</th>\n",
+       "      <th>count</th>\n",
+       "      <th>sequence</th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>From</th>\n",
+       "      <th>Entry</th>\n",
+       "      <th>Reviewed</th>\n",
+       "      <th>Entry Name</th>\n",
+       "      <th>Protein names</th>\n",
+       "      <th>Gene Names</th>\n",
+       "      <th>Organism</th>\n",
+       "      <th>Length</th>\n",
+       "      <th>InterPro</th>\n",
+       "      <th>Pfam</th>\n",
+       "      <th>Motif</th>\n",
+       "      <th>Zinc finger</th>\n",
+       "      <th>Protein families</th>\n",
+       "      <th>Binding site</th>\n",
+       "      <th>Site</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>trseq23</td>\n",
+       "      <td>6407</td>\n",
+       "      <td>MSDPQTSMAATAAVSPSDYLQPAASTTQDSQPSPLALLAATCSKIG...</td>\n",
+       "      <td>443</td>\n",
+       "      <td>SP2</td>\n",
+       "      <td>Q02086</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>SP2_HUMAN</td>\n",
+       "      <td>Transcription factor Sp2</td>\n",
+       "      <td>SP2 KIAA0048</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>613</td>\n",
+       "      <td>IPR036236;IPR013087;</td>\n",
+       "      <td>PF00096;</td>\n",
+       "      <td>MOTIF 361..369; /note=\"9aaTAD; inactive\"; /evi...</td>\n",
+       "      <td>ZN_FING 525..549; /note=\"C2H2-type 1\"; /eviden...</td>\n",
+       "      <td>Sp1 C2H2-type zinc-finger protein family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>trseq26</td>\n",
+       "      <td>873</td>\n",
+       "      <td>MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...</td>\n",
+       "      <td>273</td>\n",
+       "      <td>NFYB</td>\n",
+       "      <td>P25208</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>NFYB_HUMAN</td>\n",
+       "      <td>Nuclear transcription factor Y subunit beta (C...</td>\n",
+       "      <td>NFYB HAP3</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>207</td>\n",
+       "      <td>IPR003958;IPR009072;IPR027113;IPR003956;</td>\n",
+       "      <td>PF00808;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NFYB/HAP3 subunit family</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>trseq17</td>\n",
+       "      <td>30</td>\n",
+       "      <td>MNFETSRCATLQYCPDPYIQRFVETPAHFSWKESYYRSTMSQSTQT...</td>\n",
+       "      <td>1004</td>\n",
+       "      <td>TP63</td>\n",
+       "      <td>Q9H3D4</td>\n",
+       "      <td>reviewed</td>\n",
+       "      <td>P63_HUMAN</td>\n",
+       "      <td>Tumor protein 63 (p63) (Chronic ulcerative sto...</td>\n",
+       "      <td>TP63 KET P63 P73H P73L TP73L</td>\n",
+       "      <td>Homo sapiens (Human)</td>\n",
+       "      <td>680</td>\n",
+       "      <td>IPR008967;IPR012346;IPR011615;IPR036674;IPR010...</td>\n",
+       "      <td>PF00870;PF07710;PF07647;</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>P53 family</td>\n",
+       "      <td>BINDING 244; /ligand=\"Zn(2+)\"; /ligand_id=\"ChE...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   tr_cluster_rep  count                                           sequence  \\\n",
+       "2         trseq23   6407  MSDPQTSMAATAAVSPSDYLQPAASTTQDSQPSPLALLAATCSKIG...   \n",
+       "6         trseq26    873  MTMDGDSSTTDASQLGISADYIGGSHYVIQPHDDTEDSMNDHEDTN...   \n",
+       "13        trseq17     30  MNFETSRCATLQYCPDPYIQRFVETPAHFSWKESYYRSTMSQSTQT...   \n",
+       "\n",
+       "    Unnamed: 0  From   Entry  Reviewed  Entry Name  \\\n",
+       "2          443   SP2  Q02086  reviewed   SP2_HUMAN   \n",
+       "6          273  NFYB  P25208  reviewed  NFYB_HUMAN   \n",
+       "13        1004  TP63  Q9H3D4  reviewed   P63_HUMAN   \n",
+       "\n",
+       "                                        Protein names  \\\n",
+       "2                            Transcription factor Sp2   \n",
+       "6   Nuclear transcription factor Y subunit beta (C...   \n",
+       "13  Tumor protein 63 (p63) (Chronic ulcerative sto...   \n",
+       "\n",
+       "                      Gene Names              Organism  Length  \\\n",
+       "2                   SP2 KIAA0048  Homo sapiens (Human)     613   \n",
+       "6                      NFYB HAP3  Homo sapiens (Human)     207   \n",
+       "13  TP63 KET P63 P73H P73L TP73L  Homo sapiens (Human)     680   \n",
+       "\n",
+       "                                             InterPro  \\\n",
+       "2                                IPR036236;IPR013087;   \n",
+       "6            IPR003958;IPR009072;IPR027113;IPR003956;   \n",
+       "13  IPR008967;IPR012346;IPR011615;IPR036674;IPR010...   \n",
+       "\n",
+       "                        Pfam  \\\n",
+       "2                   PF00096;   \n",
+       "6                   PF00808;   \n",
+       "13  PF00870;PF07710;PF07647;   \n",
+       "\n",
+       "                                                Motif  \\\n",
+       "2   MOTIF 361..369; /note=\"9aaTAD; inactive\"; /evi...   \n",
+       "6                                                 NaN   \n",
+       "13                                                NaN   \n",
+       "\n",
+       "                                          Zinc finger  \\\n",
+       "2   ZN_FING 525..549; /note=\"C2H2-type 1\"; /eviden...   \n",
+       "6                                                 NaN   \n",
+       "13                                                NaN   \n",
+       "\n",
+       "                            Protein families  \\\n",
+       "2   Sp1 C2H2-type zinc-finger protein family   \n",
+       "6                   NFYB/HAP3 subunit family   \n",
+       "13                                P53 family   \n",
+       "\n",
+       "                                         Binding site Site  \n",
+       "2                                                 NaN  NaN  \n",
+       "6                                                 NaN  NaN  \n",
+       "13  BINDING 244; /ligand=\"Zn(2+)\"; /ligand_id=\"ChE...  NaN  "
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vcs.loc[vcs[\"tr_cluster_rep\"].isin(handpicked_test_trs)]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cf6b3dbc",
+   "metadata": {},
+   "source": [
+    "# Another strategy: same as above but randomly drop most of trseq4 datapoints before we start"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f963c8d2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "29\n",
+      "71134\n",
+      "0\n",
+      "0\n",
+      "Dropped most TR4 datapoints to avoid overfitting. New starting size: 33674\n",
+      "Rows in test: 7310\n",
+      "Rows to be split between train and val: 23263\n",
+      "Total rows: 30573. Test percentage: 23.91%\n",
+      "Lost rows: 3101\n",
+      "Rows in val: 4434\n",
+      "Rows left for train: 18640\n",
+      "Total rows: 23074. Test percentage: 19.22%\n",
+      "Lost rows: 3290\n",
+      "Original total: 33674\n",
+      "New, exclusive total: 30384\n",
+      "Lost rows: 3290\n",
+      "Length train: 18640/30384 (61.35%)\n",
+      "Length val: 4434/30384 (14.59%)\n",
+      "Length test: 7310/30384 (24.06%)\n",
+      "Pass! No overlap in IDs\n",
+      "Train-Val TR intersection: 0\n",
+      "Train-Test TR intersection: 0\n",
+      "Val-Test TR intersection: 0\n",
+      "Train-Val TR Cluster Rep intersection: 0\n",
+      "Train-Test TR Cluster Rep intersection: 0\n",
+      "Val-Test TR Cluster Rep intersection: 0\n",
+      "Train-Val DNA intersection: 0\n",
+      "Train-Test DNA intersection: 0\n",
+      "Val-Test DNA intersection: 0\n",
+      "Train-Val DNA Cluster Rep intersection: 0\n",
+      "Train-Test DNA Cluster Rep intersection: 0\n",
+      "Val-Test DNA Cluster Rep intersection: 0\n",
+      "Added reverse complement sequences to train_exclusive, val_exclusive, and test_exclusive (and leaky test_exclusive)\n",
+      "Pass! No overlap in IDs\n",
+      "Train-Val TR intersection: 0\n",
+      "Train-Test TR intersection: 0\n",
+      "Val-Test TR intersection: 0\n",
+      "Train-Val TR Cluster Rep intersection: 0\n",
+      "Train-Test TR Cluster Rep intersection: 0\n",
+      "Val-Test TR Cluster Rep intersection: 0\n",
+      "Train-Val DNA intersection: 0\n",
+      "Train-Test DNA intersection: 0\n",
+      "Val-Test DNA intersection: 0\n",
+      "Train-Val DNA Cluster Rep intersection: 0\n",
+      "Train-Test DNA Cluster Rep intersection: 0\n",
+      "Val-Test DNA Cluster Rep intersection: 0\n",
+      "Length of train_exclusive dataset: 37280. (% of train+val+test=61.35%) (% of train_val+test+leaky_test=55.35%)\n",
+      "Length of val_exclusive dataset: 8868.(% of train+val+test = 14.59%) (% of train_val+test+leaky_test=13.17%)\n",
+      "Length of test_exclusive dataset: 14620. (% of train+val+test= 24.06%) (% of train_val+test+leaky_test=21.71%)\n",
+      "Length of leaky_test dataset: 6580. (% of train_val+test+leaky_test=9.77%)\n",
+      "Total sequences = 67348. Same as edges size*2? True\n",
+      "Saved all splits to /home/a03-svincoff/DPACMAN/dpacman/data_files/processed/splits/handpicked_val_test_cropTR4\n"
+     ]
+    }
+   ],
+   "source": [
+    "## Full pipeline\n",
+    "import pandas as pd\n",
+    "protein_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/protein/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "protein_clusters.columns=[\"tr_cluster_rep\",\"tr_cluster_member\"]\n",
+    "protein_clusters.head()\n",
+    "\n",
+    "dna_clusters = pd.read_csv(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/dna_full/mmseqs_cluster.tsv\", sep=\"\\t\", header=None)\n",
+    "dna_clusters.columns=[\"dna_cluster_rep\",\"dna_cluster_member\"]\n",
+    "dna_clusters.head()\n",
+    "\n",
+    "all_data = pd.read_parquet(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/remap2022_crm_fimo_output_q_processed_seed0.parquet\")\n",
+    "all_data\n",
+    "\n",
+    "protein_cluster_map = dict(zip(protein_clusters[\"tr_cluster_member\"],protein_clusters[\"tr_cluster_rep\"]))\n",
+    "dna_cluster_map = dict(zip(dna_clusters[\"dna_cluster_member\"],dna_clusters[\"dna_cluster_rep\"]))\n",
+    "print(len(protein_cluster_map))\n",
+    "print(len(dna_cluster_map))\n",
+    "all_data[\"tr_cluster_rep\"] = all_data[\"tr_seqid\"].map(protein_cluster_map)\n",
+    "all_data[\"dna_cluster_rep\"] = all_data[\"dna_seqid\"].map(dna_cluster_map)\n",
+    "print(len(all_data[all_data[\"tr_cluster_rep\"].isna()]))\n",
+    "print(len(all_data[all_data[\"dna_cluster_rep\"].isna()]))\n",
+    "all_data.head()\n",
+    "\n",
+    "### DROP MOST TRSEQ4 DATAPOINTS\n",
+    "all_data_notr4 = all_data.loc[all_data[\"tr_seqid\"]!=\"trseq4\"].reset_index(drop=True)\n",
+    "all_data_tr4 = all_data.loc[all_data[\"tr_seqid\"]==\"trseq4\"].reset_index(drop=True)\n",
+    "all_data_tr4 = all_data_tr4.sample(8962,random_state=42).reset_index(drop=True)\n",
+    "all_data = pd.concat([all_data_notr4,all_data_tr4]).reset_index(drop=True)\n",
+    "print(f\"Dropped most TR4 datapoints to avoid overfitting. New starting size: {len(all_data)}\")\n",
+    "\n",
+    "### handpick test\n",
+    "# trseq29\n",
+    "handpicked_test_trs = [\"trseq23\",\"trseq26\",\"trseq17\"]\n",
+    "handpicked_test = all_data.loc[\n",
+    "    all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_test[\"dna_cluster_rep\"].unique().tolist()\n",
+    "remaining = all_data.loc[\n",
+    "    (~all_data[\"tr_cluster_rep\"].isin(handpicked_test_trs)) & \n",
+    "    (~all_data[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "test_ids = handpicked_test[\"ID\"].unique().tolist()\n",
+    "remaining_ids = remaining[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(remaining_ids))\n",
+    "]\n",
+    "print(f\"Rows in test: {len(handpicked_test)}\")\n",
+    "print(f\"Rows to be split between train and val: {len(remaining)}\")\n",
+    "total_rows = len(handpicked_test) + len(remaining)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_test)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")\n",
+    "\n",
+    "### handpick val\n",
+    "handpicked_val_trs = [\"trseq9\", \"trseq5\", \"trseq28\"]\n",
+    "\n",
+    "handpicked_val = remaining.loc[\n",
+    "    remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "off_limits_dna_clusters = handpicked_val[\"dna_cluster_rep\"].unique().tolist()\n",
+    "train_remain = remaining.loc[\n",
+    "    (~remaining[\"tr_cluster_rep\"].isin(handpicked_val_trs)) & \n",
+    "    (~remaining[\"dna_cluster_rep\"].isin(off_limits_dna_clusters))  \n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "val_ids = handpicked_val[\"ID\"].unique().tolist()\n",
+    "train_remain_ids = train_remain[\"ID\"].unique().tolist()\n",
+    "lost_rows = all_data.loc[\n",
+    "    (~all_data[\"ID\"].isin(test_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(val_ids)) & \n",
+    "    (~all_data[\"ID\"].isin(train_remain_ids))\n",
+    "]\n",
+    "print(f\"Rows in val: {len(handpicked_val)}\")\n",
+    "print(f\"Rows left for train: {len(train_remain)}\")\n",
+    "total_rows = len(handpicked_val) + len(train_remain)\n",
+    "print(f\"Total rows: {total_rows}. Test percentage: {100*len(handpicked_val)/total_rows:.2f}%\")\n",
+    "print(f\"Lost rows: {len(lost_rows)}\")\n",
+    "\n",
+    "train_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(train_remain_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "val_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(val_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "test_exclusive = all_data.loc[\n",
+    "    all_data[\"ID\"].isin(test_ids)\n",
+    "    ].reset_index(drop=True)\n",
+    "\n",
+    "leaky_test = all_data.loc[\n",
+    "    ~(all_data[\"ID\"].isin(train_exclusive[\"ID\"].tolist())) & \n",
+    "    ~(all_data[\"ID\"].isin(val_exclusive[\"ID\"].tolist())) & \n",
+    "    ~(all_data[\"ID\"].isin(test_exclusive[\"ID\"].tolist()))\n",
+    "].reset_index(drop=True)\n",
+    "\n",
+    "print(f\"Original total: {len(all_data)}\")\n",
+    "retained_total = len(train_exclusive)+len(val_exclusive)+len(test_exclusive)\n",
+    "print(f\"New, exclusive total: {retained_total}\")\n",
+    "print(f\"Lost rows: {len(all_data)-retained_total}\")\n",
+    "print(f\"Length train: {len(train_exclusive)}/{retained_total} ({100*len(train_exclusive)/retained_total:.2f}%)\")\n",
+    "print(f\"Length val: {len(val_exclusive)}/{retained_total} ({100*len(val_exclusive)/retained_total:.2f}%)\")\n",
+    "print(f\"Length test: {len(test_exclusive)}/{retained_total} ({100*len(test_exclusive)/retained_total:.2f}%)\")\n",
+    "\n",
+    "def check_validity(train_exclusive, val_exclusive, test_exclusive):\n",
+    "    train_exclusive_ids = set(train_exclusive[\"ID\"].unique().tolist())\n",
+    "    val_exclusive_ids = set(val_exclusive[\"ID\"].unique().tolist())\n",
+    "    test_exclusive_ids = set(test_exclusive[\"ID\"].unique().tolist())\n",
+    "\n",
+    "    assert len(train_exclusive_ids.intersection(val_exclusive_ids)) == 0\n",
+    "    assert len(train_exclusive_ids.intersection(test_exclusive_ids)) == 0\n",
+    "    assert len(val_exclusive_ids.intersection(test_exclusive_ids)) == 0\n",
+    "    print(f\"Pass! No overlap in IDs\")\n",
+    "\n",
+    "    # Investigate TR intersection. No assertions unless we are explicitly splitting on this. \n",
+    "    train_exclusive_tr_seqs = set(train_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "    val_exclusive_tr_seqs = set(val_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "    test_exclusive_tr_seqs = set(test_exclusive[\"tr_sequence\"].unique().tolist())\n",
+    "\n",
+    "    train_exclusive_tr_reps = set(train_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "    val_exclusive_tr_reps = set(val_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "    test_exclusive_tr_reps = set(test_exclusive[\"tr_cluster_rep\"].unique().tolist())\n",
+    "\n",
+    "    print(f\"Train-Val TR intersection: {len(train_exclusive_tr_seqs.intersection(val_exclusive_tr_seqs))}\")\n",
+    "    print(f\"Train-Test TR intersection: {len(train_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}\")\n",
+    "    print(f\"Val-Test TR intersection: {len(val_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}\")\n",
+    "\n",
+    "    print(f\"Train-Val TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(val_exclusive_tr_reps))}\")\n",
+    "    print(f\"Train-Test TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}\")\n",
+    "    print(f\"Val-Test TR Cluster Rep intersection: {len(val_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}\")\n",
+    "\n",
+    "    # Investigate DNA intersection. No assertions unless we are explicitly splitting on this. \n",
+    "    train_exclusive_dna_seqs = set(train_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "    val_exclusive_dna_seqs = set(val_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "    test_exclusive_dna_seqs = set(test_exclusive[\"dna_sequence\"].unique().tolist())\n",
+    "\n",
+    "    train_exclusive_dna_reps = set(train_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "    val_exclusive_dna_reps = set(val_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "    test_exclusive_dna_reps = set(test_exclusive[\"dna_cluster_rep\"].unique().tolist())\n",
+    "\n",
+    "    print(f\"Train-Val DNA intersection: {len(train_exclusive_dna_seqs.intersection(val_exclusive_dna_seqs))}\")\n",
+    "    print(f\"Train-Test DNA intersection: {len(train_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}\")\n",
+    "    print(f\"Val-Test DNA intersection: {len(val_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}\")\n",
+    "\n",
+    "    print(f\"Train-Val DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(val_exclusive_dna_reps))}\")\n",
+    "    print(f\"Train-Test DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}\")\n",
+    "    print(f\"Val-Test DNA Cluster Rep intersection: {len(val_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}\")\n",
+    "\n",
+    "def get_reverse_complement(s):\n",
+    "    \"\"\"\n",
+    "    Returns 5' to 3' sequence of the reverse complement\n",
+    "    \"\"\"\n",
+    "    chars = list(s)\n",
+    "    recon = []\n",
+    "    rev_map = {\n",
+    "        \"a\": \"t\",\n",
+    "        \"c\": \"g\",\n",
+    "        \"t\": \"a\",\n",
+    "        \"g\": \"c\",\n",
+    "        \"A\": \"T\",\n",
+    "        \"C\": \"G\",\n",
+    "        \"T\": \"A\",\n",
+    "        \"G\": \"C\",\n",
+    "        \"n\": \"n\",\n",
+    "        \"N\": \"N\",\n",
+    "    }\n",
+    "    for c in chars:\n",
+    "        recon += [rev_map[c]]\n",
+    "\n",
+    "    recon = \"\".join(recon)\n",
+    "    return recon[::-1]\n",
+    "\n",
+    "# now make reverse complements\n",
+    "def augment_rc(df):\n",
+    "    \"\"\"\n",
+    "    Get the reverse complement and add it as a datapoint, effectively doubling the dataset.\n",
+    "    Also flip the orientation of the scores\n",
+    "\n",
+    "    columns = [\"ID\",\"dna_sequence\",\"tr_sequence\",\"tr_cluster_rep\",\"dna_cluster_rep\", \"scores\",\"split\"]\n",
+    "    \"\"\"\n",
+    "    df_rc = df.copy(deep=True)\n",
+    "\n",
+    "    df_rc[\"dna_sequence\"] = df_rc[\"dna_sequence\"].apply(\n",
+    "        lambda x: get_reverse_complement(x)\n",
+    "    )\n",
+    "    df_rc[\"ID\"] = df_rc[\"ID\"] + \"_rc\"\n",
+    "    df_rc[\"scores\"] = df_rc[\"scores\"].apply(lambda s: \",\".join(s.split(\",\")[::-1]))\n",
+    "\n",
+    "    final_df = pd.concat([df, df_rc]).reset_index(drop=True)\n",
+    "\n",
+    "    return final_df\n",
+    "\n",
+    "def convert_scores(scores, mode=1):\n",
+    "    \"\"\"\n",
+    "    Two modes: 1 means FIMO peaks get 1. 0 means FIMO peaks get their max score\n",
+    "    \"\"\"\n",
+    "    svec = [int(x) for x in scores.split(\",\")]\n",
+    "    max_score = max(svec)\n",
+    "    if mode ==1:\n",
+    "        binary_svec = [0 if x<max_score else 1 for x in svec]\n",
+    "        assert(svec.count(max_score)==binary_svec.count(1))\n",
+    "    else:\n",
+    "        binary_svec = [0 if x<max_score else max_score for x in svec]\n",
+    "        assert(svec.count(max_score)==binary_svec.count(max_score))\n",
+    "    binary_svec = \",\".join([str(x) for x in binary_svec])\n",
+    "    return binary_svec\n",
+    "\n",
+    "check_validity(train_exclusive, val_exclusive, test_exclusive)\n",
+    "\n",
+    "train_exclusive = augment_rc(train_exclusive)\n",
+    "val_exclusive = augment_rc(val_exclusive)\n",
+    "test_exclusive = augment_rc(test_exclusive)\n",
+    "leaky_test = augment_rc(leaky_test)\n",
+    "\n",
+    "print(f\"Added reverse complement sequences to train_exclusive, val_exclusive, and test_exclusive (and leaky test_exclusive)\")\n",
+    "\n",
+    "check_validity(train_exclusive, val_exclusive, test_exclusive)\n",
+    "\n",
+    "total = sum([len(train_exclusive), len(val_exclusive), len(test_exclusive), len(leaky_test)])\n",
+    "main_total = sum([len(train_exclusive), len(val_exclusive), len(test_exclusive)])\n",
+    "print(\n",
+    "    f\"Length of train_exclusive dataset: {len(train_exclusive)}. (% of train+val+test={100*len(train_exclusive)/main_total:.2f}%) (% of train_val+test+leaky_test={100*len(train_exclusive)/total:.2f}%)\"\n",
+    ")\n",
+    "print(f\"Length of val_exclusive dataset: {len(val_exclusive)}.(% of train+val+test = {100*len(val_exclusive)/main_total:.2f}%) (% of train_val+test+leaky_test={100*len(val_exclusive)/total:.2f}%)\")\n",
+    "print(f\"Length of test_exclusive dataset: {len(test_exclusive)}. (% of train+val+test= {100*len(test_exclusive)/main_total:.2f}%) (% of train_val+test+leaky_test={100*len(test_exclusive)/total:.2f}%)\")\n",
+    "print(f\"Length of leaky_test dataset: {len(leaky_test)}. (% of train_val+test+leaky_test={100*len(leaky_test)/total:.2f}%)\")\n",
+    "print(\n",
+    "    f\"Total sequences = {total}. Same as edges size*2? {total==len(all_data)*2}\"\n",
+    ")\n",
+    "\n",
+    "# since we've added all these new DNA sequences, we do need a new mapping of seq id to dna sequence\n",
+    "all_data = pd.concat([train_exclusive, val_exclusive, test_exclusive, leaky_test])\n",
+    "all_data[\"dna_seqid\"] = all_data[\"ID\"].str.split(\"_\", n=1, expand=True)[1]\n",
+    "dna_dict = dict(zip(all_data[\"dna_seqid\"], all_data[\"dna_sequence\"]))\n",
+    "assert len(dna_dict) == len(all_data.drop_duplicates([\"dna_sequence\"]))\n",
+    "\n",
+    "# create the output dir\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "split_out_dir = Path(\"/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/splits/handpicked_val_test_cropTR4\")\n",
+    "os.makedirs(split_out_dir, exist_ok=True)\n",
+    "    \n",
+    "# add binary_scores to allow other training modes\n",
+    "train_exclusive[\"binary_scores\"] = train_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "val_exclusive[\"binary_scores\"] = val_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "test_exclusive[\"binary_scores\"] = test_exclusive[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "leaky_test[\"binary_scores\"] = leaky_test[\"scores\"].apply(lambda x: convert_scores(x, mode=1))\n",
+    "\n",
+    "train_exclusive[\"split\"] = [\"train\"]*len(train_exclusive)\n",
+    "val_exclusive[\"split\"] = [\"val\"]*len(val_exclusive)\n",
+    "test_exclusive[\"split\"] = [\"test\"]*len(test_exclusive)\n",
+    "leaky_test[\"split\"] = [\"leakytest\"]*len(leaky_test)\n",
+    "\n",
+    "# slect final cols and save\n",
+    "split_final_cols = [\"ID\", \"dna_sequence\", \"tr_sequence\", \"scores\", \"binary_scores\", \"split\"]\n",
+    "train_exclusive[split_final_cols].to_csv(split_out_dir / \"train.csv\", index=False)\n",
+    "val_exclusive[split_final_cols].to_csv(split_out_dir / \"val.csv\", index=False)\n",
+    "test_exclusive[split_final_cols].to_csv(split_out_dir / \"test.csv\", index=False)\n",
+    "leaky_test[split_final_cols].to_csv(split_out_dir / \"leakytest.csv\", index=False)\n",
+    "print(f\"Saved all splits to {split_out_dir}\")\n",
+    "\n",
+    "# make baby versions too\n",
+    "train_exclusive[split_final_cols].sample(400, random_state=42).to_csv(split_out_dir / \"babytrain.csv\", index=False)\n",
+    "val_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babyval.csv\", index=False)\n",
+    "test_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babytest.csv\", index=False)\n",
+    "leaky_test[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / \"babyleakytest.csv\", index=False)\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dnabind2",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}