Hugging Face's logo

Buckets:
graybirdowner
/
LiLab

Files
xet
graybirdowner/LiLab / data /preprocess_ERV_non_ERV /data_explorer.ipynb
graybirdowner's picture
graybirdowner
download
raw
43.4 kB
{
"cells": [
{
"cell_type": "code",
"execution_count": 38,
"id": "09908abe",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from Bio import SeqIO\n",
"import sys\n",
"\n",
"def fasta_to_dataframe(filename):\n",
" \"\"\"\n",
" Convert FASTA file to pandas DataFrame.\n",
" \n",
" Args:\n",
" filename: Path to .fa or .fna file\n",
" \n",
" Returns:\n",
" DataFrame with columns: sequence_name, sequence_length\n",
" \"\"\"\n",
" records = []\n",
" for record in SeqIO.parse(filename, \"fasta\"):\n",
" records.append({\n",
" \"sequence_name\": record.id,\n",
" \"sequence_length\": len(record.seq)\n",
" })\n",
" \n",
" df = pd.DataFrame(records)\n",
" print(f\"Loaded {len(df)} sequences from {filename}\")\n",
" print(f\"Mean length: {df['sequence_length'].mean():.1f} bp\")\n",
" return df\n",
"\n",
"# Usage example:\n",
"# df = fasta_to_dataframe(\"hg38_ERV_100bp.fa\")\n",
"# print(df.head())"
]
},
{
"cell_type": "code",
"execution_count": 39,
"id": "03793dd8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded 614316 sequences from hg38_ERV_100bp.fa\n",
"Mean length: 422.6 bp\n",
" sequence_name sequence_length\n",
"0 chr1:100028782-100029342(-) 560\n",
"1 chr1:10002898-10003145(+) 247\n",
"2 chr1:10003457-10003703(+) 246\n",
"3 chr1:100034875-100035026(+) 151\n",
"4 chr1:100036738-100037248(-) 510\n"
]
}
],
"source": [
"df = fasta_to_dataframe(\"hg38_ERV_100bp.fa\")\n",
"print(df.head())"
]
},
{
"cell_type": "code",
"execution_count": 40,
"id": "a09d76ad",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded 24 sequences from hg38_rmsk_ERV.fa\n",
"Mean length: 128677909.7 bp\n"
]
},
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
}
],
"ref": "331f7fe5-f6dc-412c-b5ab-d5e5138ca410",
"rows": [
[
"0",
"chr1",
"248956422"
],
[
"1",
"chr2",
"242193529"
],
[
"2",
"chr3",
"198295559"
],
[
"3",
"chr4",
"190214555"
],
[
"4",
"chr5",
"181538259"
]
],
"shape": {
"columns": 2,
"rows": 5
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>chr1</td>\n",
" <td>248956422</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>chr2</td>\n",
" <td>242193529</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>chr3</td>\n",
" <td>198295559</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>chr4</td>\n",
" <td>190214555</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>chr5</td>\n",
" <td>181538259</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length\n",
"0 chr1 248956422\n",
"1 chr2 242193529\n",
"2 chr3 198295559\n",
"3 chr4 190214555\n",
"4 chr5 181538259"
]
},
"execution_count": 40,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df2 = fasta_to_dataframe(\"hg38_rmsk_ERV.fa\")\n",
"df2.head()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"id": "d815fe5c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded 35624 sequences from CCDS_nucleotide.current.fna\n",
"Mean length: 1748.4 bp\n"
]
},
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
}
],
"ref": "36c4a6fd-3867-4713-bc05-a3edde9af9b7",
"rows": [
[
"0",
"CCDS2.2|Hs110|chr1",
"2046"
],
[
"1",
"CCDS3.1|Hs110|chr1",
"2250"
],
[
"2",
"CCDS4.1|Hs110|chr1",
"1836"
],
[
"3",
"CCDS5.1|Hs110|chr1",
"666"
],
[
"4",
"CCDS6.1|Hs110|chr1",
"498"
]
],
"shape": {
"columns": 2,
"rows": 5
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CCDS2.2|Hs110|chr1</td>\n",
" <td>2046</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>CCDS3.1|Hs110|chr1</td>\n",
" <td>2250</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>CCDS4.1|Hs110|chr1</td>\n",
" <td>1836</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>CCDS5.1|Hs110|chr1</td>\n",
" <td>666</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>CCDS6.1|Hs110|chr1</td>\n",
" <td>498</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length\n",
"0 CCDS2.2|Hs110|chr1 2046\n",
"1 CCDS3.1|Hs110|chr1 2250\n",
"2 CCDS4.1|Hs110|chr1 1836\n",
"3 CCDS5.1|Hs110|chr1 666\n",
"4 CCDS6.1|Hs110|chr1 498"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df3 = fasta_to_dataframe(\"CCDS_nucleotide.current.fna\")\n",
"df3.head()"
]
},
{
"cell_type": "code",
"execution_count": 42,
"id": "1794bc8d",
"metadata": {},
"outputs": [],
"source": [
"erv = df\n",
"non_erv1 = df2\n",
"non_erv2 = df3"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "d8fd3a00",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"erv: {'train': 552884, 'test': 61432}\n",
"non_erv1: {'train': 22, 'test': 2}\n",
"non_erv2: {'train': 32062, 'test': 3562}\n"
]
}
],
"source": [
"for df in (erv, non_erv1, non_erv2):\n",
" df['group'] = 'train'\n",
" test_idx = df.sample(frac=0.1, random_state=42).index\n",
" df.loc[test_idx, 'group'] = 'test'\n",
"\n",
"print(\"erv:\", erv['group'].value_counts().to_dict())\n",
"print(\"non_erv1:\", non_erv1['group'].value_counts().to_dict())\n",
"print(\"non_erv2:\", non_erv2['group'].value_counts().to_dict())"
]
},
{
"cell_type": "markdown",
"id": "5a724429",
"metadata": {},
"source": [
"### ERV processing here"
]
},
{
"cell_type": "code",
"execution_count": 44,
"id": "3e6e7220",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "float64",
"type": "float"
}
],
"ref": "32dd09a3-b267-4f74-aa25-0e8a853c3f27",
"rows": [
[
"count",
"614316.0"
],
[
"mean",
"422.5782121904688"
],
[
"std",
"464.52475378615094"
],
[
"min",
"101.0"
],
[
"25%",
"214.0"
],
[
"50%",
"360.0"
],
[
"75%",
"460.0"
],
[
"max",
"112801.0"
]
],
"shape": {
"columns": 1,
"rows": 8
}
},
"text/plain": [
"count 614316.000000\n",
"mean 422.578212\n",
"std 464.524754\n",
"min 101.000000\n",
"25% 214.000000\n",
"50% 360.000000\n",
"75% 460.000000\n",
"max 112801.000000\n",
"Name: sequence_length, dtype: float64"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"erv['sequence_length'].describe()"
]
},
{
"cell_type": "code",
"execution_count": 45,
"id": "ebc477a4",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
},
{
"name": "group",
"rawType": "str",
"type": "string"
}
],
"ref": "e60a9793-9aa8-4335-afee-6059b63e1b5f",
"rows": [
[
"0",
"chr1:100028782-100029342(-)",
"560",
"train"
],
[
"1",
"chr1:10002898-10003145(+)",
"247",
"train"
],
[
"2",
"chr1:10003457-10003703(+)",
"246",
"test"
],
[
"3",
"chr1:100034875-100035026(+)",
"151",
"train"
],
[
"4",
"chr1:100036738-100037248(-)",
"510",
"train"
],
[
"5",
"chr1:100060354-100060721(-)",
"367",
"train"
],
[
"6",
"chr1:100103858-100104462(-)",
"604",
"test"
],
[
"7",
"chr1:10011021-10011544(-)",
"523",
"test"
],
[
"8",
"chr1:100119540-100119933(+)",
"393",
"train"
],
[
"9",
"chr1:100137498-100137748(-)",
"250",
"train"
],
[
"10",
"chr1:10013996-10014374(+)",
"378",
"train"
],
[
"11",
"chr1:10014681-10015065(+)",
"384",
"train"
],
[
"12",
"chr1:100157147-100157502(+)",
"355",
"train"
],
[
"13",
"chr1:100165398-100165520(+)",
"122",
"train"
],
[
"14",
"chr1:100165697-100165869(+)",
"172",
"train"
],
[
"15",
"chr1:10016592-10016818(-)",
"226",
"train"
],
[
"16",
"chr1:100166041-100166236(+)",
"195",
"train"
],
[
"17",
"chr1:100166241-100166817(+)",
"576",
"train"
],
[
"18",
"chr1:100166814-100167026(+)",
"212",
"train"
],
[
"19",
"chr1:100167030-100167278(+)",
"248",
"train"
]
],
"shape": {
"columns": 3,
"rows": 20
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" <th>group</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>chr1:100028782-100029342(-)</td>\n",
" <td>560</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>chr1:10002898-10003145(+)</td>\n",
" <td>247</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>chr1:10003457-10003703(+)</td>\n",
" <td>246</td>\n",
" <td>test</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>chr1:100034875-100035026(+)</td>\n",
" <td>151</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>chr1:100036738-100037248(-)</td>\n",
" <td>510</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>chr1:100060354-100060721(-)</td>\n",
" <td>367</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>chr1:100103858-100104462(-)</td>\n",
" <td>604</td>\n",
" <td>test</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>chr1:10011021-10011544(-)</td>\n",
" <td>523</td>\n",
" <td>test</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>chr1:100119540-100119933(+)</td>\n",
" <td>393</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>chr1:100137498-100137748(-)</td>\n",
" <td>250</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>chr1:10013996-10014374(+)</td>\n",
" <td>378</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>chr1:10014681-10015065(+)</td>\n",
" <td>384</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>chr1:100157147-100157502(+)</td>\n",
" <td>355</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>chr1:100165398-100165520(+)</td>\n",
" <td>122</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>chr1:100165697-100165869(+)</td>\n",
" <td>172</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>chr1:10016592-10016818(-)</td>\n",
" <td>226</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>chr1:100166041-100166236(+)</td>\n",
" <td>195</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>chr1:100166241-100166817(+)</td>\n",
" <td>576</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>chr1:100166814-100167026(+)</td>\n",
" <td>212</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>chr1:100167030-100167278(+)</td>\n",
" <td>248</td>\n",
" <td>train</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length group\n",
"0 chr1:100028782-100029342(-) 560 train\n",
"1 chr1:10002898-10003145(+) 247 train\n",
"2 chr1:10003457-10003703(+) 246 test\n",
"3 chr1:100034875-100035026(+) 151 train\n",
"4 chr1:100036738-100037248(-) 510 train\n",
"5 chr1:100060354-100060721(-) 367 train\n",
"6 chr1:100103858-100104462(-) 604 test\n",
"7 chr1:10011021-10011544(-) 523 test\n",
"8 chr1:100119540-100119933(+) 393 train\n",
"9 chr1:100137498-100137748(-) 250 train\n",
"10 chr1:10013996-10014374(+) 378 train\n",
"11 chr1:10014681-10015065(+) 384 train\n",
"12 chr1:100157147-100157502(+) 355 train\n",
"13 chr1:100165398-100165520(+) 122 train\n",
"14 chr1:100165697-100165869(+) 172 train\n",
"15 chr1:10016592-10016818(-) 226 train\n",
"16 chr1:100166041-100166236(+) 195 train\n",
"17 chr1:100166241-100166817(+) 576 train\n",
"18 chr1:100166814-100167026(+) 212 train\n",
"19 chr1:100167030-100167278(+) 248 train"
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"erv.head(20)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "25b6dbf5",
"metadata": {},
"outputs": [],
"source": [
"# erv.to_csv(\"erv_seq_split.csv\", index=False)"
]
},
{
"cell_type": "code",
"execution_count": 47,
"id": "4ba81b03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"552884.4"
]
},
"execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
".9 * 614316"
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "8dbcff2b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"61431.600000000006"
]
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"614316*0.1"
]
},
{
"cell_type": "markdown",
"id": "82e9f210",
"metadata": {},
"source": [
"### Non-ERV start here"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "4541778a",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "float64",
"type": "float"
}
],
"ref": "e43cd055-ee2f-4db8-9752-d5b85e8722c4",
"rows": [
[
"count",
"24.0"
],
[
"mean",
"128677909.66666667"
],
[
"std",
"57811513.87023528"
],
[
"min",
"46709983.0"
],
[
"25%",
"82536402.0"
],
[
"50%",
"133536365.5"
],
[
"75%",
"162210974.5"
],
[
"max",
"248956422.0"
]
],
"shape": {
"columns": 1,
"rows": 8
}
},
"text/plain": [
"count 2.400000e+01\n",
"mean 1.286779e+08\n",
"std 5.781151e+07\n",
"min 4.670998e+07\n",
"25% 8.253640e+07\n",
"50% 1.335364e+08\n",
"75% 1.622110e+08\n",
"max 2.489564e+08\n",
"Name: sequence_length, dtype: float64"
]
},
"execution_count": 49,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv1['sequence_length'].describe()"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "8d4971c5",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "float64",
"type": "float"
}
],
"ref": "014d98d3-35a6-4295-be34-83e0490d2cbb",
"rows": [
[
"count",
"35624.0"
],
[
"mean",
"1748.3783123736807"
],
[
"std",
"1923.3919954082594"
],
[
"min",
"78.0"
],
[
"25%",
"801.0"
],
[
"50%",
"1323.0"
],
[
"75%",
"2127.0"
],
[
"max",
"107976.0"
]
],
"shape": {
"columns": 1,
"rows": 8
}
},
"text/plain": [
"count 35624.000000\n",
"mean 1748.378312\n",
"std 1923.391995\n",
"min 78.000000\n",
"25% 801.000000\n",
"50% 1323.000000\n",
"75% 2127.000000\n",
"max 107976.000000\n",
"Name: sequence_length, dtype: float64"
]
},
"execution_count": 50,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv2['sequence_length'].describe()"
]
},
{
"cell_type": "code",
"execution_count": 51,
"id": "baf63295",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "group",
"rawType": "str",
"type": "string"
},
{
"name": "count",
"rawType": "int64",
"type": "integer"
}
],
"ref": "249bd0af-4ce9-4b6a-b7b7-0490e7abc360",
"rows": [
[
"train",
"22"
],
[
"test",
"2"
]
],
"shape": {
"columns": 1,
"rows": 2
}
},
"text/plain": [
"group\n",
"train 22\n",
"test 2\n",
"Name: count, dtype: int64"
]
},
"execution_count": 51,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv1['group'] = 'train'\n",
"test_idx = non_erv1.sample(frac=0.1, random_state=42).index\n",
"non_erv1.loc[test_idx, 'group'] = 'test'\n",
"\n",
"non_erv1['group'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 52,
"id": "e6734624",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
},
{
"name": "group",
"rawType": "str",
"type": "string"
}
],
"ref": "00afb35e-67c1-4a0d-b870-fcf2617c2047",
"rows": [
[
"0",
"chr1",
"248956422",
"train"
],
[
"1",
"chr2",
"242193529",
"train"
],
[
"2",
"chr3",
"198295559",
"train"
],
[
"3",
"chr4",
"190214555",
"train"
],
[
"4",
"chr5",
"181538259",
"train"
]
],
"shape": {
"columns": 3,
"rows": 5
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" <th>group</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>chr1</td>\n",
" <td>248956422</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>chr2</td>\n",
" <td>242193529</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>chr3</td>\n",
" <td>198295559</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>chr4</td>\n",
" <td>190214555</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>chr5</td>\n",
" <td>181538259</td>\n",
" <td>train</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length group\n",
"0 chr1 248956422 train\n",
"1 chr2 242193529 train\n",
"2 chr3 198295559 train\n",
"3 chr4 190214555 train\n",
"4 chr5 181538259 train"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv1.head()"
]
},
{
"cell_type": "code",
"execution_count": 53,
"id": "e5a7f2a0",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "float64",
"type": "float"
}
],
"ref": "6d8f05c5-edf0-47cc-84f4-abfbf482b0c5",
"rows": [
[
"count",
"35619.0"
],
[
"mean",
"1748.611190656672"
],
[
"std",
"1923.4265456091828"
],
[
"min",
"102.0"
],
[
"25%",
"801.0"
],
[
"50%",
"1323.0"
],
[
"75%",
"2127.0"
],
[
"max",
"107976.0"
]
],
"shape": {
"columns": 1,
"rows": 8
}
},
"text/plain": [
"count 35619.000000\n",
"mean 1748.611191\n",
"std 1923.426546\n",
"min 102.000000\n",
"25% 801.000000\n",
"50% 1323.000000\n",
"75% 2127.000000\n",
"max 107976.000000\n",
"Name: sequence_length, dtype: float64"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv2 = non_erv2[non_erv2['sequence_length'] >= 100]\n",
"non_erv2['sequence_length'].describe()"
]
},
{
"cell_type": "code",
"execution_count": 54,
"id": "06ec849c",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "group",
"rawType": "str",
"type": "string"
},
{
"name": "count",
"rawType": "int64",
"type": "integer"
}
],
"ref": "008deb34-4996-4287-acf6-6fb84cf3b8c1",
"rows": [
[
"train",
"32057"
],
[
"test",
"3562"
]
],
"shape": {
"columns": 1,
"rows": 2
}
},
"text/plain": [
"group\n",
"train 32057\n",
"test 3562\n",
"Name: count, dtype: int64"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv2['group'] = 'train'\n",
"test_idx = non_erv2.sample(frac=0.1, random_state=42).index\n",
"non_erv2.loc[test_idx, 'group'] = 'test'\n",
"\n",
"non_erv2['group'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 62,
"id": "32a2498a",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "object",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
},
{
"name": "group",
"rawType": "str",
"type": "string"
}
],
"ref": "b17abd8e-70f9-4385-ae15-4e6258d27fc2",
"rows": [
[
"0",
"CCDS2.2",
"2046",
"train"
],
[
"1",
"CCDS3.1",
"2250",
"train"
],
[
"2",
"CCDS4.1",
"1836",
"train"
],
[
"3",
"CCDS5.1",
"666",
"train"
],
[
"4",
"CCDS6.1",
"498",
"test"
]
],
"shape": {
"columns": 3,
"rows": 5
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" <th>group</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>CCDS2.2</td>\n",
" <td>2046</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>CCDS3.1</td>\n",
" <td>2250</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>CCDS4.1</td>\n",
" <td>1836</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>CCDS5.1</td>\n",
" <td>666</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>CCDS6.1</td>\n",
" <td>498</td>\n",
" <td>test</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length group\n",
"0 CCDS2.2 2046 train\n",
"1 CCDS3.1 2250 train\n",
"2 CCDS4.1 1836 train\n",
"3 CCDS5.1 666 train\n",
"4 CCDS6.1 498 test"
]
},
"execution_count": 62,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"non_erv2.head()"
]
},
{
"cell_type": "code",
"execution_count": 61,
"id": "c41821b0",
"metadata": {},
"outputs": [],
"source": [
"non_erv2['sequence_name'] = non_erv2['sequence_name'].str.split('|').str[0]"
]
},
{
"cell_type": "code",
"execution_count": 63,
"id": "efe354e3",
"metadata": {},
"outputs": [],
"source": [
"non_erv2.to_csv(\"non_erv_ccds_seq_split.csv\", index=False)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "6f804db6",
"metadata": {},
"outputs": [],
"source": [
"# len('CTCAATCTAACAACCAAAATGGACCAAACATTC\\\n",
"# GAGGCACAGCTCTACAAGGAGCTGAAGAAAATGGTGGCAGCAGATATAATCTCAGATCTG\\\n",
"# GACAGCATAGCAGAGATCAGACCGTACCAGATCCAGATCTCATCCAACCCACTGGTGATG\\\n",
"# GAAATAACCAAAACAGGGATGCTGAGAATAATAATGATGAAGATAAAGGAGACGGGAATG')"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "893aa572",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded 19565 sequences from ../sequences.fasta\n",
"Mean length: 29675.9 bp\n"
]
},
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "float64",
"type": "float"
}
],
"ref": "484e77e0-334f-41d0-a30f-b9fe74326f43",
"rows": [
[
"count",
"19565.0"
],
[
"mean",
"29675.860005111168"
],
[
"std",
"71874.09261941054"
],
[
"min",
"136.0"
],
[
"25%",
"2752.0"
],
[
"50%",
"6868.0"
],
[
"75%",
"39648.0"
],
[
"max",
"2473870.0"
]
],
"shape": {
"columns": 1,
"rows": 8
}
},
"text/plain": [
"count 1.956500e+04\n",
"mean 2.967586e+04\n",
"std 7.187409e+04\n",
"min 1.360000e+02\n",
"25% 2.752000e+03\n",
"50% 6.868000e+03\n",
"75% 3.964800e+04\n",
"max 2.473870e+06\n",
"Name: sequence_length, dtype: float64"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"virus_nc = fasta_to_dataframe(\"../sequences.fasta\")\n",
"virus_nc['sequence_length'].describe()"
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "270b6f1e",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "group",
"rawType": "str",
"type": "string"
},
{
"name": "count",
"rawType": "int64",
"type": "integer"
}
],
"ref": "6ccc555e-a9c7-4fe2-a75f-b36ef819b980",
"rows": [
[
"train",
"17609"
],
[
"test",
"1956"
]
],
"shape": {
"columns": 1,
"rows": 2
}
},
"text/plain": [
"group\n",
"train 17609\n",
"test 1956\n",
"Name: count, dtype: int64"
]
},
"execution_count": 58,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"virus_nc['group'] = 'train'\n",
"test_idx = virus_nc.sample(frac=0.1, random_state=42).index\n",
"virus_nc.loc[test_idx, 'group'] = 'test'\n",
"\n",
"virus_nc['group'].value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "5cfbec06",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.microsoft.datawrangler.viewer.v0+json": {
"columns": [
{
"name": "index",
"rawType": "int64",
"type": "integer"
},
{
"name": "sequence_name",
"rawType": "str",
"type": "string"
},
{
"name": "sequence_length",
"rawType": "int64",
"type": "integer"
},
{
"name": "group",
"rawType": "str",
"type": "string"
}
],
"ref": "aa91ff10-3e0b-4942-b4c7-b8e62642b885",
"rows": [
[
"0",
"NC_115729.1",
"367",
"train"
],
[
"1",
"NC_117067.1",
"2353",
"train"
],
[
"2",
"NC_116595.1",
"19471",
"train"
],
[
"3",
"NC_116736.1",
"15665",
"test"
],
[
"4",
"NC_116742.1",
"15720",
"train"
]
],
"shape": {
"columns": 3,
"rows": 5
}
},
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>sequence_name</th>\n",
" <th>sequence_length</th>\n",
" <th>group</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>NC_115729.1</td>\n",
" <td>367</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NC_117067.1</td>\n",
" <td>2353</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>NC_116595.1</td>\n",
" <td>19471</td>\n",
" <td>train</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>NC_116736.1</td>\n",
" <td>15665</td>\n",
" <td>test</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>NC_116742.1</td>\n",
" <td>15720</td>\n",
" <td>train</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" sequence_name sequence_length group\n",
"0 NC_115729.1 367 train\n",
"1 NC_117067.1 2353 train\n",
"2 NC_116595.1 19471 train\n",
"3 NC_116736.1 15665 test\n",
"4 NC_116742.1 15720 train"
]
},
"execution_count": 59,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"virus_nc.head()"
]
},
{
"cell_type": "code",
"execution_count": 64,
"id": "5c0555ed",
"metadata": {},
"outputs": [],
"source": [
"virus_nc.to_csv(\"virus_seq_split.csv\", index=False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "vp2",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.14.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

Xet Storage Details

Size:
43.4 kB
·
Xet hash:
c935e68ea22a871505a05a810364e3e027dc033670fa6f5842b65cd8c952df4c

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.