test

.gitattributes

@@ -25,3 +25,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/** filter=lfs diff=lfs merge=lfs -text
+AbstractGenerator/** filter=lfs diff=lfs merge=lfs -text

AbstractGenerator.ipynb

@@ -0,0 +1,434 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING:tensorflow:From C:\\Users\\franz\\AppData\\Local\\Temp\\ipykernel_14092\\1198363771.py:6: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Use `tf.config.list_physical_devices('GPU')` instead.\n",
+      "GPU is available\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import gpt_2_simple as gpt2\n",
+    "import os\n",
+    "import tensorflow as tf\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "print(\"GPU is\", \"available\" if tf.test.is_gpu_available() else \"NOT AVAILABLE\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = \"124M\"\n",
+    "if not os.path.isdir(os.path.join(\"models\", model_name)):\n",
+    "\tprint(f\"Downloading {model_name} model...\")\n",
+    "\tgpt2.download_gpt2(model_name=model_name)  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path = 'AbstractGenerator/'\n",
+    "checkpoint_dir =path+'weights/'\n",
+    "data_path = path+'TrainigData/'\n",
+    "\n",
+    "\n",
+    "\n",
+    "file_name_en = 'en'\n",
+    "file_path_en = data_path+file_name_en\n",
+    "\n",
+    "file_name_es = 'es'\n",
+    "file_path_es = data_path+file_name_es\n",
+    "\n",
+    "\n",
+    "prefix= '<|startoftext|>'\n",
+    "sufix ='<|endoftext|>'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# create trainig data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "en = pd.read_csv('CSV\\scientific_paper_en.csv')[0:1000]\n",
+    "es = pd.read_csv('CSV\\scientific_paper_es.csv')[0:1000]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import codecs\n",
+    "def createTrainingData(ds,fileName= 'resumen.txt' ,path ='TrainigData/'):\n",
+    "    with codecs.open(path+fileName,'a','utf-8') as f:\n",
+    "        for i in ds.index:\n",
+    "            f.write(prefix+\"\\n\")\n",
+    "            f.write(ds.iloc[i]['text_no_abstract'])\n",
+    "            f.write(\"ABSTRACT\\n\")\n",
+    "            f.write(ds.iloc[i]['abstract']+\"\\n\")\n",
+    "            f.write(sufix)\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "createTrainingData(en,'en.txt',data_path)\n",
+    "createTrainingData(es,'es.txt',data_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# pretrained"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sess = gpt2.start_tf_sess()\n",
+    "gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# train "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tf.compat.v1.reset_default_graph()\n",
+    "sess = gpt2.start_tf_sess()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## en"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpt2.finetune(sess,\n",
+    "              file_path_en+'.txt',\n",
+    "              model_name=model_name,\n",
+    "              checkpoint_dir=checkpoint_dir,   \n",
+    "              steps=1000\n",
+    "              )   "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## es"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading checkpoint models\\124M\\model.ckpt\n",
+      "INFO:tensorflow:Restoring parameters from models\\124M\\model.ckpt\n",
+      "Loading dataset...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1/1 [00:51<00:00, 51.03s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dataset has 17511492 tokens\n",
+      "Training...\n"
+     ]
+    },
+    {
+     "ename": "ResourceExhaustedError",
+     "evalue": "Graph execution error:\n\nfailed to allocate memory\n\t [[{{node model/h10/attn/ArithmeticOptimizer/ReorderCastLikeAndValuePreserving_float_Cast_1}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1377\u001b[0m, in \u001b[0;36mBaseSession._do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1375'>1376</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1376'>1377</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m fn(\u001b[39m*\u001b[39;49margs)\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1377'>1378</a>\u001b[0m \u001b[39mexcept\u001b[39;00m errors\u001b[39m.\u001b[39mOpError \u001b[39mas\u001b[39;00m e:\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1360\u001b[0m, in \u001b[0;36mBaseSession._do_run.<locals>._run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1358'>1359</a>\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_extend_graph()\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1359'>1360</a>\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call_tf_sessionrun(options, feed_dict, fetch_list,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1360'>1361</a>\u001b[0m                                 target_list, run_metadata)\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1453\u001b[0m, in \u001b[0;36mBaseSession._call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1450'>1451</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_call_tf_sessionrun\u001b[39m(\u001b[39mself\u001b[39m, options, feed_dict, fetch_list, target_list,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1451'>1452</a>\u001b[0m                         run_metadata):\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1452'>1453</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m tf_session\u001b[39m.\u001b[39;49mTF_SessionRun_wrapper(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_session, options, feed_dict,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1453'>1454</a>\u001b[0m                                           fetch_list, target_list,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1454'>1455</a>\u001b[0m                                           run_metadata)\n",
+      "\u001b[1;31mResourceExhaustedError\u001b[0m: failed to allocate memory\n\t [[{{node model/h10/attn/ArithmeticOptimizer/ReorderCastLikeAndValuePreserving_float_Cast_1}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
+      "\u001b[1;32mc:\\Users\\franz\\OneDrive\\Documentos\\GitHub\\Generador-de-abstracts\\AbstractGenerator.ipynb Cell 15'\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=0'>1</a>\u001b[0m gpt2\u001b[39m.\u001b[39;49mfinetune(sess,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=1'>2</a>\u001b[0m               file_path_es\u001b[39m+\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m.txt\u001b[39;49m\u001b[39m'\u001b[39;49m,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=2'>3</a>\u001b[0m               model_name\u001b[39m=\u001b[39;49mmodel_name,\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=3'>4</a>\u001b[0m               checkpoint_dir\u001b[39m=\u001b[39;49mcheckpoint_dir,   \n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=4'>5</a>\u001b[0m               steps\u001b[39m=\u001b[39;49m\u001b[39m1000\u001b[39;49m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/franz/OneDrive/Documentos/GitHub/Generador-de-abstracts/AbstractGenerator.ipynb#ch0000014?line=5'>6</a>\u001b[0m               )\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\gpt_2_simple\\gpt_2.py:339\u001b[0m, in \u001b[0;36mfinetune\u001b[1;34m(sess, dataset, steps, model_name, model_dir, combine, batch_size, learning_rate, accumulate_gradients, restore_from, run_name, checkpoint_dir, sample_every, sample_length, sample_num, multi_gpu, save_every, print_every, max_checkpoints, use_memory_saving_gradients, only_train_transformer_layers, optimizer, overwrite, reuse)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=336'>337</a>\u001b[0m     sess\u001b[39m.\u001b[39mrun(opt_reset)\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=337'>338</a>\u001b[0m     \u001b[39mfor\u001b[39;00m _ \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(accumulate_gradients):\n\u001b[1;32m--> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=338'>339</a>\u001b[0m         sess\u001b[39m.\u001b[39;49mrun(\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=339'>340</a>\u001b[0m             opt_compute, feed_dict\u001b[39m=\u001b[39;49m{context: sample_batch()})\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=340'>341</a>\u001b[0m     (v_loss, v_summary) \u001b[39m=\u001b[39m sess\u001b[39m.\u001b[39mrun((opt_apply, summary_loss))\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/gpt_2_simple/gpt_2.py?line=341'>342</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:967\u001b[0m, in \u001b[0;36mBaseSession.run\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=963'>964</a>\u001b[0m run_metadata_ptr \u001b[39m=\u001b[39m tf_session\u001b[39m.\u001b[39mTF_NewBuffer() \u001b[39mif\u001b[39;00m run_metadata \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=965'>966</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=966'>967</a>\u001b[0m   result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run(\u001b[39mNone\u001b[39;49;00m, fetches, feed_dict, options_ptr,\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=967'>968</a>\u001b[0m                      run_metadata_ptr)\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=968'>969</a>\u001b[0m   \u001b[39mif\u001b[39;00m run_metadata:\n\u001b[0;32m    <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=969'>970</a>\u001b[0m     proto_data \u001b[39m=\u001b[39m tf_session\u001b[39m.\u001b[39mTF_GetBuffer(run_metadata_ptr)\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1190\u001b[0m, in \u001b[0;36mBaseSession._run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1186'>1187</a>\u001b[0m \u001b[39m# We only want to really perform the run if fetches or targets are provided,\u001b[39;00m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1187'>1188</a>\u001b[0m \u001b[39m# or if the call is a partial run that specifies feeds.\u001b[39;00m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1188'>1189</a>\u001b[0m \u001b[39mif\u001b[39;00m final_fetches \u001b[39mor\u001b[39;00m final_targets \u001b[39mor\u001b[39;00m (handle \u001b[39mand\u001b[39;00m feed_dict_tensor):\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1189'>1190</a>\u001b[0m   results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_do_run(handle, final_targets, final_fetches,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1190'>1191</a>\u001b[0m                          feed_dict_tensor, options, run_metadata)\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1191'>1192</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1192'>1193</a>\u001b[0m   results \u001b[39m=\u001b[39m []\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1370\u001b[0m, in \u001b[0;36mBaseSession._do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1366'>1367</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call_tf_sessionprun(handle, feed_dict, fetch_list)\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1368'>1369</a>\u001b[0m \u001b[39mif\u001b[39;00m handle \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1369'>1370</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1370'>1371</a>\u001b[0m                        run_metadata)\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1371'>1372</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1372'>1373</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_do_call(_prun_fn, handle, feeds, fetches)\n",
+      "File \u001b[1;32m~\\.conda\\envs\\tf-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1396\u001b[0m, in \u001b[0;36mBaseSession._do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1390'>1391</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39monly supports NHWC tensor format\u001b[39m\u001b[39m'\u001b[39m \u001b[39min\u001b[39;00m message:\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1391'>1392</a>\u001b[0m   message \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m (\u001b[39m'\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mA possible workaround: Try disabling Grappler optimizer\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1392'>1393</a>\u001b[0m               \u001b[39m'\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mby modifying the config for creating the session eg.\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1393'>1394</a>\u001b[0m               \u001b[39m'\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39msession_config.graph_options.rewrite_options.\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m   <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1394'>1395</a>\u001b[0m               \u001b[39m'\u001b[39m\u001b[39mdisable_meta_optimizer = True\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m-> <a href='file:///c%3A/Users/franz/.conda/envs/tf-gpu/lib/site-packages/tensorflow/python/client/session.py?line=1395'>1396</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mtype\u001b[39m(e)(node_def, op, message)\n",
+      "\u001b[1;31mResourceExhaustedError\u001b[0m: Graph execution error:\n\nfailed to allocate memory\n\t [[{{node model/h10/attn/ArithmeticOptimizer/ReorderCastLikeAndValuePreserving_float_Cast_1}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode."
+     ]
+    }
+   ],
+   "source": [
+    "gpt2.finetune(sess,\n",
+    "              file_path_es+'.txt',\n",
+    "              model_name=model_name,\n",
+    "              checkpoint_dir=checkpoint_dir,   \n",
+    "              steps=1000\n",
+    "              )   "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# test"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## en "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"\"\"Introduction and preliminaries\n",
+    "The focus of this paper is decompositions of (k, `)-sparse graphs into edge-disjoint subgraphs\n",
+    "that certify sparsity. We use graph to mean a multigraph, possibly with loops. We say that a\n",
+    "graph is (k, `)-sparse if no subset of n′ vertices spans more than kn′− ` edges in the graph; a\n",
+    "(k, `)-sparse graph with kn′− ` edges is (k, `)-tight. We call the range k ≤ `≤ 2k−1 the upper\n",
+    "range of sparse graphs and 0≤ `≤ k the lower range.\n",
+    "In this paper, we present efficient algorithms for finding decompositions that certify sparsity\n",
+    "in the upper range of `. Our algorithms also apply in the lower range, which was already ad-\n",
+    "dressed by [3, 4, 5, 6, 19]. A decomposition certifies the sparsity of a graph if the sparse graphs\n",
+    "and graphs admitting the decomposition coincide.\n",
+    "Our algorithms are based on a new characterization of sparse graphs, which we call the\n",
+    "pebble game with colors. The pebble game with colors is a simple graph construction rule that\n",
+    "produces a sparse graph along with a sparsity-certifying decomposition.\n",
+    "We define and study a canonical class of pebble game constructions, which correspond to\n",
+    "previously studied decompositions of sparse graphs into edge disjoint trees. Our results provide\n",
+    "a unifying framework for all the previously known special cases, including Nash-Williams-\n",
+    "Tutte and [7, 24]. Indeed, in the lower range, canonical pebble game constructions capture the\n",
+    "properties of the augmenting paths used in matroid union and intersection algorithms[5, 6].\n",
+    "Since the sparse graphs in the upper range are not known to be unions or intersections of the\n",
+    "matroids for which there are efficient augmenting path algorithms, these do not easily apply in\n",
+    "∗ Research of both authors funded by the NSF under grants NSF CCF-0430990 and NSF-DARPA CARGO\n",
+    "CCR-0310661 to the first author.\n",
+    "2 Ileana Streinu, Louis Theran\n",
+    "Term Meaning\n",
+    "Sparse graph G Every non-empty subgraph on n′ vertices has ≤ kn′− ` edges\n",
+    "Tight graph G G = (V,E) is sparse and |V |= n, |E|= kn− `\n",
+    "Block H in G G is sparse, and H is a tight subgraph\n",
+    "Component H of G G is sparse and H is a maximal block\n",
+    "Map-graph Graph that admits an out-degree-exactly-one orientation\n",
+    "(k, `)-maps-and-trees Edge-disjoint union of ` trees and (k− `) map-grpahs\n",
+    "`Tk Union of ` trees, each vertex is in exactly k of them\n",
+    "Set of tree-pieces of an `Tk induced on V ′ ⊂V Pieces of trees in the `Tk spanned by E(V ′)\n",
+    "Proper `Tk Every V ′ ⊂V contains ≥ ` pieces of trees from the `Tk\n",
+    "Table 1. Sparse graph and decomposition terminology used in this paper.\n",
+    "the upper range. Pebble game with colors constructions may thus be considered a strengthening\n",
+    "of augmenting paths to the upper range of matroidal sparse graphs.\n",
+    "1.1. Sparse graphs\n",
+    "\n",
+    "ABSTRACT\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpt2.generate(sess,prefix=text,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## es"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"\"\"El foco de este documento son las descomposicións de (k, `)-sparse gráficos en bordes-disjunto subgraphs\n",
+    "que certifique la escasez. Usamos el gráfico para significar un múltiplo, posiblemente con bucles. Nosotros decimos que un\n",
+    "grafo es (k, `)-sparse si ningún subconjunto de n′ vértices abarca más de kn ` bordes en el gráfico; a\n",
+    "(k, `)-sparse gráfico con kn ` bordes es (k, `)-estrechado. Llamamos al rango k ≤  2k−1 el superior\n",
+    "rango de gráficos escasos y 0≤  k el rango inferior.\n",
+    "En este artículo, presentamos algoritmos eficientes para encontrar descomposicións que certifiquen la escasez\n",
+    "en el rango superior de `. Nuestros algoritmos también se aplican en el rango inferior, que ya era ad-\n",
+    "vestido por [3, 4, 5, 6, 19]. Una descomposición certifica la escasez de un gráfico si los gráficos dispersos\n",
+    "y los gráficos que admiten la descomposición coinciden.\n",
+    "Nuestros algoritmos se basan en una nueva caracterización de gráficos escasos, que llamamos el\n",
+    "juego de guijarros con colores. El juego de guijarros con colores es una regla de construcción de gráficos simples que\n",
+    "produce un gráfico escaso junto con una descomposición certificadora de la escasez.\n",
+    "Definimos y estudiamos una clase canónica de construcciones de juego de guijarros, que corresponden a\n",
+    "previamente estudiado las descomposiciones de los gráficos escasos en los árboles disjuntos del borde. Nuestros resultados proporcionan\n",
+    "un marco unificador para todos los casos especiales conocidos anteriormente, incluidos Nash-Williams-\n",
+    "Tutte y [7, 24]. De hecho, en el rango inferior, las construcciones canónicas de juego de guijarros capturan la\n",
+    "propiedades de las rutas de aumento utilizadas en los algoritmos de unión de matroides y de intersección[5, 6].\n",
+    "Dado que los gráficos escasos en el rango superior no se sabe que son uniones o intersecciones de la\n",
+    "matroides para los que hay algoritmos de ruta de aumento eficiente, estos no se aplican fácilmente en\n",
+    "* Investigación de ambos autores financiada por la NSF bajo subvenciones NSF CCF-0430990 y NSF-DARPA CARGO\n",
+    "CCR-0310661 al primer autor.\n",
+    "2 Ileana Streinu, Louis Theran\n",
+    "Significado del término\n",
+    "Gráfico escaso G Cada subgrafo no vacío en n′ vértices tiene ≤ kn ` bordes\n",
+    "El gráfico ajustado G G = (V,E) es escaso y V = n, E= kn− `\n",
+    "El bloque H en G G es escaso, y H es un subgrafo apretado\n",
+    "El componente H de G G es escaso y H es un bloqueo máximo\n",
+    "Gráfico cartográfico que admite una orientación de grado-exactamente-uno\n",
+    "(k, `)-maps-and-trees Edge-disjunt union de ` árboles y (k- `) map-grpahs\n",
+    "`Tk Unión de ` árboles, cada vértice está exactamente en k de ellos\n",
+    "Conjunto de piezas arbóreas de un `Tk inducido en V ′ ́V Piezas de árboles en el `Tk extendido por E(V ′)\n",
+    "`Tk Apropiado Cada V ′ V contiene ≥ ` pedazos de árboles de la `Tk\n",
+    "Cuadro 1 Gráfico escaso y terminología de descomposición utilizada en este artículo.\n",
+    "el rango superior. Pebble juego con construcciones de colores por lo tanto puede ser considerado un fortalecimiento\n",
+    "de caminos de aumento a la gama superior de gráficos de la escasez matroidal.\n",
+    "1.1. Gráficos escasos\n",
+    "Un gráfico es (k, `)-sparse si para cualquier subgrafo no vacío con bordes m′ y n′ vértices, m′ ≤\n",
+    "kn `. Observamos que esta condición implica que 0 ≤ ` ≤ 2k− 1, y a partir de ahora en este\n",
+    "Haremos esta suposición. Un gráfico escaso que tiene n vértices y exactamente bordes kn\n",
+    "se llama apretado.\n",
+    "Para un gráfico G = (V,E), y V ′  V, utilizamos el intervalo de notación (V ′) para el número de bordes\n",
+    "en el subgráfico inducido por V ′. En un gráfico dirigido, out(V ′) es el número de bordes con la cola\n",
+    "en V ′ y la cabeza en V −V ′; para un subgráfico inducido por V ′, llamamos a tal borde un borde superior.\n",
+    "Hay dos tipos importantes de subgrafías de gráficos escasos. Un bloque es un subgrafo apretado de\n",
+    "un gráfico escaso. Un componente es un bloque máximo.\n",
+    "La Tabla 1 resume la escasa terminología gráfica utilizada en este artículo.\n",
+    "1.2. Descomposiciónes de certificación de la sparsidad\n",
+    "Un k-arborescencia es un gráfico que admite una descomposición en k borde-desjunto que abarca los árboles.\n",
+    "La Figura 1(a) muestra un ejemplo de una 3-arborescencia. Se describen los gráficos k-arborescentes\n",
+    "por los conocidos teoremas de Tutte [23] y Nash-Williams [17] como exactamente el (k,k) apretado\n",
+    "gráficos.\n",
+    "ABSTRACT\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpt2.generate(sess,prefix=text,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# gradio interface"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generateAbstract(text):\n",
+    "   # with tf.compat.v1.variable_scope(\"weight\", reuse = True):\n",
+    "    #sess = tf.compat.v1.get_variable('sess',gpt2.start_tf_sess())\n",
+    "    tf.compat.v1.reset_default_graph()\n",
+    "    sess = gpt2.start_tf_sess()\n",
+    "    gpt2.load_gpt2(sess,checkpoint_dir=checkpoint_dir,run_name='run1')\n",
+    "    txt = gpt2.generate(sess,prefix=str(text)+\"\\nABSTRACT\", return_as_list=True,truncate=sufix,checkpoint_dir=checkpoint_dir,nsamples=1)[0]\n",
+    "    return str(txt[txt.find('ABSTRACT'):])\n",
+    "\n",
+    "\n",
+    "\n",
+    "iface = gr.Interface(fn=generateAbstract, inputs=gr.inputs.Textbox(lines=10, placeholder=\"text\"), outputs=\"textbox\")\n",
+    "iface.launch(debug = True )"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "53fbdc69e3e12c371950068c144423682c30d04ec68c2bd46937202e33e0058d"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.7.11 ('receta')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

AbstractGenerator/TrainigData/en.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:814f983aa49ccc33a993a7d12f67a2eb2a7ca0b15d8697e82b50d3a19f3e1595
+size 35400974

AbstractGenerator/TrainigData/es.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2454067cfe384e1d824b3f5d29cb5c4e1ff292289ad4b37c6cbd22f5cc715295
+size 44460970

AbstractGenerator/weights/run1/encoder.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783
+size 1042301

AbstractGenerator/weights/run1/events.out.tfevents.1648184225.FRANZ96521-W11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a88ba7f3268f11289fb24fd13db1367b91acce6466c4ad394011e10ea4c304
+size 82

AbstractGenerator/weights/run1/events.out.tfevents.1648184499.FRANZ96521-W11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb8646e6bf1e1b8cc26f8128ec4e4c2e797dac297939450a8bf46057e7388a6a
+size 82

AbstractGenerator/weights/run1/events.out.tfevents.1648229481.FRANZ96521-W11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04d1f71db542da83fee4fe8574bf382cb5324b6decef506206250b8fea85abd0
+size 82

AbstractGenerator/weights/run1/hparams.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d56e4121c427164e0c55c6f03c08e1daf9002b9b672825112d19097b680318
+size 90

AbstractGenerator/weights/run1/vocab.bpe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5
+size 456318

Descarga.ipynb

@@ -0,0 +1,278 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "from pandas import json_normalize\n",
+    "import requests\n",
+    "from pathlib import Path\n",
+    "from multiprocessing.pool import ThreadPool as Pool\n",
+    "import codecs\n",
+    "import random\n",
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "URL_BASE = \"https://arxiv.org/pdf/\"\n",
+    "PDF_PATH = 'PDF'\n",
+    "TXT_PATH= 'TXT'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Arxiv\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>title</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>Text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0001</td>\n",
+       "      <td>Calculation of prompt diphoton production cros...</td>\n",
+       "      <td>A fully differential calculation in perturba...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1996</th>\n",
+       "      <td>0704.1997</td>\n",
+       "      <td>Query on Negative Temperature, Internal Intera...</td>\n",
+       "      <td>After negative temperature is restated, we f...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1997</th>\n",
+       "      <td>0704.1998</td>\n",
+       "      <td>Absence of the Fifth Force Problem in a Model ...</td>\n",
+       "      <td>A scale invariant model containing dilaton $...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1998</th>\n",
+       "      <td>0704.1999</td>\n",
+       "      <td>Dark matter caustics and the enhancement of se...</td>\n",
+       "      <td>Cold dark matter haloes are populated by cau...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1999</th>\n",
+       "      <td>0704.2000</td>\n",
+       "      <td>Search for a Higgs boson produced in associati...</td>\n",
+       "      <td>We describe a search for the standard model ...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2000</th>\n",
+       "      <td>0704.2001</td>\n",
+       "      <td>Geometry of Parallelizable Manifolds in the Co...</td>\n",
+       "      <td>In this paper, we deal with a generalization...</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2001 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             id                                              title  \\\n",
+       "0     0704.0001  Calculation of prompt diphoton production cros...   \n",
+       "1     0704.0002           Sparsity-certifying Graph Decompositions   \n",
+       "2     0704.0003  The evolution of the Earth-Moon system based o...   \n",
+       "3     0704.0004  A determinant of Stirling cycle numbers counts...   \n",
+       "4     0704.0005  From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "...         ...                                                ...   \n",
+       "1996  0704.1997  Query on Negative Temperature, Internal Intera...   \n",
+       "1997  0704.1998  Absence of the Fifth Force Problem in a Model ...   \n",
+       "1998  0704.1999  Dark matter caustics and the enhancement of se...   \n",
+       "1999  0704.2000  Search for a Higgs boson produced in associati...   \n",
+       "2000  0704.2001  Geometry of Parallelizable Manifolds in the Co...   \n",
+       "\n",
+       "                                               abstract Text  \n",
+       "0       A fully differential calculation in perturba...       \n",
+       "1       We describe a new algorithm, the $(k,\\ell)$-...       \n",
+       "2       The evolution of Earth-Moon system is descri...       \n",
+       "3       We show that a determinant of Stirling cycle...       \n",
+       "4       In this paper we show how to compute the $\\L...       \n",
+       "...                                                 ...  ...  \n",
+       "1996    After negative temperature is restated, we f...       \n",
+       "1997    A scale invariant model containing dilaton $...       \n",
+       "1998    Cold dark matter haloes are populated by cau...       \n",
+       "1999    We describe a search for the standard model ...       \n",
+       "2000    In this paper, we deal with a generalization...       \n",
+       "\n",
+       "[2001 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = pd.read_json('ARxiv/arxiv-metadata-oai-snapshot.json',lines=True, chunksize=2001,dtype={'id':'str'})\n",
+    "df = None\n",
+    "for i in data:\n",
+    "    df = i \n",
+    "    print(type(i))\n",
+    "    break\n",
+    "df = df[['id','title','abstract']]\n",
+    "df.insert(3, \"Text\", \"\") \n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def GetFileURL(file_id):\n",
+    "    url = URL_BASE+file_id\n",
+    "    r = requests.get(url, stream=True)    \n",
+    "    filename = Path(PDF_PATH+'/'+file_id+'.pdf')\n",
+    "    response = requests.get(url)\n",
+    "    filename.write_bytes(response.content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pool_size = 16  \n",
+    "def worker(file):\n",
+    "    try:\n",
+    "        GetFileURL(file)\n",
+    "    except:\n",
+    "        print('error with item '+ file)\n",
+    "        try:\n",
+    "            with codecs.open(PDF_PATH+'/log.txt', 'a') as the_file:        \n",
+    "                the_file.writelines(str(file)+\"\\n\")\n",
+    "        except:\n",
+    "            print('error en log '+ file)\n",
+    "def get_ids(iteracion,batch=100):    \n",
+    "    inicio = int(iteracion*batch)\n",
+    "    filesId = data[inicio :inicio + batch]['id']\n",
+    "    return filesId\n",
+    "\n",
+    "pool = Pool(pool_size)\n",
+    "filesId = get_ids(19)\n",
+    "for file in filesId:\n",
+    "    pool.apply_async(worker, (file,))\n",
+    "\n",
+    "pool.close()\n",
+    "pool.join()"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "3f7e9d73c32ad96f75174922c475a50b168aad887cbaa14717912a88f31d3802"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.7 ('tf-gpu')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

PDF_a_TXT.ipynb

@@ -0,0 +1,105 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: PyPDF2 in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (1.26.0)\n",
+      "Requirement already satisfied: tika in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (1.24)\n",
+      "Requirement already satisfied: requests in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from tika) (2.27.1)\n",
+      "Requirement already satisfied: setuptools in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from tika) (58.0.4)\n",
+      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from requests->tika) (2.0.4)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from requests->tika) (2021.10.8)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from requests->tika) (3.3)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\franz\\.conda\\envs\\tensorflow\\lib\\site-packages (from requests->tika) (1.26.8)\n"
+     ]
+    }
+   ],
+   "source": [
+    "! pip install PyPDF2\n",
+    "! pip install tika"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tika import parser\n",
+    "import codecs\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def obtener_texto(file_path,store_path):\n",
+    "    file_data = parser.from_file(file_path)\n",
+    "    output = file_data['content']\n",
+    "    output = output.strip()   \n",
+    "    output= output.split('\\n')\n",
+    "    with codecs.open(store_path+'.txt', 'w','utf-8') as the_file: \n",
+    "        for line in output:\n",
+    "            #print(line)\n",
+    "            if len(line)>4:       \n",
+    "                the_file.write(str(line)+'\\n')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-03-17 17:02:20,018 [MainThread  ] [WARNI]  Failed to see startup log message; retrying...\n"
+     ]
+    }
+   ],
+   "source": [
+    "PDF_PATH = 'PDF'\n",
+    "TXT_PATH= 'TXT'\n",
+    "files  = os.listdir(PDF_PATH)\n",
+    "for  file in  files:\n",
+    "    obtener_texto(PDF_PATH+'/'+file,TXT_PATH+'/'+file)\n",
+    " "
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "3f7e9d73c32ad96f75174922c475a50b168aad887cbaa14717912a88f31d3802"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.7 ('tf-gpu')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

models/124M/checkpoint

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd1b025d2e155283f5e300ce95bf6d5b6bc0f7fe010db73daa6975eb896ab9cb
+size 77

models/124M/encoder.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783
+size 1042301

models/124M/hparams.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d56e4121c427164e0c55c6f03c08e1daf9002b9b672825112d19097b680318
+size 90

models/124M/model.ckpt.data-00000-of-00001

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2060c885360cc0cf41d7a6dbc4d24b5127aae20260c8b5ae521b5a6578407118
+size 497759232

models/124M/model.ckpt.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71916f763f9746f9b2a06b12d91996cf1084ae008d0424543d39391c5f2dc687
+size 5215

models/124M/model.ckpt.meta

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4668c448fa11531fd6700460487f73e82d3272960cea942252f8744bf225c77b
+size 471155

models/124M/vocab.bpe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5
+size 456318

txt_to_csv.ipynb

@@ -0,0 +1,662 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import os\n",
+    "from easynmt import EasyNMT\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "URL_BASE = \"https://arxiv.org/pdf/\"\n",
+    "PDF_PATH = 'PDF'\n",
+    "TXT_PATH= 'TXT'\n",
+    "CSV_PATH = 'CSV'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Get Data from TXT"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = pd.read_json('ARxiv/arxiv-metadata-oai-snapshot.json',lines=True, chunksize=2001,dtype={'id':'str'})\n",
+    "df = None\n",
+    "for i in data:\n",
+    "    df = i \n",
+    "    print(type(i))\n",
+    "    break\n",
+    "df = df[['id','title','abstract']]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for file in df['id']:\n",
+    "    file_path = TXT_PATH+'/'+str(file)+'.pdf.txt'\n",
+    "    if  os.path.isfile(file_path):\n",
+    "        with open(file_path,'r',encoding='utf8') as f:\n",
+    "            s =str( f.read()) \n",
+    "            df.loc[df['id'] == str(file),'full_text'] = s          "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.dropna()\n",
+    "df.reset_index()\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_en.csv',index=False,encoding='utf-8')\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# first run \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(CSV_PATH +'/scientific_paper_en.csv',dtype={'id':'str'})\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# leer datos"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(CSV_PATH +'/scientific_paper_full_text_translated.csv',dtype={'id':'str'})\n",
+    "print(len(df.index))\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# translate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model = EasyNMT('opus-mt')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## translate full text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max = len(df.index)\n",
+    "for i in range(0,1754):\n",
+    "    text = df.iloc[i]['full_text']\n",
+    "    translated_text = model.translate(text, target_lang='es')\n",
+    "    df.loc[i,'translated'] = translated_text   \n",
+    "    print(\"listo documento \",i)\n",
+    "    if(i%10==0):\n",
+    "        df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## translate abstract"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max = len(df.index)\n",
+    "for i in range(0,1754):\n",
+    "    text = df.iloc[i]['abstract']\n",
+    "    translated_text = model.translate(text, target_lang='es')\n",
+    "    df.loc[i,'translated_abstract'] = translated_text   \n",
+    "    print(\"listo documento \",i)\n",
+    "    if(i%100==0):\n",
+    "        df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# remove abstract"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max = len(df.index)-1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "end = 'Introducción'\n",
+    "for i in range(0,max):\n",
+    "    text = df.iloc[i]['translated']    \n",
+    "    p = text.find(end)\n",
+    "    if(p != -1):        \n",
+    "        df.loc[i,'translated_no_abstract'] = text[p:]   \n",
+    "    else:\n",
+    "        df.loc[i,'translated_no_abstract']= text\n",
+    "    print(\"listo documento \",i,p)\n",
+    "    if(i%1000==0):\n",
+    "        df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "end = 'Abstract'\n",
+    "for i in range(0,max):\n",
+    "    text = df.iloc[i]['full_text']    \n",
+    "    p = text.find(end)\n",
+    "    if(p != -1):        \n",
+    "        df.loc[i,'text_no_abstract'] = text[p:]   \n",
+    "    else:\n",
+    "        df.loc[i,'text_no_abstract']= text    \n",
+    "    if(i%1000==0):\n",
+    "        df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')\n",
+    "df.to_csv(CSV_PATH+'/scientific_paper_full_text_translated.csv',index=False,encoding='utf-8')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# split data to csv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(CSV_PATH +'/scientific_paper_full_text_translated.csv',dtype={'id':'str'})\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>title</th>\n",
+       "      <th>full_text</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>text_no_abstract</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>Descomposiciones del gráfico de certificación ...</td>\n",
+       "      <td>Describimos un nuevo algoritmo, el juego de ...</td>\n",
+       "      <td>Introducción y preliminares\\nEl foco de este d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>La evolución del sistema Tierra-Luna basado en...</td>\n",
+       "      <td>La evolución del sistema Tierra-Luna es desc...</td>\n",
+       "      <td>Introducción \\nLa teoría aceptada popularmente...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>Un determinante de los números de ciclo de Sti...</td>\n",
+       "      <td>Demostramos que un determinante de los númer...</td>\n",
+       "      <td>Introducción El propósito principal de este ar...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>DE DÍA A DÍA\\nWAEL ABU-SHAMMALA Y ALBERTO TORC...</td>\n",
+       "      <td>En este artículo mostramos cómo calcular la ...</td>\n",
+       "      <td>DE DÍA A DÍA\\nWAEL ABU-SHAMMALA Y ALBERTO TORC...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0007</td>\n",
+       "      <td>Polymer Quantum Mechanics and its Continuum Limit</td>\n",
+       "      <td>La mecánica cuántica de polímeros y su límite ...</td>\n",
+       "      <td>Una representación cuántica no estándar de l...</td>\n",
+       "      <td>La mecánica cuántica de polímeros y su límite ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1749</th>\n",
+       "      <td>0704.1996</td>\n",
+       "      <td>A Wave-function for Stringy Universes</td>\n",
+       "      <td>LPTENS–07/16\\nAbril de 2007\\nUna función de on...</td>\n",
+       "      <td>Definimos una función de onda para los fondo...</td>\n",
+       "      <td>Introducción\\nNuestro objetivo en este documen...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1750</th>\n",
+       "      <td>0704.1997</td>\n",
+       "      <td>Query on Negative Temperature, Internal Intera...</td>\n",
+       "      <td>Microsoft Word - negEntr.doc\\nConsulta sobre t...</td>\n",
+       "      <td>Después de que la temperatura negativa se vu...</td>\n",
+       "      <td>Microsoft Word - negEntr.doc\\nConsulta sobre t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1751</th>\n",
+       "      <td>0704.1998</td>\n",
+       "      <td>Absence of the Fifth Force Problem in a Model ...</td>\n",
+       "      <td>Ausencia del problema de la quinta fuerza en u...</td>\n",
+       "      <td>Un modelo de escala invariante que contiene ...</td>\n",
+       "      <td>Introducción\\n\\tBase de Dos Medidas Teoría de ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1752</th>\n",
+       "      <td>0704.1999</td>\n",
+       "      <td>Dark matter caustics and the enhancement of se...</td>\n",
+       "      <td>Proyecto de versión 16 de noviembre de 2018\\nT...</td>\n",
+       "      <td>Los haloes fríos de materia oscura están pob...</td>\n",
+       "      <td>Proyecto de versión 16 de noviembre de 2018\\nT...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1753</th>\n",
+       "      <td>0704.2000</td>\n",
+       "      <td>Search for a Higgs boson produced in associati...</td>\n",
+       "      <td>FERMILAB-PUB-07/076-E\\nBúsqueda de un bosón Hi...</td>\n",
+       "      <td>Describimos una búsqueda para el modelo está...</td>\n",
+       "      <td>FERMILAB-PUB-07/076-E\\nBúsqueda de un bosón Hi...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1754 rows × 5 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             id                                              title  \\\n",
+       "0     0704.0002           Sparsity-certifying Graph Decompositions   \n",
+       "1     0704.0003  The evolution of the Earth-Moon system based o...   \n",
+       "2     0704.0004  A determinant of Stirling cycle numbers counts...   \n",
+       "3     0704.0005  From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "4     0704.0007  Polymer Quantum Mechanics and its Continuum Limit   \n",
+       "...         ...                                                ...   \n",
+       "1749  0704.1996              A Wave-function for Stringy Universes   \n",
+       "1750  0704.1997  Query on Negative Temperature, Internal Intera...   \n",
+       "1751  0704.1998  Absence of the Fifth Force Problem in a Model ...   \n",
+       "1752  0704.1999  Dark matter caustics and the enhancement of se...   \n",
+       "1753  0704.2000  Search for a Higgs boson produced in associati...   \n",
+       "\n",
+       "                                              full_text  \\\n",
+       "0     Descomposiciones del gráfico de certificación ...   \n",
+       "1     La evolución del sistema Tierra-Luna basado en...   \n",
+       "2     Un determinante de los números de ciclo de Sti...   \n",
+       "3     DE DÍA A DÍA\\nWAEL ABU-SHAMMALA Y ALBERTO TORC...   \n",
+       "4     La mecánica cuántica de polímeros y su límite ...   \n",
+       "...                                                 ...   \n",
+       "1749  LPTENS–07/16\\nAbril de 2007\\nUna función de on...   \n",
+       "1750  Microsoft Word - negEntr.doc\\nConsulta sobre t...   \n",
+       "1751  Ausencia del problema de la quinta fuerza en u...   \n",
+       "1752  Proyecto de versión 16 de noviembre de 2018\\nT...   \n",
+       "1753  FERMILAB-PUB-07/076-E\\nBúsqueda de un bosón Hi...   \n",
+       "\n",
+       "                                               abstract  \\\n",
+       "0       Describimos un nuevo algoritmo, el juego de ...   \n",
+       "1       La evolución del sistema Tierra-Luna es desc...   \n",
+       "2       Demostramos que un determinante de los númer...   \n",
+       "3       En este artículo mostramos cómo calcular la ...   \n",
+       "4       Una representación cuántica no estándar de l...   \n",
+       "...                                                 ...   \n",
+       "1749    Definimos una función de onda para los fondo...   \n",
+       "1750    Después de que la temperatura negativa se vu...   \n",
+       "1751    Un modelo de escala invariante que contiene ...   \n",
+       "1752    Los haloes fríos de materia oscura están pob...   \n",
+       "1753    Describimos una búsqueda para el modelo está...   \n",
+       "\n",
+       "                                       text_no_abstract  \n",
+       "0     Introducción y preliminares\\nEl foco de este d...  \n",
+       "1     Introducción \\nLa teoría aceptada popularmente...  \n",
+       "2     Introducción El propósito principal de este ar...  \n",
+       "3     DE DÍA A DÍA\\nWAEL ABU-SHAMMALA Y ALBERTO TORC...  \n",
+       "4     La mecánica cuántica de polímeros y su límite ...  \n",
+       "...                                                 ...  \n",
+       "1749  Introducción\\nNuestro objetivo en este documen...  \n",
+       "1750  Microsoft Word - negEntr.doc\\nConsulta sobre t...  \n",
+       "1751  Introducción\\n\\tBase de Dos Medidas Teoría de ...  \n",
+       "1752  Proyecto de versión 16 de noviembre de 2018\\nT...  \n",
+       "1753  FERMILAB-PUB-07/076-E\\nBúsqueda de un bosón Hi...  \n",
+       "\n",
+       "[1754 rows x 5 columns]"
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "es = df[['id','title','translated','translated_abstract','translated_no_abstract']]\n",
+    "es.columns = [\"id\",\"title\", \"full_text\",\"abstract\",\"text_no_abstract\"]\n",
+    "es.to_csv(CSV_PATH+'/scientific_paper_es.csv',index=False,encoding='utf-8')\n",
+    "es"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>title</th>\n",
+       "      <th>full_text</th>\n",
+       "      <th>abstract</th>\n",
+       "      <th>text_no_abstract</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0704.0002</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
+       "      <td>Sparsity-certifying Graph Decompositions\\nIlea...</td>\n",
+       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
+       "      <td>Introduction and preliminaries\\nThe focus of t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0704.0003</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
+       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
+       "      <td>Introduction \\nThe popularly accepted theory f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0704.0004</td>\n",
+       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
+       "      <td>A Determinant of Stirling Cycle Numbers Counts...</td>\n",
+       "      <td>We show that a determinant of Stirling cycle...</td>\n",
+       "      <td>Introduction The chief purpose of this paper i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0704.0005</td>\n",
+       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
+       "      <td>FROM DYADIC Λα TO Λα\\nWAEL ABU-SHAMMALA AND AL...</td>\n",
+       "      <td>In this paper we show how to compute the $\\L...</td>\n",
+       "      <td>FROM DYADIC Λα TO Λα\\nWAEL ABU-SHAMMALA AND AL...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0704.0007</td>\n",
+       "      <td>Polymer Quantum Mechanics and its Continuum Limit</td>\n",
+       "      <td>Polymer Quantum Mechanics and its Continuum Li...</td>\n",
+       "      <td>A rather non-standard quantum representation...</td>\n",
+       "      <td>Polymer Quantum Mechanics and its Continuum Li...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1749</th>\n",
+       "      <td>0704.1996</td>\n",
+       "      <td>A Wave-function for Stringy Universes</td>\n",
+       "      <td>LPTENS–07/16\\nApril 2007\\nA Wave-function for ...</td>\n",
+       "      <td>We define a wave-function for string theory ...</td>\n",
+       "      <td>Introduction\\nOur goal in this paper is to emb...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1750</th>\n",
+       "      <td>0704.1997</td>\n",
+       "      <td>Query on Negative Temperature, Internal Intera...</td>\n",
+       "      <td>Microsoft Word - negEntr.doc\\nQuery on Negativ...</td>\n",
+       "      <td>After negative temperature is restated, we f...</td>\n",
+       "      <td>Microsoft Word - negEntr.doc\\nQuery on Negativ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1751</th>\n",
+       "      <td>0704.1998</td>\n",
+       "      <td>Absence of the Fifth Force Problem in a Model ...</td>\n",
+       "      <td>Absence of the Fifth Force Problem in a Model ...</td>\n",
+       "      <td>A scale invariant model containing dilaton $...</td>\n",
+       "      <td>Introduction\\n\\tBasis of Two Measures Field Th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1752</th>\n",
+       "      <td>0704.1999</td>\n",
+       "      <td>Dark matter caustics and the enhancement of se...</td>\n",
+       "      <td>Draft version November 16, 2018\\nPreprint type...</td>\n",
+       "      <td>Cold dark matter haloes are populated by cau...</td>\n",
+       "      <td>Draft version November 16, 2018\\nPreprint type...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1753</th>\n",
+       "      <td>0704.2000</td>\n",
+       "      <td>Search for a Higgs boson produced in associati...</td>\n",
+       "      <td>FERMILAB-PUB-07/076-E\\nSearch for a Higgs boso...</td>\n",
+       "      <td>We describe a search for the standard model ...</td>\n",
+       "      <td>FERMILAB-PUB-07/076-E\\nSearch for a Higgs boso...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1754 rows × 5 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "             id                                              title  \\\n",
+       "0     0704.0002           Sparsity-certifying Graph Decompositions   \n",
+       "1     0704.0003  The evolution of the Earth-Moon system based o...   \n",
+       "2     0704.0004  A determinant of Stirling cycle numbers counts...   \n",
+       "3     0704.0005  From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
+       "4     0704.0007  Polymer Quantum Mechanics and its Continuum Limit   \n",
+       "...         ...                                                ...   \n",
+       "1749  0704.1996              A Wave-function for Stringy Universes   \n",
+       "1750  0704.1997  Query on Negative Temperature, Internal Intera...   \n",
+       "1751  0704.1998  Absence of the Fifth Force Problem in a Model ...   \n",
+       "1752  0704.1999  Dark matter caustics and the enhancement of se...   \n",
+       "1753  0704.2000  Search for a Higgs boson produced in associati...   \n",
+       "\n",
+       "                                              full_text  \\\n",
+       "0     Sparsity-certifying Graph Decompositions\\nIlea...   \n",
+       "1     The evolution of the Earth-Moon system based o...   \n",
+       "2     A Determinant of Stirling Cycle Numbers Counts...   \n",
+       "3     FROM DYADIC Λα TO Λα\\nWAEL ABU-SHAMMALA AND AL...   \n",
+       "4     Polymer Quantum Mechanics and its Continuum Li...   \n",
+       "...                                                 ...   \n",
+       "1749  LPTENS–07/16\\nApril 2007\\nA Wave-function for ...   \n",
+       "1750  Microsoft Word - negEntr.doc\\nQuery on Negativ...   \n",
+       "1751  Absence of the Fifth Force Problem in a Model ...   \n",
+       "1752  Draft version November 16, 2018\\nPreprint type...   \n",
+       "1753  FERMILAB-PUB-07/076-E\\nSearch for a Higgs boso...   \n",
+       "\n",
+       "                                               abstract  \\\n",
+       "0       We describe a new algorithm, the $(k,\\ell)$-...   \n",
+       "1       The evolution of Earth-Moon system is descri...   \n",
+       "2       We show that a determinant of Stirling cycle...   \n",
+       "3       In this paper we show how to compute the $\\L...   \n",
+       "4       A rather non-standard quantum representation...   \n",
+       "...                                                 ...   \n",
+       "1749    We define a wave-function for string theory ...   \n",
+       "1750    After negative temperature is restated, we f...   \n",
+       "1751    A scale invariant model containing dilaton $...   \n",
+       "1752    Cold dark matter haloes are populated by cau...   \n",
+       "1753    We describe a search for the standard model ...   \n",
+       "\n",
+       "                                       text_no_abstract  \n",
+       "0     Introduction and preliminaries\\nThe focus of t...  \n",
+       "1     Introduction \\nThe popularly accepted theory f...  \n",
+       "2     Introduction The chief purpose of this paper i...  \n",
+       "3     FROM DYADIC Λα TO Λα\\nWAEL ABU-SHAMMALA AND AL...  \n",
+       "4     Polymer Quantum Mechanics and its Continuum Li...  \n",
+       "...                                                 ...  \n",
+       "1749  Introduction\\nOur goal in this paper is to emb...  \n",
+       "1750  Microsoft Word - negEntr.doc\\nQuery on Negativ...  \n",
+       "1751  Introduction\\n\\tBasis of Two Measures Field Th...  \n",
+       "1752  Draft version November 16, 2018\\nPreprint type...  \n",
+       "1753  FERMILAB-PUB-07/076-E\\nSearch for a Higgs boso...  \n",
+       "\n",
+       "[1754 rows x 5 columns]"
+      ]
+     },
+     "execution_count": 79,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "en = df[['id','title','full_text','abstract','text_no_abstract']]\n",
+    "en.columns = [\"id\",\"title\", \"full_text\",\"abstract\",\"text_no_abstract\"]\n",
+    "en.to_csv(CSV_PATH+'/scientific_paper_en.csv',index=False,encoding='utf-8')\n",
+    "en"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "05def4d9d0834781cbeb6b95fd92421f8bd6a45e945308f90d88567f4afc1911"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.12 ('tensorflow')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}