Data Cleaning & Preparation

CARSE_00_Cleaning.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -64,7 +64,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -115,7 +115,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -152,7 +152,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -209,7 +209,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -230,7 +230,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -288,7 +288,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -345,7 +345,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -461,7 +461,7 @@
        "10  Vale mi amor, disfruta tu baño\\nSabes que me e...  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 20,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -505,7 +505,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -529,7 +529,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [
     {
@@ -589,33 +589,33 @@
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>665</th>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...</td>\n",
+       "      <th>745</th>\n",
+       "      <td>Descansaste?\\nQue soñaste?</td>\n",
+       "      <td>No muy bien amor, la verdad es que me desperté...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>666</th>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Yo digo que tal vez el universo ha dicho que e...</td>\n",
+       "      <th>746</th>\n",
+       "      <td>Hay que ver una peli de miedoooo</td>\n",
+       "      <td>Sí amooor, vamos al cine a verla</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>667</th>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Deja veo si ya me depositaste en mi cuenta y v...</td>\n",
+       "      <th>747</th>\n",
+       "      <td>Hay que venir a comer aquí</td>\n",
+       "      <td>Cuando quieras bb, ya sabes que yo estoy super...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>668</th>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...</td>\n",
+       "      <th>748</th>\n",
+       "      <td>Estoy muy triste amor</td>\n",
+       "      <td>Por qué mi amor? Cuéntame si quieres</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>669</th>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Solo si veo un depósito en mi cuenta en menos ...</td>\n",
+       "      <th>749</th>\n",
+       "      <td>Te extraño aquí en el gym :(</td>\n",
+       "      <td>Yo también te extraño bb, pero por ahora disfr...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>670 rows × 2 columns</p>\n",
+       "<p>750 rows × 2 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -626,11 +626,11 @@
        "3    Buenos días mi amor, espero que hayas podido d...   \n",
        "4    Hellouuuuu, te amo mucho guapiiii, ten lindo dia    \n",
        "..                                                 ...   \n",
-       "665                                   Dime algo bonito   \n",
-       "666                                   Dime algo bonito   \n",
-       "667                               Oye dime algo bonito   \n",
-       "668                               Oye dime algo bonito   \n",
-       "669                               Oye dime algo bonito   \n",
+       "745                         Descansaste?\\nQue soñaste?   \n",
+       "746                   Hay que ver una peli de miedoooo   \n",
+       "747                         Hay que venir a comer aquí   \n",
+       "748                              Estoy muy triste amor   \n",
+       "749                       Te extraño aquí en el gym :(   \n",
        "\n",
        "                                                 Sebas  \n",
        "0    Buenos d��as mi amorrrr\\nBien bien, pero hacía ...  \n",
@@ -639,22 +639,26 @@
        "3    Hola mi amorcito, cómo amaneciste hoyyy???\\nTa...  \n",
        "4    Holi mi vida, cómo estás hoy??\\nTe amo mucho m...  \n",
        "..                                                 ...  \n",
-       "665  Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...  \n",
-       "666  Yo digo que tal vez el universo ha dicho que e...  \n",
-       "667  Deja veo si ya me depositaste en mi cuenta y v...  \n",
-       "668  Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...  \n",
-       "669  Solo si veo un depósito en mi cuenta en menos ...  \n",
+       "745  No muy bien amor, la verdad es que me desperté...  \n",
+       "746                   Sí amooor, vamos al cine a verla  \n",
+       "747  Cuando quieras bb, ya sabes que yo estoy super...  \n",
+       "748               Por qué mi amor? Cuéntame si quieres  \n",
+       "749  Yo también te extraño bb, pero por ahora disfr...  \n",
        "\n",
-       "[670 rows x 2 columns]"
+       "[750 rows x 2 columns]"
       ]
      },
-     "execution_count": 11,
+     "execution_count": 23,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "chat_df = crear_dataset(texto_sin_timestamp)\n",
+    "# Leer el contenido del archivo\n",
+    "with open(\"Raw_Data/Transformed_Prompts.txt\", 'r', encoding='utf-8') as archivo:\n",
+    "    texto = archivo.read()\n",
+    "\n",
+    "chat_df = crear_dataset(texto)\n",
     "chat_df"
    ]
   },
@@ -667,7 +671,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -683,7 +687,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [
     {
@@ -750,38 +754,38 @@
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>665</th>\n",
+       "      <th>745</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...</td>\n",
+       "      <td>Descansaste?\\nQue soñaste?</td>\n",
+       "      <td>No muy bien amor, la verdad es que me desperté...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>666</th>\n",
+       "      <th>746</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Yo digo que tal vez el universo ha dicho que e...</td>\n",
+       "      <td>Hay que ver una peli de miedoooo</td>\n",
+       "      <td>Sí amooor, vamos al cine a verla</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>667</th>\n",
+       "      <th>747</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Deja veo si ya me depositaste en mi cuenta y v...</td>\n",
+       "      <td>Hay que venir a comer aquí</td>\n",
+       "      <td>Cuando quieras bb, ya sabes que yo estoy super...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>668</th>\n",
+       "      <th>748</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...</td>\n",
+       "      <td>Estoy muy triste amor</td>\n",
+       "      <td>Por qué mi amor? Cuéntame si quieres</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>669</th>\n",
+       "      <th>749</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Solo si veo un depósito en mi cuenta en menos ...</td>\n",
+       "      <td>Te extraño aquí en el gym :(</td>\n",
+       "      <td>Yo también te extraño bb, pero por ahora disfr...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>670 rows × 3 columns</p>\n",
+       "<p>750 rows × 3 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -792,11 +796,11 @@
        "3    CARSE es un chatbot que imita el estilo en que...   \n",
        "4    CARSE es un chatbot que imita el estilo en que...   \n",
        "..                                                 ...   \n",
-       "665  CARSE es un chatbot que imita el estilo en que...   \n",
-       "666  CARSE es un chatbot que imita el estilo en que...   \n",
-       "667  CARSE es un chatbot que imita el estilo en que...   \n",
-       "668  CARSE es un chatbot que imita el estilo en que...   \n",
-       "669  CARSE es un chatbot que imita el estilo en que...   \n",
+       "745  CARSE es un chatbot que imita el estilo en que...   \n",
+       "746  CARSE es un chatbot que imita el estilo en que...   \n",
+       "747  CARSE es un chatbot que imita el estilo en que...   \n",
+       "748  CARSE es un chatbot que imita el estilo en que...   \n",
+       "749  CARSE es un chatbot que imita el estilo en que...   \n",
        "\n",
        "                                               CarmenQ  \\\n",
        "0          Buenos días mi amorchis, cómo dormiste hoy?   \n",
@@ -805,11 +809,11 @@
        "3    Buenos días mi amor, espero que hayas podido d...   \n",
        "4    Hellouuuuu, te amo mucho guapiiii, ten lindo dia    \n",
        "..                                                 ...   \n",
-       "665                                   Dime algo bonito   \n",
-       "666                                   Dime algo bonito   \n",
-       "667                               Oye dime algo bonito   \n",
-       "668                               Oye dime algo bonito   \n",
-       "669                               Oye dime algo bonito   \n",
+       "745                         Descansaste?\\nQue soñaste?   \n",
+       "746                   Hay que ver una peli de miedoooo   \n",
+       "747                         Hay que venir a comer aquí   \n",
+       "748                              Estoy muy triste amor   \n",
+       "749                       Te extraño aquí en el gym :(   \n",
        "\n",
        "                                                 Sebas  \n",
        "0    Buenos días mi amorrrr\\nBien bien, pero hacía ...  \n",
@@ -818,16 +822,16 @@
        "3    Hola mi amorcito, cómo amaneciste hoyyy???\\nTa...  \n",
        "4    Holi mi vida, cómo estás hoy??\\nTe amo mucho m...  \n",
        "..                                                 ...  \n",
-       "665  Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...  \n",
-       "666  Yo digo que tal vez el universo ha dicho que e...  \n",
-       "667  Deja veo si ya me depositaste en mi cuenta y v...  \n",
-       "668  Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...  \n",
-       "669  Solo si veo un depósito en mi cuenta en menos ...  \n",
+       "745  No muy bien amor, la verdad es que me desperté...  \n",
+       "746                   Sí amooor, vamos al cine a verla  \n",
+       "747  Cuando quieras bb, ya sabes que yo estoy super...  \n",
+       "748               Por qué mi amor? Cuéntame si quieres  \n",
+       "749  Yo también te extraño bb, pero por ahora disfr...  \n",
        "\n",
-       "[670 rows x 3 columns]"
+       "[750 rows x 3 columns]"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 25,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -840,7 +844,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -853,7 +857,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
@@ -920,38 +924,38 @@
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>665</th>\n",
+       "      <th>745</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...</td>\n",
+       "      <td>Descansaste?\\nQue soñaste?</td>\n",
+       "      <td>No muy bien amor, la verdad es que me desperté...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>666</th>\n",
+       "      <th>746</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Dime algo bonito</td>\n",
-       "      <td>Yo digo que tal vez el universo ha dicho que e...</td>\n",
+       "      <td>Hay que ver una peli de miedoooo</td>\n",
+       "      <td>Sí amooor, vamos al cine a verla</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>667</th>\n",
+       "      <th>747</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Deja veo si ya me depositaste en mi cuenta y v...</td>\n",
+       "      <td>Hay que venir a comer aquí</td>\n",
+       "      <td>Cuando quieras bb, ya sabes que yo estoy super...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>668</th>\n",
+       "      <th>748</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...</td>\n",
+       "      <td>Estoy muy triste amor</td>\n",
+       "      <td>Por qué mi amor? Cuéntame si quieres</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>669</th>\n",
+       "      <th>749</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Oye dime algo bonito</td>\n",
-       "      <td>Solo si veo un depósito en mi cuenta en menos ...</td>\n",
+       "      <td>Te extraño aquí en el gym :(</td>\n",
+       "      <td>Yo también te extraño bb, pero por ahora disfr...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>670 rows × 3 columns</p>\n",
+       "<p>750 rows × 3 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -962,11 +966,11 @@
        "3    CARSE es un chatbot que imita el estilo en que...   \n",
        "4    CARSE es un chatbot que imita el estilo en que...   \n",
        "..                                                 ...   \n",
-       "665  CARSE es un chatbot que imita el estilo en que...   \n",
-       "666  CARSE es un chatbot que imita el estilo en que...   \n",
-       "667  CARSE es un chatbot que imita el estilo en que...   \n",
-       "668  CARSE es un chatbot que imita el estilo en que...   \n",
-       "669  CARSE es un chatbot que imita el estilo en que...   \n",
+       "745  CARSE es un chatbot que imita el estilo en que...   \n",
+       "746  CARSE es un chatbot que imita el estilo en que...   \n",
+       "747  CARSE es un chatbot que imita el estilo en que...   \n",
+       "748  CARSE es un chatbot que imita el estilo en que...   \n",
+       "749  CARSE es un chatbot que imita el estilo en que...   \n",
        "\n",
        "                                                  user  \\\n",
        "0          Buenos días mi amorchis, cómo dormiste hoy?   \n",
@@ -975,11 +979,11 @@
        "3    Buenos días mi amor, espero que hayas podido d...   \n",
        "4    Hellouuuuu, te amo mucho guapiiii, ten lindo dia    \n",
        "..                                                 ...   \n",
-       "665                                   Dime algo bonito   \n",
-       "666                                   Dime algo bonito   \n",
-       "667                               Oye dime algo bonito   \n",
-       "668                               Oye dime algo bonito   \n",
-       "669                               Oye dime algo bonito   \n",
+       "745                         Descansaste?\\nQue soñaste?   \n",
+       "746                   Hay que ver una peli de miedoooo   \n",
+       "747                         Hay que venir a comer aquí   \n",
+       "748                              Estoy muy triste amor   \n",
+       "749                       Te extraño aquí en el gym :(   \n",
        "\n",
        "                                             assistant  \n",
        "0    Buenos días mi amorrrr\\nBien bien, pero hacía ...  \n",
@@ -988,16 +992,16 @@
        "3    Hola mi amorcito, cómo amaneciste hoyyy???\\nTa...  \n",
        "4    Holi mi vida, cómo estás hoy??\\nTe amo mucho m...  \n",
        "..                                                 ...  \n",
-       "665  Hmmmm vemos. Deposítame 5 mil pesotes y va JAJ...  \n",
-       "666  Yo digo que tal vez el universo ha dicho que e...  \n",
-       "667  Deja veo si ya me depositaste en mi cuenta y v...  \n",
-       "668  Algo bonito? Tuuuuu, mi C, estás hermosa\\nTeng...  \n",
-       "669  Solo si veo un depósito en mi cuenta en menos ...  \n",
+       "745  No muy bien amor, la verdad es que me desperté...  \n",
+       "746                   Sí amooor, vamos al cine a verla  \n",
+       "747  Cuando quieras bb, ya sabes que yo estoy super...  \n",
+       "748               Por qué mi amor? Cuéntame si quieres  \n",
+       "749  Yo también te extraño bb, pero por ahora disfr...  \n",
        "\n",
-       "[670 rows x 3 columns]"
+       "[750 rows x 3 columns]"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 27,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1010,7 +1014,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [
     {
@@ -1043,32 +1047,32 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Tengo mucho sueño aún</td>\n",
-       "      <td>Ay amorcito, a ver si te echas a dormir en el ...</td>\n",
+       "      <td>Yo me iré al gym en unos minutos</td>\n",
+       "      <td>Super mi amorrr, disfruta mucho tu ejercicio, ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Ay amor que frío hace</td>\n",
-       "      <td>Pues que bueno, mínimo no estamos sudando</td>\n",
+       "      <td>Ya ando en el gym amor</td>\n",
+       "      <td>Super mi amorrr, disfruta mucho tu ejercicio\\n...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Ya adivina\\nLo que se te ocurra</td>\n",
-       "      <td>Esque no se me ocurrió nada mas</td>\n",
+       "      <td>Te mandaré nota</td>\n",
+       "      <td>Vale amor, mándame lo que quieras, que sí me g...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Biennnn, pero tengo sueño aún</td>\n",
-       "      <td>Si tienes descanso, duerme</td>\n",
+       "      <td>Te extraño poquis</td>\n",
+       "      <td>Igual te extraño poquis\\nIntentaré organizarme...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
-       "      <td>Yo te amo massssss</td>\n",
-       "      <td>Eso es mega feik eh</td>\n",
+       "      <td>He dormido mejor\\nSiento que no descansé nada</td>\n",
+       "      <td>Y por qué no duermes un rato más?</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -1077,38 +1081,38 @@
        "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>665</th>\n",
+       "      <th>745</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
        "      <td>Pues ya toca casarnos</td>\n",
        "      <td>Por fiiiin, ya te habías tardado en decir eso</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>666</th>\n",
+       "      <th>746</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
        "      <td>Biennnnn\\n¿Y tu?</td>\n",
        "      <td>Bien igual</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>667</th>\n",
+       "      <th>747</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
        "      <td>JAJAJJAJAJA no</td>\n",
        "      <td>Ya dimeeeee</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>668</th>\n",
+       "      <th>748</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
        "      <td>Casarme contigo algún día</td>\n",
        "      <td>Awwww amoorr, sí quiero casarme contigo algún ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>669</th>\n",
+       "      <th>749</th>\n",
        "      <td>CARSE es un chatbot que imita el estilo en que...</td>\n",
        "      <td>Me estaba acabando el café</td>\n",
        "      <td>Disfruta tu cafecito</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>670 rows × 3 columns</p>\n",
+       "<p>750 rows × 3 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -1119,42 +1123,42 @@
        "3    CARSE es un chatbot que imita el estilo en que...   \n",
        "4    CARSE es un chatbot que imita el estilo en que...   \n",
        "..                                                 ...   \n",
-       "665  CARSE es un chatbot que imita el estilo en que...   \n",
-       "666  CARSE es un chatbot que imita el estilo en que...   \n",
-       "667  CARSE es un chatbot que imita el estilo en que...   \n",
-       "668  CARSE es un chatbot que imita el estilo en que...   \n",
-       "669  CARSE es un chatbot que imita el estilo en que...   \n",
+       "745  CARSE es un chatbot que imita el estilo en que...   \n",
+       "746  CARSE es un chatbot que imita el estilo en que...   \n",
+       "747  CARSE es un chatbot que imita el estilo en que...   \n",
+       "748  CARSE es un chatbot que imita el estilo en que...   \n",
+       "749  CARSE es un chatbot que imita el estilo en que...   \n",
        "\n",
-       "                                user  \\\n",
-       "0              Tengo mucho sueño aún   \n",
-       "1              Ay amor que frío hace   \n",
-       "2    Ya adivina\\nLo que se te ocurra   \n",
-       "3      Biennnn, pero tengo sueño aún   \n",
-       "4                 Yo te amo massssss   \n",
-       "..                               ...   \n",
-       "665            Pues ya toca casarnos   \n",
-       "666                 Biennnnn\\n¿Y tu?   \n",
-       "667                   JAJAJJAJAJA no   \n",
-       "668        Casarme contigo algún día   \n",
-       "669       Me estaba acabando el café   \n",
+       "                                              user  \\\n",
+       "0                 Yo me iré al gym en unos minutos   \n",
+       "1                           Ya ando en el gym amor   \n",
+       "2                                  Te mandaré nota   \n",
+       "3                                Te extraño poquis   \n",
+       "4    He dormido mejor\\nSiento que no descansé nada   \n",
+       "..                                             ...   \n",
+       "745                          Pues ya toca casarnos   \n",
+       "746                               Biennnnn\\n¿Y tu?   \n",
+       "747                                 JAJAJJAJAJA no   \n",
+       "748                      Casarme contigo algún día   \n",
+       "749                     Me estaba acabando el café   \n",
        "\n",
        "                                             assistant  \n",
-       "0    Ay amorcito, a ver si te echas a dormir en el ...  \n",
-       "1            Pues que bueno, mínimo no estamos sudando  \n",
-       "2                      Esque no se me ocurrió nada mas  \n",
-       "3                           Si tienes descanso, duerme  \n",
-       "4                                  Eso es mega feik eh  \n",
+       "0    Super mi amorrr, disfruta mucho tu ejercicio, ...  \n",
+       "1    Super mi amorrr, disfruta mucho tu ejercicio\\n...  \n",
+       "2    Vale amor, mándame lo que quieras, que sí me g...  \n",
+       "3    Igual te extraño poquis\\nIntentaré organizarme...  \n",
+       "4                    Y por qué no duermes un rato más?  \n",
        "..                                                 ...  \n",
-       "665      Por fiiiin, ya te habías tardado en decir eso  \n",
-       "666                                         Bien igual  \n",
-       "667                                        Ya dimeeeee  \n",
-       "668  Awwww amoorr, sí quiero casarme contigo algún ...  \n",
-       "669                               Disfruta tu cafecito  \n",
+       "745      Por fiiiin, ya te habías tardado en decir eso  \n",
+       "746                                         Bien igual  \n",
+       "747                                        Ya dimeeeee  \n",
+       "748  Awwww amoorr, sí quiero casarme contigo algún ...  \n",
+       "749                               Disfruta tu cafecito  \n",
        "\n",
-       "[670 rows x 3 columns]"
+       "[750 rows x 3 columns]"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 28,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1174,7 +1178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1198,7 +1202,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1209,14 +1213,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "El archivo tiene 670 líneas.\n"
+      "El archivo tiene 750 líneas.\n"
      ]
     }
    ],

CARSE_01_Prep.ipynb

@@ -1 +1 @@
-{"cells":[{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":173,"status":"ok","timestamp":1702156660830,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"pEb6CJl3ZFKP"},"outputs":[],"source":["#! pip install tiktoken"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156660831,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"zj6dSJC3Y01I"},"outputs":[],"source":["import json\n","import tiktoken # for token counting\n","import numpy as np\n","from collections import defaultdict"]},{"cell_type":"markdown","metadata":{"id":"SmN-A6KJZQPo"},"source":["<br>\n","<br>\n","\n","## Data Loading"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":6,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-geBXCMTZObN","outputId":"924e87bd-4c01-4eff-ac93-10f07e27fe41"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples: 670\n","First example:\n","{'role': 'system', 'content': 'CARSE es un chatbot que imita el estilo en que Sebas conversa. Fue creado con amor solo para Carmen. Su objetivo es ofrecer una experiencia de chat divertida y familiar que exprese la personalidad, el humor y el amor de Sebas hacia Carmen.'}\n","{'role': 'user', 'content': 'Tengo mucho sueño aún'}\n","{'role': 'assistant', 'content': 'Ay amorcito, a ver si te echas a dormir en el coche un rato'}\n"]}],"source":["data_path = \"Training_data/Training_Prompts.jsonl\"\n","\n","# Load the dataset\n","with open(data_path, 'r', encoding='utf-8') as f:\n","    dataset = [json.loads(line) for line in f]\n","\n","# Initial dataset stats\n","print(\"\\nNum examples:\", len(dataset))\n","print(\"First example:\")\n","for message in dataset[0][\"messages\"]:\n","    print(message)"]},{"cell_type":"markdown","metadata":{"id":"UfxL9TQMaCCG"},"source":["<br>\n","<br>\n","\n","## Format Validation"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"bOXkqnLgZakM","outputId":"8c716779-0f83-4fbf-c302-56d834735327"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","No errors found\n"]}],"source":["# Format error checks\n","format_errors = defaultdict(int)\n","\n","for ex in dataset:\n","    if not isinstance(ex, dict):\n","        format_errors[\"data_type\"] += 1\n","        continue\n","\n","    messages = ex.get(\"messages\", None)\n","    if not messages:\n","        format_errors[\"missing_messages_list\"] += 1\n","        continue\n","\n","    for message in messages:\n","        if \"role\" not in message or \"content\" not in message:\n","            format_errors[\"message_missing_key\"] += 1\n","\n","        if any(k not in (\"role\", \"content\", \"name\", \"function_call\") for k in message):\n","            format_errors[\"message_unrecognized_key\"] += 1\n","\n","        if message.get(\"role\", None) not in (\"system\", \"user\", \"assistant\", \"function\"):\n","            format_errors[\"unrecognized_role\"] += 1\n","\n","        content = message.get(\"content\", None)\n","        function_call = message.get(\"function_call\", None)\n","\n","        if (not content and not function_call) or not isinstance(content, str):\n","            format_errors[\"missing_content\"] += 1\n","\n","    if not any(message.get(\"role\", None) == \"assistant\" for message in messages):\n","        format_errors[\"example_missing_assistant_message\"] += 1\n","\n","if format_errors:\n","    print(\"\\nFound errors:\")\n","    for k, v in format_errors.items():\n","        print(f\"{k}: {v}\")\n","else:\n","    print(\"\\nNo errors found\")"]},{"cell_type":"markdown","metadata":{"id":"UXrIdBKtaHkx"},"source":["<br>\n","<br>\n","\n","## Token Counting Utilities"]},{"cell_type":"code","execution_count":8,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"A6ev3s_TaGeJ"},"outputs":[],"source":["encoding = tiktoken.get_encoding(\"cl100k_base\")\n","\n","# not exact!\n","# simplified from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb\n","def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):\n","    num_tokens = 0\n","    for message in messages:\n","        num_tokens += tokens_per_message\n","        for key, value in message.items():\n","            num_tokens += len(encoding.encode(value))\n","            if key == \"name\":\n","                num_tokens += tokens_per_name\n","    num_tokens += 3\n","    return num_tokens\n","\n","def num_assistant_tokens_from_messages(messages):\n","    num_tokens = 0\n","    for message in messages:\n","        if message[\"role\"] == \"assistant\":\n","            num_tokens += len(encoding.encode(message[\"content\"]))\n","    return num_tokens\n","\n","def print_distribution(values, name):\n","    print(f\"\\n#### Distribution of {name}:\")\n","    print(f\"min / max: {min(values)}, {max(values)}\")\n","    print(f\"mean / median: {np.mean(values)}, {np.median(values)}\")\n","    print(f\"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}\")"]},{"cell_type":"markdown","metadata":{"id":"2duSc8L7aYki"},"source":["<br>\n","<br>\n","\n","## Data Warnings and Token Counts"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"CfDEmpgiaLuS","outputId":"f1e8ed95-f77b-4340-80ce-b08ee6025a2e"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples missing system message: 0\n","Num examples missing user message: 0\n","\n","#### Distribution of num_messages_per_example:\n","min / max: 3, 3\n","mean / median: 3.0, 3.0\n","p5 / p95: 3.0, 3.0\n","\n","#### Distribution of num_total_tokens_per_example:\n","min / max: 77, 263\n","mean / median: 102.54776119402985, 97.0\n","p5 / p95: 84.0, 120.10000000000002\n","\n","#### Distribution of num_assistant_tokens_per_example:\n","min / max: 2, 184\n","mean / median: 19.9955223880597, 15.0\n","p5 / p95: 6.0, 34.0\n","\n","0 examples may be over the 4096 token limit, they will be truncated during fine-tuning\n"]}],"source":["# Warnings and tokens counts\n","n_missing_system = 0\n","n_missing_user = 0\n","n_messages = []\n","convo_lens = []\n","assistant_message_lens = []\n","\n","for ex in dataset:\n","    messages = ex[\"messages\"]\n","    if not any(message[\"role\"] == \"system\" for message in messages):\n","        n_missing_system += 1\n","    if not any(message[\"role\"] == \"user\" for message in messages):\n","        n_missing_user += 1\n","    n_messages.append(len(messages))\n","    convo_lens.append(num_tokens_from_messages(messages))\n","    assistant_message_lens.append(num_assistant_tokens_from_messages(messages))\n","\n","print(\"\\nNum examples missing system message:\", n_missing_system)\n","print(\"Num examples missing user message:\", n_missing_user)\n","print_distribution(n_messages, \"num_messages_per_example\")\n","print_distribution(convo_lens, \"num_total_tokens_per_example\")\n","print_distribution(assistant_message_lens, \"num_assistant_tokens_per_example\")\n","n_too_long = sum(l > 4096 for l in convo_lens)\n","print(f\"\\n{n_too_long} examples may be over the 4096 token limit, they will be truncated during fine-tuning\")"]},{"cell_type":"markdown","metadata":{"id":"x8GusuUOapUB"},"source":["<br>\n","<br>\n","\n","## Cost Estimation"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":194,"status":"ok","timestamp":1702156661200,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"EnAWqf3SadOj","outputId":"3d0527e9-a3cf-4b6e-9ce4-89a99611d219"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Dataset has ~68707 tokens that will be charged for during training\n","By default, you'll train for 3 epochs on this dataset\n","By default, you'll be charged for ~206121 tokens\n"]}],"source":["# Pricing and default n_epochs estimate\n","MAX_TOKENS_PER_EXAMPLE = 4096\n","\n","TARGET_EPOCHS = 3\n","MIN_TARGET_EXAMPLES = 100\n","MAX_TARGET_EXAMPLES = 25000\n","MIN_DEFAULT_EPOCHS = 1\n","MAX_DEFAULT_EPOCHS = 25\n","\n","n_epochs = TARGET_EPOCHS\n","n_train_examples = len(dataset)\n","if n_train_examples * TARGET_EPOCHS < MIN_TARGET_EXAMPLES:\n","    n_epochs = min(MAX_DEFAULT_EPOCHS, MIN_TARGET_EXAMPLES // n_train_examples)\n","elif n_train_examples * TARGET_EPOCHS > MAX_TARGET_EXAMPLES:\n","    n_epochs = max(MIN_DEFAULT_EPOCHS, MAX_TARGET_EXAMPLES // n_train_examples)\n","\n","n_billing_tokens_in_dataset = sum(min(MAX_TOKENS_PER_EXAMPLE, length) for length in convo_lens)\n","print(f\"\\nDataset has ~{n_billing_tokens_in_dataset} tokens that will be charged for during training\")\n","print(f\"By default, you'll train for {n_epochs} epochs on this dataset\")\n","print(f\"By default, you'll be charged for ~{n_epochs * n_billing_tokens_in_dataset} tokens\")"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156777183,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-a820sVicdat","outputId":"c96c5167-c91e-4e1c-e8a4-0bd16a6aeb70"},"outputs":[{"data":{"text/plain":["1.648968"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["# Suponiendo que cargan $0.008/1k tokens\n","value = 0.008\n","\n","final_money = (n_epochs * n_billing_tokens_in_dataset)*value/1000\n","final_money"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0anBNWtKczWR"},"outputs":[],"source":[]}],"metadata":{"colab":{"authorship_tag":"ABX9TyMzN+lsyb4smMC9hf/gYgCs","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.7"}},"nbformat":4,"nbformat_minor":0}
+{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":173,"status":"ok","timestamp":1702156660830,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"pEb6CJl3ZFKP"},"outputs":[],"source":["#! pip install tiktoken"]},{"cell_type":"code","execution_count":2,"metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156660831,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"zj6dSJC3Y01I"},"outputs":[],"source":["import json\n","import tiktoken # for token counting\n","import numpy as np\n","from collections import defaultdict"]},{"cell_type":"markdown","metadata":{"id":"SmN-A6KJZQPo"},"source":["<br>\n","<br>\n","\n","## Data Loading"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":6,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-geBXCMTZObN","outputId":"924e87bd-4c01-4eff-ac93-10f07e27fe41"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples: 750\n","First example:\n","{'role': 'system', 'content': 'CARSE es un chatbot que imita el estilo en que Sebas conversa. Fue creado con amor solo para Carmen. Su objetivo es ofrecer una experiencia de chat divertida y familiar que exprese la personalidad, el humor y el amor de Sebas hacia Carmen.'}\n","{'role': 'user', 'content': 'Yo me iré al gym en unos minutos'}\n","{'role': 'assistant', 'content': 'Super mi amorrr, disfruta mucho tu ejercicio, ponte muy fuerte'}\n"]}],"source":["data_path = \"Training_data/Training_Prompts.jsonl\"\n","\n","# Load the dataset\n","with open(data_path, 'r', encoding='utf-8') as f:\n","    dataset = [json.loads(line) for line in f]\n","\n","# Initial dataset stats\n","print(\"\\nNum examples:\", len(dataset))\n","print(\"First example:\")\n","for message in dataset[0][\"messages\"]:\n","    print(message)"]},{"cell_type":"markdown","metadata":{"id":"UfxL9TQMaCCG"},"source":["<br>\n","<br>\n","\n","## Format Validation"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"bOXkqnLgZakM","outputId":"8c716779-0f83-4fbf-c302-56d834735327"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","No errors found\n"]}],"source":["# Format error checks\n","format_errors = defaultdict(int)\n","\n","for ex in dataset:\n","    if not isinstance(ex, dict):\n","        format_errors[\"data_type\"] += 1\n","        continue\n","\n","    messages = ex.get(\"messages\", None)\n","    if not messages:\n","        format_errors[\"missing_messages_list\"] += 1\n","        continue\n","\n","    for message in messages:\n","        if \"role\" not in message or \"content\" not in message:\n","            format_errors[\"message_missing_key\"] += 1\n","\n","        if any(k not in (\"role\", \"content\", \"name\", \"function_call\") for k in message):\n","            format_errors[\"message_unrecognized_key\"] += 1\n","\n","        if message.get(\"role\", None) not in (\"system\", \"user\", \"assistant\", \"function\"):\n","            format_errors[\"unrecognized_role\"] += 1\n","\n","        content = message.get(\"content\", None)\n","        function_call = message.get(\"function_call\", None)\n","\n","        if (not content and not function_call) or not isinstance(content, str):\n","            format_errors[\"missing_content\"] += 1\n","\n","    if not any(message.get(\"role\", None) == \"assistant\" for message in messages):\n","        format_errors[\"example_missing_assistant_message\"] += 1\n","\n","if format_errors:\n","    print(\"\\nFound errors:\")\n","    for k, v in format_errors.items():\n","        print(f\"{k}: {v}\")\n","else:\n","    print(\"\\nNo errors found\")"]},{"cell_type":"markdown","metadata":{"id":"UXrIdBKtaHkx"},"source":["<br>\n","<br>\n","\n","## Token Counting Utilities"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"A6ev3s_TaGeJ"},"outputs":[],"source":["encoding = tiktoken.get_encoding(\"cl100k_base\")\n","\n","# not exact!\n","# simplified from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb\n","def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):\n","    num_tokens = 0\n","    for message in messages:\n","        num_tokens += tokens_per_message\n","        for key, value in message.items():\n","            num_tokens += len(encoding.encode(value))\n","            if key == \"name\":\n","                num_tokens += tokens_per_name\n","    num_tokens += 3\n","    return num_tokens\n","\n","def num_assistant_tokens_from_messages(messages):\n","    num_tokens = 0\n","    for message in messages:\n","        if message[\"role\"] == \"assistant\":\n","            num_tokens += len(encoding.encode(message[\"content\"]))\n","    return num_tokens\n","\n","def print_distribution(values, name):\n","    print(f\"\\n#### Distribution of {name}:\")\n","    print(f\"min / max: {min(values)}, {max(values)}\")\n","    print(f\"mean / median: {np.mean(values)}, {np.median(values)}\")\n","    print(f\"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}\")"]},{"cell_type":"markdown","metadata":{"id":"2duSc8L7aYki"},"source":["<br>\n","<br>\n","\n","## Data Warnings and Token Counts"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1702156661009,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"CfDEmpgiaLuS","outputId":"f1e8ed95-f77b-4340-80ce-b08ee6025a2e"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Num examples missing system message: 0\n","Num examples missing user message: 0\n","\n","#### Distribution of num_messages_per_example:\n","min / max: 3, 3\n","mean / median: 3.0, 3.0\n","p5 / p95: 3.0, 3.0\n","\n","#### Distribution of num_total_tokens_per_example:\n","min / max: 77, 263\n","mean / median: 102.228, 97.0\n","p5 / p95: 85.0, 119.10000000000002\n","\n","#### Distribution of num_assistant_tokens_per_example:\n","min / max: 2, 184\n","mean / median: 19.618666666666666, 15.0\n","p5 / p95: 6.0, 33.0\n","\n","0 examples may be over the 4096 token limit, they will be truncated during fine-tuning\n"]}],"source":["# Warnings and tokens counts\n","n_missing_system = 0\n","n_missing_user = 0\n","n_messages = []\n","convo_lens = []\n","assistant_message_lens = []\n","\n","for ex in dataset:\n","    messages = ex[\"messages\"]\n","    if not any(message[\"role\"] == \"system\" for message in messages):\n","        n_missing_system += 1\n","    if not any(message[\"role\"] == \"user\" for message in messages):\n","        n_missing_user += 1\n","    n_messages.append(len(messages))\n","    convo_lens.append(num_tokens_from_messages(messages))\n","    assistant_message_lens.append(num_assistant_tokens_from_messages(messages))\n","\n","print(\"\\nNum examples missing system message:\", n_missing_system)\n","print(\"Num examples missing user message:\", n_missing_user)\n","print_distribution(n_messages, \"num_messages_per_example\")\n","print_distribution(convo_lens, \"num_total_tokens_per_example\")\n","print_distribution(assistant_message_lens, \"num_assistant_tokens_per_example\")\n","n_too_long = sum(l > 4096 for l in convo_lens)\n","print(f\"\\n{n_too_long} examples may be over the 4096 token limit, they will be truncated during fine-tuning\")"]},{"cell_type":"markdown","metadata":{"id":"x8GusuUOapUB"},"source":["<br>\n","<br>\n","\n","## Cost Estimation"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":194,"status":"ok","timestamp":1702156661200,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"EnAWqf3SadOj","outputId":"3d0527e9-a3cf-4b6e-9ce4-89a99611d219"},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","Dataset has ~76671 tokens that will be charged for during training\n","By default, you'll train for 3 epochs on this dataset\n","By default, you'll be charged for ~230013 tokens\n"]}],"source":["# Pricing and default n_epochs estimate\n","MAX_TOKENS_PER_EXAMPLE = 4096\n","\n","TARGET_EPOCHS = 3\n","MIN_TARGET_EXAMPLES = 100\n","MAX_TARGET_EXAMPLES = 25000\n","MIN_DEFAULT_EPOCHS = 1\n","MAX_DEFAULT_EPOCHS = 25\n","\n","n_epochs = TARGET_EPOCHS\n","n_train_examples = len(dataset)\n","if n_train_examples * TARGET_EPOCHS < MIN_TARGET_EXAMPLES:\n","    n_epochs = min(MAX_DEFAULT_EPOCHS, MIN_TARGET_EXAMPLES // n_train_examples)\n","elif n_train_examples * TARGET_EPOCHS > MAX_TARGET_EXAMPLES:\n","    n_epochs = max(MIN_DEFAULT_EPOCHS, MAX_TARGET_EXAMPLES // n_train_examples)\n","\n","n_billing_tokens_in_dataset = sum(min(MAX_TOKENS_PER_EXAMPLE, length) for length in convo_lens)\n","print(f\"\\nDataset has ~{n_billing_tokens_in_dataset} tokens that will be charged for during training\")\n","print(f\"By default, you'll train for {n_epochs} epochs on this dataset\")\n","print(f\"By default, you'll be charged for ~{n_epochs * n_billing_tokens_in_dataset} tokens\")"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2,"status":"ok","timestamp":1702156777183,"user":{"displayName":"Sebastián M.","userId":"05974347998580401662"},"user_tz":360},"id":"-a820sVicdat","outputId":"c96c5167-c91e-4e1c-e8a4-0bd16a6aeb70"},"outputs":[{"data":{"text/plain":["1.840104"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["# Suponiendo que cargan $0.008/1k tokens\n","value = 0.008\n","\n","final_money = (n_epochs * n_billing_tokens_in_dataset)*value/1000\n","final_money"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0anBNWtKczWR"},"outputs":[],"source":[]}],"metadata":{"colab":{"authorship_tag":"ABX9TyMzN+lsyb4smMC9hf/gYgCs","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.7"}},"nbformat":4,"nbformat_minor":0}