Update

app/language_detection.ipynb

@@ -14,7 +14,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -29,7 +29,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
@@ -38,7 +38,7 @@
        "'1.3.0'"
       ]
      },
-     "execution_count": 1,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -51,124 +51,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 35,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>Text</th>\n",
-       "      <th>Language</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>Nature, in the broadest sense, is the natural...</td>\n",
-       "      <td>English</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>\"Nature\" can refer to the phenomena of the phy...</td>\n",
-       "      <td>English</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>The study of nature is a large, if not the onl...</td>\n",
-       "      <td>English</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>Although humans are part of nature, human acti...</td>\n",
-       "      <td>English</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>[1] The word nature is borrowed from the Old F...</td>\n",
-       "      <td>English</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10332</th>\n",
-       "      <td>ನಿಮ್ಮ ತಪ್ಪು ಏನು ಬಂದಿದೆಯೆಂದರೆ ಆ ದಿನದಿಂದ ನಿಮಗೆ ಒ...</td>\n",
-       "      <td>Kannada</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10333</th>\n",
-       "      <td>ನಾರ್ಸಿಸಾ ತಾನು ಮೊದಲಿಗೆ ಹೆಣಗಾಡುತ್ತಿದ್ದ ಮಾರ್ಗಗಳನ್...</td>\n",
-       "      <td>Kannada</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10334</th>\n",
-       "      <td>ಹೇಗೆ ' ನಾರ್ಸಿಸಿಸಮ್ ಈಗ ಮರಿಯನ್ ಅವರಿಗೆ ಸಂಭವಿಸಿದ ಎ...</td>\n",
-       "      <td>Kannada</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10335</th>\n",
-       "      <td>ಅವಳು ಈಗ ಹೆಚ್ಚು ಚಿನ್ನದ ಬ್ರೆಡ್ ಬಯಸುವುದಿಲ್ಲ ಎಂದು ...</td>\n",
-       "      <td>Kannada</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>10336</th>\n",
-       "      <td>ಟೆರ್ರಿ ನೀವು ನಿಜವಾಗಿಯೂ ಆ ದೇವದೂತನಂತೆ ಸ್ವಲ್ಪ ಕಾಣು...</td>\n",
-       "      <td>Kannada</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>10337 rows × 2 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                                    Text Language\n",
-       "0       Nature, in the broadest sense, is the natural...  English\n",
-       "1      \"Nature\" can refer to the phenomena of the phy...  English\n",
-       "2      The study of nature is a large, if not the onl...  English\n",
-       "3      Although humans are part of nature, human acti...  English\n",
-       "4      [1] The word nature is borrowed from the Old F...  English\n",
-       "...                                                  ...      ...\n",
-       "10332  ನಿಮ್ಮ ತಪ್ಪು ಏನು ಬಂದಿದೆಯೆಂದರೆ ಆ ದಿನದಿಂದ ನಿಮಗೆ ಒ...  Kannada\n",
-       "10333  ನಾರ್ಸಿಸಾ ತಾನು ಮೊದಲಿಗೆ ಹೆಣಗಾಡುತ್ತಿದ್ದ ಮಾರ್ಗಗಳನ್...  Kannada\n",
-       "10334  ಹೇಗೆ ' ನಾರ್ಸಿಸಿಸಮ್ ಈಗ ಮರಿಯನ್ ಅವರಿಗೆ ಸಂಭವಿಸಿದ ಎ...  Kannada\n",
-       "10335  ಅವಳು ಈಗ ಹೆಚ್ಚು ಚಿನ್ನದ ಬ್ರೆಡ್ ಬಯಸ��ವುದಿಲ್ಲ ಎಂದು ...  Kannada\n",
-       "10336  ಟೆರ್ರಿ ನೀವು ನಿಜವಾಗಿಯೂ ಆ ದೇವದೂತನಂತೆ ಸ್ವಲ್ಪ ಕಾಣು...  Kannada\n",
-       "\n",
-       "[10337 rows x 2 columns]"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "data = pd.read_csv('language_detection.csv')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 36,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -178,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -189,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
@@ -200,7 +92,7 @@
        "       'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish'], dtype=object)"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -211,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 39,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -225,7 +117,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 40,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -236,7 +128,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 42,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -248,19 +140,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 43,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div>"
+       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div>"
       ],
       "text/plain": [
        "MultinomialNB()"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 43,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -274,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 44,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -283,7 +175,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 45,
    "metadata": {},
    "outputs": [
     {
@@ -370,20 +262,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 48,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Save the model\n",
-    "import pickle\n",
+    "# # Save the model\n",
+    "# import pickle\n",
     "\n",
-    "with open('trained-01.pkl', 'wb') as file:\n",
-    "    pickle.dump(model, file)"
+    "# with open('trained-01.pkl', 'wb') as file:\n",
+    "#     pickle.dump(model, file)\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
+   "execution_count": 46,
    "metadata": {},
    "outputs": [
     {
@@ -392,7 +284,7 @@
        "array(['Portugeese', 'English'], dtype=object)"
       ]
      },
-     "execution_count": 45,
+     "execution_count": 46,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -409,19 +301,74 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Portugeese'"
+      ]
+     },
+     "execution_count": 50,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classes = ['Arabic', 'Danish', 'Dutch', 'English', 'French', 'German',\n",
+    "       'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',\n",
+    "       'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']\n",
+    "\n",
+    "\n",
+    "\n",
+    "def predict_language(text):\n",
+    "    text = re.sub(r'[!@#$(),\\n\"%^*?\\:;~`0-9]', ' ', text)\n",
+    "    text = re.sub(r'[\\[\\]]', ' ', text)\n",
+    "    text = text.lower()\n",
+    "    text = cv.transform([text]).toarray()\n",
+    "    pred = model.predict(text)\n",
+    "    return classes[pred[0]]\n",
+    "\n",
+    "\n",
+    "predict_language('Oi tudo bem?')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
    "metadata": {},
    "outputs": [],
    "source": [
+    "import requests\n",
+    "#  teste api local \n",
+    "url = 'http://localhost:8080/predict'\n",
+    "\n",
+    "text1 = 'Oi tudo bem, como voce vai?'\n",
+    "\n",
+    "response = requests.post(url, json={'text': text1})\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 31,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [500]>"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response"
+   ]
   },
   {
    "cell_type": "code",

app/main.py

@@ -18,4 +18,5 @@ def predict(payload: TextIn):
         language = predict_language(payload.text)
         return {"language": language}
     except Exception as e:
-        raise HTTPException(status_code=500, detail="Internal Server Error")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+    

app/model/model.py

@@ -1,7 +1,8 @@
 import pickle
 import re
 from pathlib import Path
-
+from sklearn.feature_extraction.text import CountVectorizer
+cv = CountVectorizer(max_features = 1500)
 __version__ = '01'
 
 
@@ -21,5 +22,6 @@ def predict_language(text):
     text = re.sub(r'[!@#$(),\n"%^*?\:;~`0-9]', ' ', text)
     text = re.sub(r'[\[\]]', ' ', text)
     text = text.lower()
+    text = cv.transform([text]).toarray()
     pred = model.predict([text])
     return classes[pred[0]]