Spaces:
Sleeping
Sleeping
HudsonArauj
commited on
Commit
·
679d180
1
Parent(s):
e5aa1e6
Language Portuguese
Browse files- app/language_detection.ipynb +17 -43
- app/model/model.py +5 -2
app/language_detection.ipynb
CHANGED
|
@@ -302,16 +302,16 @@
|
|
| 302 |
},
|
| 303 |
{
|
| 304 |
"cell_type": "code",
|
| 305 |
-
"execution_count":
|
| 306 |
"metadata": {},
|
| 307 |
"outputs": [
|
| 308 |
{
|
| 309 |
"data": {
|
| 310 |
"text/plain": [
|
| 311 |
-
"'
|
| 312 |
]
|
| 313 |
},
|
| 314 |
-
"execution_count":
|
| 315 |
"metadata": {},
|
| 316 |
"output_type": "execute_result"
|
| 317 |
}
|
|
@@ -321,6 +321,9 @@
|
|
| 321 |
" 'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',\n",
|
| 322 |
" 'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']\n",
|
| 323 |
"\n",
|
|
|
|
|
|
|
|
|
|
| 324 |
"\n",
|
| 325 |
"\n",
|
| 326 |
"def predict_language(text):\n",
|
|
@@ -329,7 +332,7 @@
|
|
| 329 |
" text = text.lower()\n",
|
| 330 |
" text = cv.transform([text]).toarray()\n",
|
| 331 |
" pred = model.predict(text)\n",
|
| 332 |
-
" return classes[pred[0]]\n",
|
| 333 |
"\n",
|
| 334 |
"\n",
|
| 335 |
"predict_language('How are?')"
|
|
@@ -337,22 +340,22 @@
|
|
| 337 |
},
|
| 338 |
{
|
| 339 |
"cell_type": "code",
|
| 340 |
-
"execution_count":
|
| 341 |
"metadata": {},
|
| 342 |
"outputs": [
|
| 343 |
{
|
| 344 |
"name": "stdout",
|
| 345 |
"output_type": "stream",
|
| 346 |
"text": [
|
| 347 |
-
"{'language': '
|
| 348 |
]
|
| 349 |
}
|
| 350 |
],
|
| 351 |
"source": [
|
| 352 |
"import requests\n",
|
| 353 |
"\n",
|
| 354 |
-
"url = '
|
| 355 |
-
"text1 = '
|
| 356 |
"\n",
|
| 357 |
"response = requests.post(url, json={'text': text1})\n",
|
| 358 |
"\n",
|
|
@@ -364,23 +367,10 @@
|
|
| 364 |
},
|
| 365 |
{
|
| 366 |
"cell_type": "code",
|
| 367 |
-
"execution_count":
|
| 368 |
"metadata": {},
|
| 369 |
-
"outputs": [
|
| 370 |
-
|
| 371 |
-
"data": {
|
| 372 |
-
"text/plain": [
|
| 373 |
-
"<Response [500]>"
|
| 374 |
-
]
|
| 375 |
-
},
|
| 376 |
-
"execution_count": 30,
|
| 377 |
-
"metadata": {},
|
| 378 |
-
"output_type": "execute_result"
|
| 379 |
-
}
|
| 380 |
-
],
|
| 381 |
-
"source": [
|
| 382 |
-
"response"
|
| 383 |
-
]
|
| 384 |
},
|
| 385 |
{
|
| 386 |
"cell_type": "code",
|
|
@@ -391,26 +381,10 @@
|
|
| 391 |
},
|
| 392 |
{
|
| 393 |
"cell_type": "code",
|
| 394 |
-
"execution_count":
|
| 395 |
"metadata": {},
|
| 396 |
-
"outputs": [
|
| 397 |
-
|
| 398 |
-
"data": {
|
| 399 |
-
"text/html": [
|
| 400 |
-
"<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div>"
|
| 401 |
-
],
|
| 402 |
-
"text/plain": [
|
| 403 |
-
"MultinomialNB()"
|
| 404 |
-
]
|
| 405 |
-
},
|
| 406 |
-
"execution_count": 39,
|
| 407 |
-
"metadata": {},
|
| 408 |
-
"output_type": "execute_result"
|
| 409 |
-
}
|
| 410 |
-
],
|
| 411 |
-
"source": [
|
| 412 |
-
"model"
|
| 413 |
-
]
|
| 414 |
},
|
| 415 |
{
|
| 416 |
"cell_type": "code",
|
|
|
|
| 302 |
},
|
| 303 |
{
|
| 304 |
"cell_type": "code",
|
| 305 |
+
"execution_count": 53,
|
| 306 |
"metadata": {},
|
| 307 |
"outputs": [
|
| 308 |
{
|
| 309 |
"data": {
|
| 310 |
"text/plain": [
|
| 311 |
+
"'Inglês'"
|
| 312 |
]
|
| 313 |
},
|
| 314 |
+
"execution_count": 53,
|
| 315 |
"metadata": {},
|
| 316 |
"output_type": "execute_result"
|
| 317 |
}
|
|
|
|
| 321 |
" 'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',\n",
|
| 322 |
" 'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']\n",
|
| 323 |
"\n",
|
| 324 |
+
"dict_por = {'Arabic': 'Árabe', 'Danish': 'Dinamarquês', 'Dutch': 'Holandês', 'English': 'Inglês', 'French': 'Francês', 'German': 'Alemão',\n",
|
| 325 |
+
" 'Greek': 'Grego', 'Hindi': 'Hindi', 'Italian': 'Italiano', 'Kannada': 'Kannada', 'Malayalam': 'Malaiala', 'Portugeese': 'Português',\n",
|
| 326 |
+
" 'Russian': 'Russo', 'Spanish': 'Espanhol', 'Sweedish': 'Sueco', 'Tamil': 'Tâmil', 'Turkish': 'Turco'}\n",
|
| 327 |
"\n",
|
| 328 |
"\n",
|
| 329 |
"def predict_language(text):\n",
|
|
|
|
| 332 |
" text = text.lower()\n",
|
| 333 |
" text = cv.transform([text]).toarray()\n",
|
| 334 |
" pred = model.predict(text)\n",
|
| 335 |
+
" return dict_por[classes[pred[0]]]\n",
|
| 336 |
"\n",
|
| 337 |
"\n",
|
| 338 |
"predict_language('How are?')"
|
|
|
|
| 340 |
},
|
| 341 |
{
|
| 342 |
"cell_type": "code",
|
| 343 |
+
"execution_count": 52,
|
| 344 |
"metadata": {},
|
| 345 |
"outputs": [
|
| 346 |
{
|
| 347 |
"name": "stdout",
|
| 348 |
"output_type": "stream",
|
| 349 |
"text": [
|
| 350 |
+
"{'language': 'English'}\n"
|
| 351 |
]
|
| 352 |
}
|
| 353 |
],
|
| 354 |
"source": [
|
| 355 |
"import requests\n",
|
| 356 |
"\n",
|
| 357 |
+
"url = 'https://hudsonma-simpleapplicationdocker.hf.space/predict'\n",
|
| 358 |
+
"text1 = ' How are u bro?'\n",
|
| 359 |
"\n",
|
| 360 |
"response = requests.post(url, json={'text': text1})\n",
|
| 361 |
"\n",
|
|
|
|
| 367 |
},
|
| 368 |
{
|
| 369 |
"cell_type": "code",
|
| 370 |
+
"execution_count": null,
|
| 371 |
"metadata": {},
|
| 372 |
+
"outputs": [],
|
| 373 |
+
"source": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
},
|
| 375 |
{
|
| 376 |
"cell_type": "code",
|
|
|
|
| 381 |
},
|
| 382 |
{
|
| 383 |
"cell_type": "code",
|
| 384 |
+
"execution_count": null,
|
| 385 |
"metadata": {},
|
| 386 |
+
"outputs": [],
|
| 387 |
+
"source": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
},
|
| 389 |
{
|
| 390 |
"cell_type": "code",
|
app/model/model.py
CHANGED
|
@@ -5,7 +5,6 @@ from pathlib import Path
|
|
| 5 |
from sklearn.feature_extraction.text import CountVectorizer
|
| 6 |
|
| 7 |
cv = CountVectorizer(max_features=1500)
|
| 8 |
-
__version__ = '01'
|
| 9 |
|
| 10 |
BASE_DIR = Path(__file__).resolve(strict=True).parent
|
| 11 |
|
|
@@ -18,10 +17,14 @@ classes = ['Arabic', 'Danish', 'Dutch', 'English', 'French', 'German',
|
|
| 18 |
'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',
|
| 19 |
'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
def predict_language(text):
|
| 22 |
text = re.sub(r'[!@#$(),\n"%^*?\:;~`0-9]', ' ', text)
|
| 23 |
text = re.sub(r'[\[\]]', ' ', text)
|
| 24 |
text = text.lower()
|
| 25 |
text = vocab.transform([text]).toarray()
|
| 26 |
pred = model.predict(text)
|
| 27 |
-
return classes[pred[0]]
|
|
|
|
| 5 |
from sklearn.feature_extraction.text import CountVectorizer
|
| 6 |
|
| 7 |
cv = CountVectorizer(max_features=1500)
|
|
|
|
| 8 |
|
| 9 |
BASE_DIR = Path(__file__).resolve(strict=True).parent
|
| 10 |
|
|
|
|
| 17 |
'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',
|
| 18 |
'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']
|
| 19 |
|
| 20 |
+
dict_por = {'Arabic': 'Árabe', 'Danish': 'Dinamarquês', 'Dutch': 'Holandês', 'English': 'Inglês', 'French': 'Francês', 'German': 'Alemão',
|
| 21 |
+
'Greek': 'Grego', 'Hindi': 'Hindi', 'Italian': 'Italiano', 'Kannada': 'Kannada', 'Malayalam': 'Malaiala', 'Portugeese': 'Português',
|
| 22 |
+
'Russian': 'Russo', 'Spanish': 'Espanhol', 'Sweedish': 'Sueco', 'Tamil': 'Tâmil', 'Turkish': 'Turco'}
|
| 23 |
+
|
| 24 |
def predict_language(text):
|
| 25 |
text = re.sub(r'[!@#$(),\n"%^*?\:;~`0-9]', ' ', text)
|
| 26 |
text = re.sub(r'[\[\]]', ' ', text)
|
| 27 |
text = text.lower()
|
| 28 |
text = vocab.transform([text]).toarray()
|
| 29 |
pred = model.predict(text)
|
| 30 |
+
return dict_por[classes[pred[0]]]
|