{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!unzip archive.zip\n",
"\n",
"#https://www.kaggle.com/datasets/basilb2s/language-detection\n",
"#https://github.com/basil-b2s/Language-Detector"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"import warnings\n",
"import re\n",
"warnings.filterwarnings('ignore')"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'1.3.0'"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sklearn\n",
"\n",
"sklearn.__version__"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"data = pd.read_csv('language_detection.csv')"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"X = data['Text']\n",
"y = data['Language']"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.preprocessing import LabelEncoder\n",
"le = LabelEncoder()\n",
"y = le.fit_transform(y)\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array(['Arabic', 'Danish', 'Dutch', 'English', 'French', 'German',\n",
" 'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',\n",
" 'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish'], dtype=object)"
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"le.classes_"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"data_list =[]\n",
"for text in X:\n",
" text = re.sub(r'[!@#$(),n\"%^*?:;~`0-9]', ' ', text)\n",
" text = re.sub(r'[[]]', ' ', text)\n",
" text = text.lower()\n",
" data_list.append(text)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.model_selection import train_test_split\n",
"\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.feature_extraction.text import CountVectorizer\n",
"cv = CountVectorizer()\n",
"X_train = cv.fit_transform(X_train).toarray()\n",
"X_test = cv.transform(X_test).toarray()"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"MultinomialNB() In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"figure = plt.figure(figsize=(10, 10))\n",
"sns.heatmap(cm, annot=True)\n",
"plt.xlabel('Predicted')\n",
"plt.ylabel('Truth')\n",
"\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"# Salvar o modelo e o vocabulário\n",
"import pickle\n",
"\n",
"with open('filename.pkl', 'wb') as f:\n",
" pickle.dump({'model': model, 'vocab': cv}, f)\n"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array(['Portugeese', 'English'], dtype=object)"
]
},
"execution_count": 48,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"text1 = 'Oi tudo bem?'\n",
"text2 = 'Hello how are you?'\n",
"text = cv.transform([text1,text2]).toarray()\n",
"model.predict(text)\n",
"\n",
"\n",
"le.classes_[model.predict(text)]"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'Inglês'"
]
},
"execution_count": 53,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"classes = ['Arabic', 'Danish', 'Dutch', 'English', 'French', 'German',\n",
" 'Greek', 'Hindi', 'Italian', 'Kannada', 'Malayalam', 'Portugeese',\n",
" 'Russian', 'Spanish', 'Sweedish', 'Tamil', 'Turkish']\n",
"\n",
"dict_por = {'Arabic': 'Árabe', 'Danish': 'Dinamarquês', 'Dutch': 'Holandês', 'English': 'Inglês', 'French': 'Francês', 'German': 'Alemão',\n",
" 'Greek': 'Grego', 'Hindi': 'Hindi', 'Italian': 'Italiano', 'Kannada': 'Kannada', 'Malayalam': 'Malaiala', 'Portugeese': 'Português',\n",
" 'Russian': 'Russo', 'Spanish': 'Espanhol', 'Sweedish': 'Sueco', 'Tamil': 'Tâmil', 'Turkish': 'Turco'}\n",
"\n",
"\n",
"def predict_language(text):\n",
" text = re.sub(r'[!@#$(),\\n\"%^*?\\:;~`0-9]', ' ', text)\n",
" text = re.sub(r'[\\[\\]]', ' ', text)\n",
" text = text.lower()\n",
" text = cv.transform([text]).toarray()\n",
" pred = model.predict(text)\n",
" return dict_por[classes[pred[0]]]\n",
"\n",
"\n",
"predict_language('How are?')"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'language': 'English'}\n"
]
}
],
"source": [
"import requests\n",
"\n",
"url = 'https://hudsonma-simpleapplicationdocker.hf.space/predict'\n",
"text1 = ' How are u bro?'\n",
"\n",
"response = requests.post(url, json={'text': text1})\n",
"\n",
"if response.status_code == 200:\n",
" print(response.json())\n",
"else:\n",
" print(f\"Error: {response.status_code}, Detail: {response.text}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}