Upload 2 files

TextClassification_Patient_Symptoms_and_Diseases (1).ipynb

@@ -0,0 +1,203 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "AIMERS"
+      ],
+      "metadata": {
+        "id": "D1-ngpe5C5_X"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FU57l9-06L5O"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install transformers\n",
+        "import pandas as pd\n",
+        "import re\n",
+        "import spacy\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+        "from sklearn.pipeline import Pipeline\n",
+        "from sklearn.metrics import accuracy_score, classification_report\n",
+        "from sklearn.linear_model import LogisticRegression\n",
+        "#from transformers import TfidfVectorizerForTransformers\n",
+        "\n",
+        "\n",
+        "# Load the data\n",
+        "data = pd.read_csv('symptomssingle.csv')\n",
+        "\n",
+        "# Check for any missing values and remove them\n",
+        "data = data.dropna()\n",
+        "\n",
+        "# Define a function to separate symptoms and diseases from the text\n",
+        "def separate_symptoms_and_diseases(text):\n",
+        "    symptoms = re.findall(r'{\"symptoms\":\"(.*?)\"}', text)\n",
+        "    disease = re.sub(r'(?:{\"symptoms\":\".*?\"},?)+', '', text).strip()\n",
+        "    disease = disease.replace('],', '').strip()  # Remove '],' from the disease name\n",
+        "    return symptoms, disease\n",
+        "\n",
+        "# Apply the function to the data\n",
+        "data['symptoms_and_diseases'] = data['data'].apply(separate_symptoms_and_diseases)\n",
+        "data[['symptoms', 'disease']] = pd.DataFrame(data['symptoms_and_diseases'].tolist(), index=data.index)\n",
+        "data = data.drop(columns=['data', 'symptoms_and_diseases'])\n",
+        "\n",
+        "# Load the spaCy model\n",
+        "nlp = spacy.load('en_core_web_sm')\n",
+        "\n",
+        "# Preprocessing function\n",
+        "def preprocess(symptoms):\n",
+        "    processed_symptoms = []\n",
+        "    for symptom in symptoms:\n",
+        "        doc = nlp(symptom)\n",
+        "        processed_symptom = ' '.join(token.lemma_.lower() for token in doc if not token.is_stop and token.is_alpha)\n",
+        "        processed_symptoms.append(processed_symptom)\n",
+        "    return ' '.join(processed_symptoms)\n",
+        "\n",
+        "# Preprocess the symptoms column\n",
+        "data['symptoms_preprocessed'] = data['symptoms'].apply(preprocess)\n",
+        "\n",
+        "\n",
+        "# Split the data into train and test sets\n",
+        "X_train, X_test, y_train, y_test = train_test_split(data['symptoms_preprocessed'], data['disease'], test_size=0.2, random_state=42)\n",
+        "\n",
+        "\n",
+        "# Create a pipeline for text classification\n",
+        "pipeline = Pipeline([\n",
+        "    ('tfidf', TfidfVectorizer(ngram_range=(1, 2))),\n",
+        "    ('classifier', LogisticRegression(solver='liblinear', C=10))\n",
+        "])\n",
+        "# Train the model\n",
+        "pipeline.fit(X_train, y_train)\n",
+        "\n",
+        "# Make predictions\n",
+        "y_pred = pipeline.predict(X_test)\n",
+        "\n",
+        "# Evaluate the model\n",
+        "print(\"Accuracy: \", accuracy_score(y_test, y_pred))\n",
+        "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "_ERQV-cI1ENp"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install joblib\n",
+        "\n",
+        "import joblib\n",
+        "\n",
+        "# Save the TfidfVectorizer\n",
+        "joblib.dump(pipeline.named_steps['tfidf'], 'tfidf_vectorizer.joblib')\n",
+        "\n",
+        "# Save the Logistic Regression model\n",
+        "joblib.dump(pipeline.named_steps['classifier'], 'logistic_regression_classifier.joblib')\n",
+        "\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "emwnJJVwAupA",
+        "outputId": "9fa7f22f-2909-4431-d57e-5528a7e81663"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: joblib in /usr/local/lib/python3.9/dist-packages (1.1.1)\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "['logistic_regression_classifier.joblib']"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 8
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import joblib\n",
+        "\n",
+        "\n",
+        "# Convert the TfidfVectorizer to a Hugging Face compatible format\n",
+        "tfidf_transformer = TfidfVectorizerForTransformers(pipeline.named_steps['tfidf'])\n",
+        "\n",
+        "# Save the TfidfVectorizer\n",
+        "tfidf_transformer.save_pretrained('tfidf_transformer')\n",
+        "\n",
+        "\n",
+        "# Load the saved model\n",
+        "loaded_pipeline = joblib.load('DiseasePredictionBasedonSymptoms.joblib')\n",
+        "\n",
+        "# Make predictions using the loaded model (example)\n",
+        "sample_symptom = \"Skin Rash\"\n",
+        "processed_symptom = preprocess([sample_symptom])\n",
+        "prediction = loaded_pipeline.predict([processed_symptom])\n",
+        "\n",
+        "print(\"Predicted disease:\", prediction[0])\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "Tu4fmj1bBYNw",
+        "outputId": "a1a33056-3a0d-49ad-8cb8-b356fba6dd73"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Predicted disease: Contact dermatitis\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "CY5qrRCkBGuJ"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}