{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "19c5ce1a-039d-4c1e-9795-6eb6116cd1ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import pymorphy3\n",
    "from pymorphy3 import MorphAnalyzer\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from collections import defaultdict\n",
    "from sklearn import metrics\n",
    "from time import time\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import Normalizer\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "import requests\n",
    "\n",
    "import sklearn\n",
    "sklearn.set_config(transform_output='pandas')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "991d7d3f-8968-4e99-9148-e17b66761316",
   "metadata": {},
   "source": [
    "Датасет от Валерии"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1de2cc8a-1740-4ebd-aa83-2291cf12e4ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_url</th>\n",
       "      <th>image_url</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10957168</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Маринина А.</td>\n",
       "      <td>Ричард Третий и Генрих Восьмой глазами Шекспира</td>\n",
       "      <td>Расследование жизни и деяний двух самых опороч...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10997075</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Михалкова Е.И.</td>\n",
       "      <td>Знак истинного пути</td>\n",
       "      <td>Сыщики Макар Илюшин и Сергей Бабкин расследуют...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10941892</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Маринина  А.</td>\n",
       "      <td>Генрих Шестой глазами Шекспира</td>\n",
       "      <td>Генрих Шестой – самая трагическая фигура на ан...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10978116</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Маринина А.</td>\n",
       "      <td>Иоанн Безземельный, Эдуард Третий и Ричард Вто...</td>\n",
       "      <td>Истории трех знаменитых королей под одной обло...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10845007</td>\n",
       "      <td>https://static1.bgshop.ru/extraimagehandler.as...</td>\n",
       "      <td>Маринина А.</td>\n",
       "      <td>Шпаргалка для ленивых любителей истории. Корол...</td>\n",
       "      <td>Первая книга в необычном для Александры Марини...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10928065</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Елена Ивановна Михалкова</td>\n",
       "      <td>Мертвый кролик, живой кролик</td>\n",
       "      <td>Самый громкий провал Илюшина: преступление, ко...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         page_url  \\\n",
       "0  https://www.biblio-globus.ru//product/10957168   \n",
       "1  https://www.biblio-globus.ru//product/10997075   \n",
       "2  https://www.biblio-globus.ru//product/10941892   \n",
       "3  https://www.biblio-globus.ru//product/10978116   \n",
       "4  https://www.biblio-globus.ru//product/10845007   \n",
       "5  https://www.biblio-globus.ru//product/10928065   \n",
       "\n",
       "                                           image_url  \\\n",
       "0  https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "1  https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "2  https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "3  https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "4  https://static1.bgshop.ru/extraimagehandler.as...   \n",
       "5  https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "\n",
       "                     author  \\\n",
       "0               Маринина А.   \n",
       "1            Михалкова Е.И.   \n",
       "2              Маринина  А.   \n",
       "3               Маринина А.   \n",
       "4               Маринина А.   \n",
       "5  Елена Ивановна Михалкова   \n",
       "\n",
       "                                               title  \\\n",
       "0    Ричард Третий и Генрих Восьмой глазами Шекспира   \n",
       "1                                Знак истинного пути   \n",
       "2                     Генрих Шестой глазами Шекспира   \n",
       "3  Иоанн Безземельный, Эдуард Третий и Ричард Вто...   \n",
       "4  Шпаргалка для ленивых любителей истории. Корол...   \n",
       "5                       Мертвый кролик, живой кролик   \n",
       "\n",
       "                                          annotation  \n",
       "0  Расследование жизни и деяний двух самых опороч...  \n",
       "1  Сыщики Макар Илюшин и Сергей Бабкин расследуют...  \n",
       "2  Генрих Шестой – самая трагическая фигура на ан...  \n",
       "3  Истории трех знаменитых королей под одной обло...  \n",
       "4  Первая книга в необычном для Александры Марини...  \n",
       "5  Самый громкий провал Илюшина: преступление, ко...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_1 = pd.read_csv('csv_file.csv')\n",
    "data_1.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "57e58eee-625e-47d3-9d48-25a05dca21a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2149 entries, 0 to 2148\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    2149 non-null   object\n",
      " 1   image_url   2149 non-null   object\n",
      " 2   author      2143 non-null   object\n",
      " 3   title       2149 non-null   object\n",
      " 4   annotation  2146 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 84.1+ KB\n"
     ]
    }
   ],
   "source": [
    "data_1.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "79ae4aac-9301-4e90-85a8-65b6a7b6cd33",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_data_1 = data_1.dropna(subset=['author'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d305b67d-4763-46e1-8363-9e2c0f9fb823",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 2140 entries, 0 to 2148\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    2140 non-null   object\n",
      " 1   image_url   2140 non-null   object\n",
      " 2   author      2140 non-null   object\n",
      " 3   title       2140 non-null   object\n",
      " 4   annotation  2140 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 100.3+ KB\n"
     ]
    }
   ],
   "source": [
    "filtered_data_1 = filtered_data_1.dropna(subset=['annotation'])\n",
    "filtered_data_1.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6bb97f1-3efc-45a3-acfa-d3957e6a299c",
   "metadata": {},
   "source": [
    "Датасет от Савра"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7464feea-b7e1-48f0-8206-4f5afdea242f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_2 = pd.read_csv('csv_file_Savr.csv', on_bad_lines='skip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5922c272-303f-4899-b4e3-26ac7133651d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_url</th>\n",
       "      <th>image_url</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10983286</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Лю Цысинь</td>\n",
       "      <td>Темный лес (Воспоминания о прошлом Земли #2. Б...</td>\n",
       "      <td>Трисолярианский кризис продолжается. У землян ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10864732</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Чэнь Д.</td>\n",
       "      <td>Вайолет, созданная из шипов (#1)</td>\n",
       "      <td>Вайолет — предсказательница.  Но никто не знае...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10988948</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Лю Цысинь</td>\n",
       "      <td>Задача трех тел (Воспоминания о прошлом Земли ...</td>\n",
       "      <td>В те времена, когда Китай переживал последстви...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10922727</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Парди К.</td>\n",
       "      <td>Рассвет костяной волшебницы (#2)</td>\n",
       "      <td>Священный долг каждой костяной волшебницы – пе...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10864123</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Кунц Д.</td>\n",
       "      <td>Холодный огонь</td>\n",
       "      <td>Их свел случай, странное происшествие, которое...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         page_url  \\\n",
       "0  https://www.biblio-globus.ru//product/10983286   \n",
       "1  https://www.biblio-globus.ru//product/10864732   \n",
       "2  https://www.biblio-globus.ru//product/10988948   \n",
       "3  https://www.biblio-globus.ru//product/10922727   \n",
       "4  https://www.biblio-globus.ru//product/10864123   \n",
       "\n",
       "                                           image_url     author  \\\n",
       "0  https://static1.bgshop.ru/imagehandler.ashx?fi...  Лю Цысинь   \n",
       "1  https://static1.bgshop.ru/imagehandler.ashx?fi...    Чэнь Д.   \n",
       "2  https://static1.bgshop.ru/imagehandler.ashx?fi...  Лю Цысинь   \n",
       "3  https://static1.bgshop.ru/imagehandler.ashx?fi...   Парди К.   \n",
       "4  https://static1.bgshop.ru/imagehandler.ashx?fi...    Кунц Д.   \n",
       "\n",
       "                                               title  \\\n",
       "0  Темный лес (Воспоминания о прошлом Земли #2. Б...   \n",
       "1                   Вайолет, созданная из шипов (#1)   \n",
       "2  Задача трех тел (Воспоминания о прошлом Земли ...   \n",
       "3                   Рассвет костяной волшебницы (#2)   \n",
       "4                                     Холодный огонь   \n",
       "\n",
       "                                          annotation  \n",
       "0  Трисолярианский кризис продолжается. У землян ...  \n",
       "1  Вайолет — предсказательница.  Но никто не знае...  \n",
       "2  В те времена, когда Китай переживал последстви...  \n",
       "3  Священный долг каждой костяной волшебницы – пе...  \n",
       "4  Их свел случай, странное происшествие, которое...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5ff461c2-aef9-4e77-bf6e-d5b3f7229732",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3531 entries, 0 to 3530\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    3531 non-null   object\n",
      " 1   image_url   1970 non-null   object\n",
      " 2   author      1375 non-null   object\n",
      " 3   title       920 non-null    object\n",
      " 4   annotation  581 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 138.1+ KB\n"
     ]
    }
   ],
   "source": [
    "data_2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c5da28ad-122b-4ef0-900c-08230a0729a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 581 entries, 0 to 3529\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    581 non-null    object\n",
      " 1   image_url   581 non-null    object\n",
      " 2   author      579 non-null    object\n",
      " 3   title       581 non-null    object\n",
      " 4   annotation  581 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 27.2+ KB\n"
     ]
    }
   ],
   "source": [
    "filtered_data_2 = data_2.dropna(subset=['annotation'])\n",
    "filtered_data_2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4df00d79-f0b9-463b-b495-b2696ea6fac3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 579 entries, 0 to 3529\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    579 non-null    object\n",
      " 1   image_url   579 non-null    object\n",
      " 2   author      579 non-null    object\n",
      " 3   title       579 non-null    object\n",
      " 4   annotation  579 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 27.1+ KB\n"
     ]
    }
   ],
   "source": [
    "filtered_data_2 = filtered_data_2.dropna(subset=['author'])\n",
    "filtered_data_2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2b549079-a2e1-4132-9b21-0d77d0ba2e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_1_2 = pd.concat([filtered_data_1, filtered_data_2], ignore_index=True) # объединенный с библиоглобуса"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b6f4c3ae-270a-4a0e-842a-899c08ef878f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2719 entries, 0 to 2718\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    2719 non-null   object\n",
      " 1   image_url   2719 non-null   object\n",
      " 2   author      2719 non-null   object\n",
      " 3   title       2719 non-null   object\n",
      " 4   annotation  2719 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 106.3+ KB\n"
     ]
    }
   ],
   "source": [
    "data_1_2.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ab092ab-8b1f-4cbd-a43c-670c398fee74",
   "metadata": {},
   "source": [
    "Датасет от Сауле (с loveread)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "98e27ab5-74b6-4811-8fec-d499ba024d9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_3 = pd.read_csv('books_v7_500.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b2483fc7-99fa-4456-80d5-9da972f0ec2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_url</th>\n",
       "      <th>image_url</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=10138</td>\n",
       "      <td>http://loveread.ec/img/photo_books/10138.jpg</td>\n",
       "      <td>Нил Шустерман</td>\n",
       "      <td>Книга Громила</td>\n",
       "      <td>Когда Бронте начинает встречаться с Брюстером ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=10150</td>\n",
       "      <td>http://loveread.ec/img/photo_books/10150.jpg</td>\n",
       "      <td>Шеррилин Кеньон</td>\n",
       "      <td>Книга Дьявол может плакать [= Син и Катра ]</td>\n",
       "      <td>Месть. Некоторые говорят, что это яд, просачив...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=10162</td>\n",
       "      <td>http://loveread.ec/img/photo_books/10162.jpg</td>\n",
       "      <td>Бекка Фицпатрик</td>\n",
       "      <td>Книга Тишина</td>\n",
       "      <td>Ссоры между Патчем и Норой закончились. Они ра...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=10166</td>\n",
       "      <td>http://loveread.ec/img/photo_books/10166.jpg</td>\n",
       "      <td>Элизабет Гоудж</td>\n",
       "      <td>Книга Маленькая белая лошадка в серебряном све...</td>\n",
       "      <td>…Лунная Долина полнится воспоминаниями о Лунно...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=10206</td>\n",
       "      <td>http://loveread.ec/img/photo_books/10206.jpg</td>\n",
       "      <td>Кэтрин Коултер</td>\n",
       "      <td>Книга Дикая звезда</td>\n",
       "      <td>С той секунды, как очаровательная Байрони впер...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        page_url  \\\n",
       "495  http://loveread.ec/view_global.php?id=10138   \n",
       "496  http://loveread.ec/view_global.php?id=10150   \n",
       "497  http://loveread.ec/view_global.php?id=10162   \n",
       "498  http://loveread.ec/view_global.php?id=10166   \n",
       "499  http://loveread.ec/view_global.php?id=10206   \n",
       "\n",
       "                                        image_url           author  \\\n",
       "495  http://loveread.ec/img/photo_books/10138.jpg    Нил Шустерман   \n",
       "496  http://loveread.ec/img/photo_books/10150.jpg  Шеррилин Кеньон   \n",
       "497  http://loveread.ec/img/photo_books/10162.jpg  Бекка Фицпатрик   \n",
       "498  http://loveread.ec/img/photo_books/10166.jpg   Элизабет Гоудж   \n",
       "499  http://loveread.ec/img/photo_books/10206.jpg   Кэтрин Коултер   \n",
       "\n",
       "                                                 title  \\\n",
       "495                                      Книга Громила   \n",
       "496        Книга Дьявол может плакать [= Син и Катра ]   \n",
       "497                                       Книга Тишина   \n",
       "498  Книга Маленькая белая лошадка в серебряном све...   \n",
       "499                                 Книга Дикая звезда   \n",
       "\n",
       "                                            annotation  \n",
       "495  Когда Бронте начинает встречаться с Брюстером ...  \n",
       "496  Месть. Некоторые говорят, что это яд, просачив...  \n",
       "497  Ссоры между Патчем и Норой закончились. Они ра...  \n",
       "498  …Лунная Долина полнится воспоминаниями о Лунно...  \n",
       "499  С той секунды, как очаровательная Байрони впер...  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_3.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0f1ea0d3-9845-4550-9d5d-7d4159e4325f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_url</th>\n",
       "      <th>image_url</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=41</td>\n",
       "      <td>http://loveread.ec/img/photo_books/41.jpg</td>\n",
       "      <td>Владимир Набоков</td>\n",
       "      <td>Приглашение на казнь</td>\n",
       "      <td>Это - собрание самых, наверное, странных произ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=1166</td>\n",
       "      <td>http://loveread.ec/img/photo_books/1166.jpg</td>\n",
       "      <td>Агата Кристи</td>\n",
       "      <td>Третья девушка</td>\n",
       "      <td>Молодая девушка по имени Норма Рестарик обраща...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=1187</td>\n",
       "      <td>http://loveread.ec/img/photo_books/1187.jpg</td>\n",
       "      <td>Роджер Желязны</td>\n",
       "      <td>Принеси мне голову Прекрасного принца</td>\n",
       "      <td>Несоврешеннолетний демон Аззи вообразил себя н...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=1289</td>\n",
       "      <td>http://loveread.ec/img/photo_books/1289.jpg</td>\n",
       "      <td>Гарри Гаррисон</td>\n",
       "      <td>Стальную крысу - в президенты !</td>\n",
       "      <td>Великолепный Джим ди Гриз - знаменитый межзвез...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=1352</td>\n",
       "      <td>http://loveread.ec/img/photo_books/1352.jpg</td>\n",
       "      <td>Юз Алешковский</td>\n",
       "      <td>Рука</td>\n",
       "      <td>Роман Юза Алешковского «Рука» (1977, опубл. 19...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     page_url  \\\n",
       "0    http://loveread.ec/view_global.php?id=41   \n",
       "1  http://loveread.ec/view_global.php?id=1166   \n",
       "2  http://loveread.ec/view_global.php?id=1187   \n",
       "3  http://loveread.ec/view_global.php?id=1289   \n",
       "4  http://loveread.ec/view_global.php?id=1352   \n",
       "\n",
       "                                     image_url            author  \\\n",
       "0    http://loveread.ec/img/photo_books/41.jpg  Владимир Набоков   \n",
       "1  http://loveread.ec/img/photo_books/1166.jpg      Агата Кристи   \n",
       "2  http://loveread.ec/img/photo_books/1187.jpg    Роджер Желязны   \n",
       "3  http://loveread.ec/img/photo_books/1289.jpg    Гарри Гаррисон   \n",
       "4  http://loveread.ec/img/photo_books/1352.jpg    Юз Алешковский   \n",
       "\n",
       "                                   title  \\\n",
       "0                   Приглашение на казнь   \n",
       "1                         Третья девушка   \n",
       "2  Принеси мне голову Прекрасного принца   \n",
       "3        Стальную крысу - в президенты !   \n",
       "4                                   Рука   \n",
       "\n",
       "                                          annotation  \n",
       "0  Это - собрание самых, наверное, странных произ...  \n",
       "1  Молодая девушка по имени Норма Рестарик обраща...  \n",
       "2  Несоврешеннолетний демон Аззи вообразил себя н...  \n",
       "3  Великолепный Джим ди Гриз - знаменитый межзвез...  \n",
       "4  Роман Юза Алешковского «Рука» (1977, опубл. 19...  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_3['title'] = data_3['title'].str.replace(r'^Книга\\s+', '', regex=True)\n",
    "data_3.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "17ce45b0-3a29-45ef-8625-33d2f59570ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_main = pd.concat([data_1_2, data_3], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "08f59813-2436-4c2d-b9c9-1ea3222ffd43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3219 entries, 0 to 3218\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    3219 non-null   object\n",
      " 1   image_url   3219 non-null   object\n",
      " 2   author      3219 non-null   object\n",
      " 3   title       3219 non-null   object\n",
      " 4   annotation  3219 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 125.9+ KB\n"
     ]
    }
   ],
   "source": [
    "data_main.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "748b7773-0cb1-430e-aa89-46836488c35e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 726 entries, 0 to 725\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    726 non-null    object\n",
      " 1   image_url   726 non-null    object\n",
      " 2   author      726 non-null    object\n",
      " 3   title       726 non-null    object\n",
      " 4   annotation  726 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 28.5+ KB\n"
     ]
    }
   ],
   "source": [
    "data_4 = pd.read_csv('books_1000.csv')\n",
    "data_4.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a3078611-6002-42b5-b168-329e8c35fc91",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_main = pd.concat([data_main, data_4], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0ce908be-2d12-480d-916c-449c4965f20c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3945 entries, 0 to 3944\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    3945 non-null   object\n",
      " 1   image_url   3945 non-null   object\n",
      " 2   author      3945 non-null   object\n",
      " 3   title       3945 non-null   object\n",
      " 4   annotation  3945 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 154.2+ KB\n"
     ]
    }
   ],
   "source": [
    "data_main.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2f94edbf-16d9-4ed4-bd3b-35d54a9cd594",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_url</th>\n",
       "      <th>image_url</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10974400</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Свечин Н.</td>\n",
       "      <td>Ледяной ветер Суоми</td>\n",
       "      <td>Немудрено, что кассир крупного банка, уроженец...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10974399</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Свечин Н.</td>\n",
       "      <td>Ледяной ветер Суоми</td>\n",
       "      <td>Немудрено, что кассир крупного банка, уроженец...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10893792</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Елена Ивановна Михалкова</td>\n",
       "      <td>Перо бумажной птицы</td>\n",
       "      <td>Пропала девушка. Ее любящая семья нанимает час...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10871804</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Бочарова Т.А.</td>\n",
       "      <td>Восстание Феникса</td>\n",
       "      <td>Всю жизнь Регина Сергеевна проработала учитель...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>https://www.biblio-globus.ru//product/10989481</td>\n",
       "      <td>https://static1.bgshop.ru/imagehandler.ashx?fi...</td>\n",
       "      <td>Корецкий Д.А.</td>\n",
       "      <td>Возвращение не гарантируется</td>\n",
       "      <td>Они познакомились на пустынном ночном шоссе, г...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3423</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=12245</td>\n",
       "      <td>http://loveread.ec/img/photo_books/12245.jpg</td>\n",
       "      <td>Лилия Белая</td>\n",
       "      <td>Книга Мечта</td>\n",
       "      <td>Что ты будешь делать, когда, попадав в другой ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3655</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=14473</td>\n",
       "      <td>http://loveread.ec/img/photo_books/14473.jpg</td>\n",
       "      <td>Эмиль Золя</td>\n",
       "      <td>Книга Мечта</td>\n",
       "      <td>Стояла суровая зима 1860 года, Уаза замерзла, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3800</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=15496</td>\n",
       "      <td>http://loveread.ec/img/photo_books/15496.jpg</td>\n",
       "      <td>Франц Кафка</td>\n",
       "      <td>Книга Превращение</td>\n",
       "      <td>В настоящий том вошли роман Кафки «Замок», при...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3849</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=16065</td>\n",
       "      <td>http://loveread.ec/img/photo_books/16065.jpg</td>\n",
       "      <td>Келли Армстронг</td>\n",
       "      <td>Книга Пробуждение</td>\n",
       "      <td>Если бы мы встретились несколько недель назад,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3933</th>\n",
       "      <td>http://loveread.ec/view_global.php?id=16381</td>\n",
       "      <td>http://loveread.ec/img/photo_books/16381.jpg</td>\n",
       "      <td>Дэй Леклер</td>\n",
       "      <td>Книга Родственные души</td>\n",
       "      <td>Джей Ранделл начала работать в фирме, которая ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            page_url  \\\n",
       "7     https://www.biblio-globus.ru//product/10974400   \n",
       "15    https://www.biblio-globus.ru//product/10974399   \n",
       "31    https://www.biblio-globus.ru//product/10893792   \n",
       "33    https://www.biblio-globus.ru//product/10871804   \n",
       "46    https://www.biblio-globus.ru//product/10989481   \n",
       "...                                              ...   \n",
       "3423     http://loveread.ec/view_global.php?id=12245   \n",
       "3655     http://loveread.ec/view_global.php?id=14473   \n",
       "3800     http://loveread.ec/view_global.php?id=15496   \n",
       "3849     http://loveread.ec/view_global.php?id=16065   \n",
       "3933     http://loveread.ec/view_global.php?id=16381   \n",
       "\n",
       "                                              image_url  \\\n",
       "7     https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "15    https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "31    https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "33    https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "46    https://static1.bgshop.ru/imagehandler.ashx?fi...   \n",
       "...                                                 ...   \n",
       "3423       http://loveread.ec/img/photo_books/12245.jpg   \n",
       "3655       http://loveread.ec/img/photo_books/14473.jpg   \n",
       "3800       http://loveread.ec/img/photo_books/15496.jpg   \n",
       "3849       http://loveread.ec/img/photo_books/16065.jpg   \n",
       "3933       http://loveread.ec/img/photo_books/16381.jpg   \n",
       "\n",
       "                        author                         title  \\\n",
       "7                    Свечин Н.           Ледяной ветер Суоми   \n",
       "15                   Свечин Н.           Ледяной ветер Суоми   \n",
       "31    Елена Ивановна Михалкова           Перо бумажной птицы   \n",
       "33               Бочарова Т.А.             Восстание Феникса   \n",
       "46               Корецкий Д.А.  Возвращение не гарантируется   \n",
       "...                        ...                           ...   \n",
       "3423               Лилия Белая                   Книга Мечта   \n",
       "3655                Эмиль Золя                   Книга Мечта   \n",
       "3800               Франц Кафка             Книга Превращение   \n",
       "3849           Келли Армстронг             Книга Пробуждение   \n",
       "3933                Дэй Леклер        Книга Родственные души   \n",
       "\n",
       "                                             annotation  \n",
       "7     Немудрено, что кассир крупного банка, уроженец...  \n",
       "15    Немудрено, что кассир крупного банка, уроженец...  \n",
       "31    Пропала девушка. Ее любящая семья нанимает час...  \n",
       "33    Всю жизнь Регина Сергеевна проработала учитель...  \n",
       "46    Они познакомились на пустынном ночном шоссе, г...  \n",
       "...                                                 ...  \n",
       "3423  Что ты будешь делать, когда, попадав в другой ...  \n",
       "3655  Стояла суровая зима 1860 года, Уаза замерзла, ...  \n",
       "3800  В настоящий том вошли роман Кафки «Замок», при...  \n",
       "3849  Если бы мы встретились несколько недель назад,...  \n",
       "3933  Джей Ранделл начала работать в фирме, которая ...  \n",
       "\n",
       "[485 rows x 5 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duplicates = data_main[data_main['title'].duplicated(keep=False)]\n",
    "duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "06b29a83-5375-4513-9eee-a164534dc2a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_main_no_duplicates = data_main.drop_duplicates(subset='title', keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "6d63e0b8-e22a-4264-909f-d928edad28d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 3691 entries, 0 to 3944\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   page_url    3691 non-null   object\n",
      " 1   image_url   3691 non-null   object\n",
      " 2   author      3691 non-null   object\n",
      " 3   title       3691 non-null   object\n",
      " 4   annotation  3691 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 173.0+ KB\n"
     ]
    }
   ],
   "source": [
    "data_main_no_duplicates.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65c7f96b-6633-4e41-9802-1919e1b554c9",
   "metadata": {},
   "source": [
    "Preproccesing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e409f07e-3d21-4318-a24a-40dcff1a77e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://raw.githubusercontent.com/stopwords-iso/stopwords-ru/master/stopwords-ru.txt\"\n",
    "stop_words_russian = requests.get(url).text.splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "89a1ff27-3f90-40cd-9eb6-97c13eb00ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(text):\n",
    "    text = re.sub(r'([А-Я])', r' \\1', text) # делаем пробел между верхним и нижним регистром(123/473)\n",
    "    text = text.lower() # нижний регистр (194)\n",
    "    text = re.sub(r'[\\w\\.-]+\\.ru', \" \", text) # удаляем сайты (4)\n",
    "    text = re.sub(r'http\\S+', \" \", text) # удаляем ссылки (10/1076)\n",
    "    text = re.sub(r'@\\w+',' ',text) # удаляем упоминания пользователей (1937/1923)\n",
    "    text = re.sub(r'#\\w+', ' ', text) # удаляем хэштеги (10/77/1923)\n",
    "    text = re.sub(r'\\d+', ' ', text) # удаляем числа()\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    # Удаление английских слов\n",
    "    text = ' '.join(re.findall(r'\\b[а-яА-ЯёЁ]+\\b', text))\n",
    "    # Удаление стоп-слов\n",
    "    tokens = word_tokenize(text)\n",
    "    text = ' '.join([word for word in tokens if word.lower() not in stop_words_russian])\n",
    "    # text = re.sub(r'<.*?>',' ', text) #\n",
    "    text = re.sub(r'['u'\\U0001F600-\\U0001F64F'\n",
    "                  u'\\U0001F300-\\U0001F5FF'\n",
    "                  u'\\U0001F680-\\U0001F6FF'\n",
    "                  u'\\U0001F1E0-\\U0001F1FF'\n",
    "                  u'\\U00002500-\\U00002BEF'\n",
    "                  u'\\U00002702-\\U000027B0'\n",
    "                  u'\\U000024C2-\\U0001F251'\n",
    "                  u'\\U0001f926-\\U0001f937'\n",
    "                  u'\\U00010000-\\U0010ffff'\n",
    "                    ']+', '', text, flags=re.UNICODE) # удаляем эмоджи(4/1707)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3e7d68bb-09b8-48c3-875c-eece6f9cb7c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3152/2295399223.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data_main_no_duplicates['cleaned_text'] = data_main_no_duplicates['annotation'].apply(clean)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0       расследование жизни деяний самых опороченных в...\n",
       "1       сыщики макар илюшин сергей бабкин расследуют п...\n",
       "2       генрих самая трагическая фигура английском пре...\n",
       "3       истории трех знаменитых королей обложкой посво...\n",
       "4       первая необычном александры марининой жанре по...\n",
       "                              ...                        \n",
       "3940    ирма повести бизнесплан аспирантки помышляла б...\n",
       "3941    основная идея романа давление обстоятельств жи...\n",
       "3942    небольшой горный пансионат игл нест туристичес...\n",
       "3943    пола привыкла разговоры любви духовном физичес...\n",
       "3944    последняя пристройка крепости мак леннан завер...\n",
       "Name: cleaned_text, Length: 3691, dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_main_no_duplicates['cleaned_text'] = data_main_no_duplicates['annotation'].apply(clean)\n",
    "data_cleaned = data_main_no_duplicates['cleaned_text']\n",
    "data_cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b7ddc1cf-6824-4b6c-9364-0f14327c96b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "morph = pymorphy3.MorphAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "c4fc3c3e-92a6-4853-a399-f7ef4669b70c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_batch(texts, batch_size=1000):\n",
    "    \"\"\"Лемматизация текста по батчам для ускорения.\"\"\"\n",
    "    lemmatized_texts = []\n",
    "    for i in tqdm(range(0, len(texts), batch_size)):\n",
    "        batch = texts[i:i+batch_size]\n",
    "        lemmatized_batch = [' '.join([morph.parse(word)[0].normal_form for word in text.split()]) for text in batch]\n",
    "        lemmatized_texts.extend(lemmatized_batch)\n",
    "    return lemmatized_texts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "caff9227-3d13-49f4-898a-781fd3283d2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 4/4 [00:20<00:00,  5.15s/it]\n",
      "/tmp/ipykernel_3152/3826416347.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data_main_no_duplicates['lemmatized_text'] = lemmatize_batch(data_main_no_duplicates['cleaned_text'].tolist())\n"
     ]
    }
   ],
   "source": [
    "data_main_no_duplicates['lemmatized_text'] = lemmatize_batch(data_main_no_duplicates['cleaned_text'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f9271faa-f093-4f7c-9944-f822765f538c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       расследование жизнь деяние самый опорочить воз...\n",
       "1       сыщик макар илюшин сергей бабкин расследовать ...\n",
       "2       генрих самый трагический фигура английский пре...\n",
       "3       история три знаменитый король обложка посвоему...\n",
       "4       первый необычный александра маринин жанр попул...\n",
       "                              ...                        \n",
       "3940    ирма повесть бизнесплан аспирантка помышлять б...\n",
       "3941    основный идея роман давление обстоятельство жи...\n",
       "3942    небольшой горный пансионат игла несть туристич...\n",
       "3943    пол привыкнуть разговор любовь духовный физиче...\n",
       "3944    последний пристройка крепость мак леннана заве...\n",
       "Name: lemmatized_text, Length: 3691, dtype: object"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_main_no_duplicates['lemmatized_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1158703d-4ebe-4a18-93d2-ef412363deb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_main_no_duplicates['lemmatized_text'].to_csv('lemmatized_data')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f540af02-2165-4138-b77c-88d0ef89cd3a",
   "metadata": {},
   "source": [
    "Убрать из названия знаки препинания"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cc96e83-9893-4dd7-ba86-c466d6fcebf0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}