add more files to repo

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY . /app
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 8501
+
+CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]

notebooks/embeddings.ipynb

@@ -0,0 +1,374 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:02:38.555535Z",
+     "start_time": "2024-08-15T13:02:37.106569Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "pd.options.display.float_format = '{:,.2f}'.format\n",
+    "import torch\n",
+    "import faiss"
+   ],
+   "id": "ae2dfff5aa25ae18",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:02:42.684945Z",
+     "start_time": "2024-08-15T13:02:41.287350Z"
+    }
+   },
+   "source": "from sentence_transformers import SentenceTransformer",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T12:54:42.598181Z",
+     "start_time": "2024-08-14T12:54:39.398163Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "model = SentenceTransformer(\"cointegrated/rubert-tiny2\")",
+   "id": "989af6384d4e5a8e",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "dd56bb4ebdc84557b8dee2060631e9d3"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "README.md:   0%|          | 0.00/2.19k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "14e4f50b6d6c481e9d76ca68ef5bbef5"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "sentence_bert_config.json:   0%|          | 0.00/54.0 [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "d11b7d08f8d7464c8f4e3b6509d01d1f"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "1_Pooling/config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "b943691acd3343d199d063bc87c1baaf"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "execution_count": 10
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T12:55:31.493814Z",
+     "start_time": "2024-08-14T12:55:31.471146Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "df = pd.read_csv('../data/data.csv')",
+   "id": "1a0b35d6e0fbb3c1",
+   "outputs": [],
+   "execution_count": 13
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T12:56:16.355759Z",
+     "start_time": "2024-08-14T12:56:16.353642Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "sentences = df['description'].tolist()",
+   "id": "23b45de76e63745f",
+   "outputs": [],
+   "execution_count": 15
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T12:56:33.698140Z",
+     "start_time": "2024-08-14T12:56:26.610169Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "embeddings = model.encode(sentences)",
+   "id": "d1b4f38e54c2c927",
+   "outputs": [],
+   "execution_count": 16
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T12:56:36.694104Z",
+     "start_time": "2024-08-14T12:56:36.690947Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "print(embeddings.shape)",
+   "id": "9724188f0fc24546",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(1049, 312)\n"
+     ]
+    }
+   ],
+   "execution_count": 17
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T13:02:23.235437Z",
+     "start_time": "2024-08-14T13:02:23.233630Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "text = 'Сериал про джедаев'",
+   "id": "1d44f85a9a1b5b85",
+   "outputs": [],
+   "execution_count": 48
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T13:02:23.758117Z",
+     "start_time": "2024-08-14T13:02:23.689210Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "text_embedding = model.encode(text)",
+   "id": "e98b8b855ab1fdcf",
+   "outputs": [],
+   "execution_count": 49
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": 60,
+   "source": "search = model.similarity(embeddings, text_embedding)",
+   "id": "5b23c0aef0e3022"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T13:05:08.343582Z",
+     "start_time": "2024-08-14T13:05:08.340619Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "search.shape",
+   "id": "d2059301c9734464",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([1049, 1])"
+      ]
+     },
+     "execution_count": 61,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 61
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T13:38:59.864081Z",
+     "start_time": "2024-08-14T13:38:59.860326Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "torch.argmax(search).item()",
+   "id": "e696066aab627853",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "905"
+      ]
+     },
+     "execution_count": 65,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 65
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T20:45:20.736400Z",
+     "start_time": "2024-08-14T20:45:20.734292Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def load_model():\n",
+    "    return SentenceTransformer(\"paraphrase-multilingual-mpnet-base-v2\")"
+   ],
+   "id": "9e2ed7e66abc6a49",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:03:08.428996Z",
+     "start_time": "2024-08-15T13:03:00.332695Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "model = load_model()",
+   "id": "532e538411ab573f",
+   "outputs": [],
+   "execution_count": 4
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:03:11.008048Z",
+     "start_time": "2024-08-15T13:03:11.003802Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def compute_index(_model, sentences):\n",
+    "    embeddings = _model.encode(sentences).astype('float32')\n",
+    "    normalized_embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)\n",
+    "\n",
+    "    index_l2 = faiss.IndexFlatL2(embeddings.shape[1])\n",
+    "    index_dot = faiss.IndexFlatIP(embeddings.shape[1])\n",
+    "    index_cosine = faiss.IndexFlatIP(embeddings.shape[1])\n",
+    "\n",
+    "    index_l2.add(embeddings)\n",
+    "    index_dot.add(embeddings)\n",
+    "    index_cosine.add(normalized_embeddings)\n",
+    "\n",
+    "    return index_l2, index_dot, index_cosine"
+   ],
+   "id": "e7af30a720365bd5",
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:03:13.039314Z",
+     "start_time": "2024-08-15T13:03:12.964545Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "data = pd.read_csv('../data/data.csv')\n",
+    "sentences = data['description'].tolist()"
+   ],
+   "id": "ed466f547af86849",
+   "outputs": [],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:05:33.501428Z",
+     "start_time": "2024-08-15T13:03:14.381677Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "index_l2, index_dot, index_cosine = compute_index(model, sentences)",
+   "id": "1ae620dfaa1177d7",
+   "outputs": [],
+   "execution_count": 7
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-15T13:05:33.537496Z",
+     "start_time": "2024-08-15T13:05:33.503326Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "faiss.write_index(index_l2, \"../models/index_l2.faiss\")\n",
+    "faiss.write_index(index_dot, \"../models/index_dot.faiss\")\n",
+    "faiss.write_index(index_cosine, \"../models/index_cosine.faiss\")"
+   ],
+   "id": "cf07f22fb88110ed",
+   "outputs": [],
+   "execution_count": 8
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/parsing.ipynb

@@ -0,0 +1,367 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:49:15.441508Z",
+     "start_time": "2024-08-14T17:49:14.960186Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "import re\n",
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "from selenium.webdriver.common.keys import Keys\n",
+    "from selenium.webdriver.chrome.service import Service\n",
+    "from webdriver_manager.chrome import ChromeDriverManager\n",
+    "import time\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "import pandas as pd"
+   ],
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:49:16.227697Z",
+     "start_time": "2024-08-14T17:49:16.224674Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "url_list = []\n",
+    "image_list = []\n",
+    "title_list = []\n",
+    "info_list = []\n",
+    "description_list = []\n",
+    "rating_list = []"
+   ],
+   "id": "3a5a63adab6eb73d",
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:49:18.720761Z",
+     "start_time": "2024-08-14T17:49:18.718782Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "url = 'https://www.film.ru/serials/vo-vse-tyazhkie'\n",
+    "headers = {\n",
+    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'\n",
+    "}"
+   ],
+   "id": "201dd8c0282195e6",
+   "outputs": [],
+   "execution_count": 3
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:49:27.912392Z",
+     "start_time": "2024-08-14T17:49:27.908710Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def add_info(url):\n",
+    "    response = requests.get(url, headers=headers)\n",
+    "    soup = BeautifulSoup(response.text, 'lxml')\n",
+    "    description = soup.find(class_='wrapper_movies_text').text.replace('\\n', '')\n",
+    "    title = soup.find('h1').text\n",
+    "    info = soup.find('div', class_='block_table').text.replace('\\n', ' ').strip()\n",
+    "    img = soup.find('a', class_='wrapper_block_stack wrapper_movies_poster').get('data-src')\n",
+    "    ratings = soup.find_all('div', class_='wrapper_movies_scores_score')\n",
+    "    \n",
+    "    imdb_score = '-'\n",
+    "    \n",
+    "    for rating in ratings:\n",
+    "        try:\n",
+    "            rating = rating.text.split()\n",
+    "            if rating[1] == 'IMDb':\n",
+    "                imdb_score = rating[0]\n",
+    "                break\n",
+    "        except:\n",
+    "            pass\n",
+    "    \n",
+    "    match = re.match(r'([^()]+)', title)\n",
+    "    if match:\n",
+    "        title = match.group(1).strip()\n",
+    "        \n",
+    "    info = info.replace('длительность ', '')\n",
+    "    \n",
+    "    url_list.append(url)\n",
+    "    image_list.append('https://www.film.ru' + img)\n",
+    "    title_list.append(title)\n",
+    "    info_list.append(info)\n",
+    "    description_list.append(description)\n",
+    "    rating_list.append(imdb_score)"
+   ],
+   "id": "c6cefa2a540572b1",
+   "outputs": [],
+   "execution_count": 4
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T15:04:04.815204Z",
+     "start_time": "2024-08-14T15:03:59.311545Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))\n",
+    "driver.get('https://www.film.ru/a-z/serials')"
+   ],
+   "id": "69dbfb4122848980",
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T15:30:19.653512Z",
+     "start_time": "2024-08-14T15:04:17.495683Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "base_url = 'https://www.film.ru'\n",
+    "\n",
+    "top_url = 'https://www.film.ru/a-z/serials'\n",
+    "\n",
+    "response = requests.get(top_url, headers=headers)\n",
+    "soup = BeautifulSoup(response.text, 'lxml')\n",
+    "\n",
+    "links = set()\n",
+    "while len(links) < 5000:\n",
+    "    driver.find_element(By.TAG_NAME, 'body').send_keys(Keys.END)\n",
+    "    time.sleep(2)\n",
+    "    \n",
+    "    elements = driver.find_elements(By.CSS_SELECTOR, 'a.redesign_afisha_movie_main_title')\n",
+    "\n",
+    "    for element in elements:\n",
+    "        link = element.get_attribute('href')\n",
+    "        if link and link.startswith('https://www.film.ru/'):\n",
+    "            links.add(link)\n",
+    "\n",
+    "driver.quit()"
+   ],
+   "id": "28b12c8e79a361b",
+   "outputs": [],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T15:30:31.175426Z",
+     "start_time": "2024-08-14T15:30:31.172648Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "links = list(links)",
+   "id": "8ff5f0471f4c761c",
+   "outputs": [],
+   "execution_count": 7
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:50:57.870245Z",
+     "start_time": "2024-08-14T17:50:57.865304Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "links = []\n",
+    "with open('../data/links.txt', 'r') as f:\n",
+    "    for line in f:\n",
+    "        links.append(line.strip())"
+   ],
+   "id": "ed6bf0739d9afda2",
+   "outputs": [],
+   "execution_count": 10
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T17:50:59.134056Z",
+     "start_time": "2024-08-14T17:50:59.130899Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(links)",
+   "id": "ebc31a62f4d06d31",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "5040"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 11
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T15:31:34.654081Z",
+     "start_time": "2024-08-14T15:31:34.650167Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# with open('../data/links.txt', 'w') as f:\n",
+    "#     for link in links:\n",
+    "#         f.write(link + '\\n')"
+   ],
+   "id": "32b00a8e9ba45c05",
+   "outputs": [],
+   "execution_count": 9
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:29.984692Z",
+     "start_time": "2024-08-14T17:51:14.351523Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "for link in tqdm(links, desc=\"Парсинг ссылок\"):\n",
+    "    try:\n",
+    "        add_info(link)\n",
+    "    except:\n",
+    "        pass"
+   ],
+   "id": "78799c96450db456",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Парсинг ссылок: 100%|██████████| 5040/5040 [28:15<00:00,  2.97it/s]  \n"
+     ]
+    }
+   ],
+   "execution_count": 13
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:35.016156Z",
+     "start_time": "2024-08-14T18:19:35.012126Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "len(description_list) == len(title_list) == len(info_list) == len(image_list) == len(url_list)==len(rating_list)",
+   "id": "a058eaa1450a171d",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 14
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:47.473578Z",
+     "start_time": "2024-08-14T18:19:47.469096Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "data = pd.DataFrame({'page_url': url_list, 'image_url': image_list, 'tvshow_title': title_list, 'info': info_list, 'description': description_list, 'rating': rating_list})",
+   "id": "301b579630a1ef56",
+   "outputs": [],
+   "execution_count": 16
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:47.847888Z",
+     "start_time": "2024-08-14T18:19:47.845311Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "def clean_description(text):\n",
+    "    cleaned_text = re.sub(r'Сезон \\d+', '', text)\n",
+    "    cleaned_text = re.sub(r'\\s+', ' ', cleaned_text).strip()\n",
+    "    return cleaned_text"
+   ],
+   "id": "94a061cf7812343e",
+   "outputs": [],
+   "execution_count": 17
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:48.380448Z",
+     "start_time": "2024-08-14T18:19:48.317552Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "data['description'] = data['description'].apply(clean_description)",
+   "id": "8597746104442cc7",
+   "outputs": [],
+   "execution_count": 18
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-08-14T18:19:49.497581Z",
+     "start_time": "2024-08-14T18:19:49.438608Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "data.to_csv('../data/data.csv')",
+   "id": "4423b3bffd94bbbf",
+   "outputs": [],
+   "execution_count": 19
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}