fix blake3 (vaex dependency)

requirements.txt

@@ -6,6 +6,7 @@ xlrd==2.0.1
 openpyxl==3.0.9
 watchdog==2.1.6
 vaex==4.7.0
+blake3==0.2.1  # to make vaex work
 
 # english
 https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0.tar.gz#egg=en_core_web_sm

tests/notebook.ipynb

@@ -12,7 +12,150 @@
     "import sys\n",
     "sys.path.append(\"..\")\n",
     "from src.preprocessing import PreprocessingPipeline\n",
-    "import pandas as pd"
+    "import pandas as pd\n",
+    "import vaex"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "----\n",
+    "### Test vaex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(\"../data/test_en.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>#                                </th><th>label  </th><th>text                                               </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td><i style='opacity: 0.6'>0</i>    </td><td>0      </td><td>&quot;I think it&#x27;s time John Rambo move on with his l...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>1</i>    </td><td>1      </td><td>&quot;I&#x27;ve just watch 2 films of Pang brothers, The E...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>2</i>    </td><td>1      </td><td>&#x27;Jewel Thief is *THE* crime thriller of Bollywoo...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>3</i>    </td><td>0      </td><td>&#x27;This so called remake is terrible. I went to se...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4</i>    </td><td>1      </td><td>&#x27;When Northfork debuted at the Cannes Film Festi...</td></tr>\n",
+       "<tr><td>...                              </td><td>...    </td><td>...                                                </td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4,995</i></td><td>0      </td><td>&#x27;The title tells it all -- Ed Gein, the butcher ...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4,996</i></td><td>0      </td><td>&quot;This film makes about as much sense as an &#x27;Ozzi...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4,997</i></td><td>0      </td><td>&#x27;&quot;Sex and the City&quot; has some great things going ...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4,998</i></td><td>0      </td><td>&#x27;Please...if anybody gets the chance to read thi...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4,999</i></td><td>0      </td><td>&#x27;...a film comes along that manages to be absolu...</td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "#      label    text\n",
+       "0      0        \"I think it's time John Rambo move on with his l...\n",
+       "1      1        \"I've just watch 2 films of Pang brothers, The E...\n",
+       "2      1        'Jewel Thief is *THE* crime thriller of Bollywoo...\n",
+       "3      0        'This so called remake is terrible. I went to se...\n",
+       "4      1        'When Northfork debuted at the Cannes Film Festi...\n",
+       "...    ...      ...\n",
+       "4,995  0        'The title tells it all -- Ed Gein, the butcher ...\n",
+       "4,996  0        \"This film makes about as much sense as an 'Ozzi...\n",
+       "4,997  0        '\"Sex and the City\" has some great things going ...\n",
+       "4,998  0        'Please...if anybody gets the chance to read thi...\n",
+       "4,999  0        '...a film comes along that manages to be absolu..."
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vaex.from_pandas(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_small = df.iloc[:1000]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>#                              </th><th>label  </th><th>text                                               </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td><i style='opacity: 0.6'>0</i>  </td><td>0      </td><td>&quot;I think it&#x27;s time John Rambo move on with his l...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>1</i>  </td><td>1      </td><td>&quot;I&#x27;ve just watch 2 films of Pang brothers, The E...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>2</i>  </td><td>1      </td><td>&#x27;Jewel Thief is *THE* crime thriller of Bollywoo...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>3</i>  </td><td>0      </td><td>&#x27;This so called remake is terrible. I went to se...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>4</i>  </td><td>1      </td><td>&#x27;When Northfork debuted at the Cannes Film Festi...</td></tr>\n",
+       "<tr><td>...                            </td><td>...    </td><td>...                                                </td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>995</i></td><td>1      </td><td>&quot;It&#x27;s a funny business, reviewing movies. These ...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>996</i></td><td>1      </td><td>&#x27;Right from the start you see that &quot;Anchors Awei...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>997</i></td><td>0      </td><td>&#x27;I saw this movie in NEW York city. I was waitin...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>998</i></td><td>0      </td><td>&#x27;Firstly, this is NOT an adaptation of a Stephen...</td></tr>\n",
+       "<tr><td><i style='opacity: 0.6'>999</i></td><td>1      </td><td>&quot;Barbra Streisand&#x27;s debut television special is ...</td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "#    label    text\n",
+       "0    0        \"I think it's time John Rambo move on with his l...\n",
+       "1    1        \"I've just watch 2 films of Pang brothers, The E...\n",
+       "2    1        'Jewel Thief is *THE* crime thriller of Bollywoo...\n",
+       "3    0        'This so called remake is terrible. I went to se...\n",
+       "4    1        'When Northfork debuted at the Cannes Film Festi...\n",
+       "...  ...      ...\n",
+       "995  1        \"It's a funny business, reviewing movies. These ...\n",
+       "996  1        'Right from the start you see that \"Anchors Awei...\n",
+       "997  0        'I saw this movie in NEW York city. I was waitin...\n",
+       "998  0        'Firstly, this is NOT an adaptation of a Stephen...\n",
+       "999  1        \"Barbra Streisand's debut television special is ..."
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vaex.from_pandas(df_small)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "----"
    ]
   },
   {