added dataset and put them into disk for allowing training on them

preprocessed_dataset/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a54c7d588895d262dda3b3da71b9387d49880a7c7b68c7f9e704faf19151b74c
+size 64395784

preprocessed_dataset/dataset_info.json

@@ -0,0 +1,16 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "label": {
+      "dtype": "int64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

preprocessed_dataset/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "b5c71b9d09ecfff8",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

preprocessing.ipynb

@@ -11,9 +11,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\ippe\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "from datasets import load_dataset, Dataset\n",
     "import pandas as pd\n",
@@ -29,7 +38,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -46,17 +55,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "0\n",
       "Amount of duplications in unsupervised: 493\n",
       "Amount of duplications in train: 96\n",
       "Amount of duplications in test: 199\n",
-      "Amount of duplications combined: 911\n",
+      "Amount of duplications combined: 418\n",
       "Amount of duplications between sets: 123\n"
      ]
     }
@@ -68,7 +78,7 @@
     "df_test = pd.DataFrame(dataset[\"test\"][:])  \n",
     "\n",
     "# Combine to check for duplication between sets (inspired by Pandas Docs)\n",
-    "frames = [df_unsupervised, df_test, df_train]\n",
+    "frames = [ df_test, df_train]\n",
     "df_total = pd.concat(frames)\n",
     "\n",
     "# Calculate amount of duplicates for each dataframe\n",
@@ -81,7 +91,9 @@
     "print(\"Amount of duplications in train: \" + str(dups_train)) \n",
     "print(\"Amount of duplications in test: \" + str(dups_test )) \n",
     "print(\"Amount of duplications combined: \" + str(dups_total )) \n",
-    "print(\"Amount of duplications between sets: \" + str(dups_total - (dups_unsupervised + dups_train + dups_test)))\n"
+    "print(\"Amount of duplications between sets: \" + str(dups_total - (  dups_train + dups_test)))\n",
+    "\n",
+    "\n"
    ]
   },
   {
@@ -94,30 +106,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "This TV-series of 10 episodes, broadcast at the end of 2005 on the Russian Telekanal Rossiia, scored unprecedented ratings.<br /><br />It was the second attempt of director Vladimir Bortko to film Bulgakov's masterpiece. In 2000 he had already been solicited by the Kino-Most film studio, associated with the competing channel NTV, but at the last moment the company did not succeed to come to an agreement with Sergei Shilovsky, grandson of Bulgakov's third wife, and owner of the copyrights. This time, with Rossiia, it worked. And it did not pass unnoticed.<br /><br />This TV-epopee of more than 8 hours was heavily criticized, or at least regarded with much skepticism, before it was shown on screen. Sometimes it was sincere and well-grounded concern about the authenticity, but sometimes it seemed as if the Bulgakov die-hards behaved like modern Latunsky's by reproaching a movie they hadn't seen yet with sacrilege. Or maybe it was because of the gigantic publicity campaign that was launched to promote the series, and that could give reasons to fear an ambitious, but superficial Hollywood-ish production. But fortunately it wasn't the case.<br /><br />In contrast with the earlier screen adaptation of Aleksandar Petrovic in 1972, director Vladimire Bortko (° Moscow, 1946) followed the book meticulously. If you have 10 times 52 minutes available for it, it is of course, easier than when you're supposed to deliver a 90 minutes movie picture. The setting of a TV-series appeared to be an ideal format to elaborate the complicated, multidimensional work with many different characters. Bortko had already shown his talent with his TV-adaptation of Fyodor Dostoevsky's The Idiot in 2003. Besides, he already filmed another novel of Bulgakov before: \"Heart of a Dog\", in 1988. He followed the dialogues almost word for word because, so he said, Bulgakov wrote the novel almost like a screenplay.<br /><br />Ik was skeptical too when I saw the DVD at дом книги (Dom Knigi or \"House of Books\") in Moscow. But curiosity was stronger than skepticism and, frankly speaking, I was pleasantly surprised from the first images. Woland's meeting with Ivan and Berlioz, and the first confrontation of Pilate and Yeshua Ha-Notsri are not only beautifully portrayed and well performed, but in addition they matched remarkably well with the images that I had in mind when I first read the book.<br /><br />The three layers of the novel are reflected more than well, with a well manipulated alternation of colour and black-and-white. The actors are casted accurately and they play the characters faithfully to the novel's intentions that even the most convinced skeptics shut their mouths, despite the huge success – on December 29, 2005 more than 80 million people were watching.<br /><br />Must I find demerits? Well... maybe the depiction of Behemoth then. With the existing technologies it could have been done better, but after all I can only conclude that, even though it is \"only\" TV, this series doesn't disenchant and its main merit is probably the the fact that Bulgakov now found a much bigger audience than he ever could have had with his books.\n",
-      "================================================================================\n",
-      "What am I supposed to say about a war film made by Uwe Boll? I know the man by reputation alone and this is my first venture into his film-making domain. It seems he's brought about quite an aura for horrifically bad films, and yet there I was watching Tunnel Rats and genuinely thinking it was a good effort. Am I supposed to sit here and say it's a horrid, pointless mess of fast edits and nonsensical action running on a paper thin script complete with horrid acting? Should that sort of summary be synonymous with a Uwe Boll war film? Well surprise, surprise Tunnel Rats is actually a damn fine effort and it proves people are willing to jump on certain critical bandwagons just as easily as people are willing to jump on positive bandwagons.<br /><br />The film succeeds in the sense it captures the madness of war as well as delivering scenes of strong, bloody violence that repulses more than it does excite as these various action set-pieces and scenarios play out. Hey, this is more than what the recent Rambo film offered when all we got was a plethora of gore and disembowelment as 'justified' warfare was played out between those poor, poor Christians and those evil, evil Burmese soldiers. The primary content and the 'tunnel rats' of the title refers to soldiers whom engage in activity you feel you'd have to be mad to partake in; an activity that is not about capturing or defending terrain; or searching out an individual alá Apocalypse Now or Saving Private Ryan, but about clearing Vietcong tunnels located beneath the battlefields.<br /><br />The Tunnel Rats of the title are three jeep loads of soldiers assigned to the Củ Chi tunnel complex, Vietnam, in 1968. Their task is to clear out the tunnels surrounding their base camp – traps, enemies and all. The platoon are made up of all sorts; these are not just faceless characters called in to spawn some bloody violence/action as they 'blow some stuff up real good' for the benefit of a passive audience. Some are white, some are black; some are younger than others; some are innocent, naive and soft-bodied whereas some others feel the need to stamp authority within the group. Some even share certain religious beliefs that others do not subscribe to.<br /><br />There are some points in which you want the characters whom are down in those tunnels out and 'safe' as soon as possible, then there are others during which you want them down there and 'safe' as potential danger approaches on the surface. Other times, soldiers survive the ordeal of the tunnels only to emerge and face new horrors. Boll toys with the audience in this regard, using each respective 'space' as both a safe haven and a potential death trap at various times to really good effect.<br /><br />The team assigned to deal with this tunnel network share some thoughts and memories from childhood the night before they ship out to begin work. We know the tunnels are a dingy and claustrophobic space on top of a dangerous locale thanks to the opening scene. Further talk of the tunnels being death traps plays out with some characters speculating the dangers through past stories and rumour as well as how the Vietcong can 'smell' you. This makes the scenes later on when a character lights up a cigarette down there even more harrowing. The talk of the tunnels further prolongs anxiety, as the brief but memorable opening scene floats in and around our memory. The tunnels, however, remain off screen and we know what awaits the group, giving us a position of power – a position of power that is further emphasised when we witness entire scenes dedicated to the Vietcong, the American's enemy, one occurrence of which sees the camera crane directly below a Tunnel Rat to reveal a makeshift Vietcong war room.<br /><br />Initially, the first tunnel is a bit of a disaster. It is a dead end and while eliminating two of the enemy, they loose three guys. The sense of failure and frustration at such a cost for so little is clearly evident, very briefly creating a helpless and desperate atmosphere in the film and in our own minds about the situation. Boll captures the horror and the cramped conditions of the tunnels perfectly. Shooting in low light and keeping his camera rock steady as his subject scurries and struggles about erratically, we feel frightened when people venture into the unknown and horrified when altercation with the enemy arises.<br /><br />Boll even finds room to develop scenarios within the already established conventions by including the character of Vo Mai (Jane Le) as this frightened Vietnamese woman who lives within the tunnels with her two young children. The award winning Jane Le does a great job in portraying the fear and madness of it all. The final thirty minutes or so are pure, gripping, impressive war genre cinema. I didn't notice it beforehand, but there is a certain electronic pulsating sound effect/musical number that plays on a loop during this time, which really captures the horror and the suspense you're witnessing as people scrap for their lives – it's fascinating to watch.<br /><br />Whereas Michael Bay can just fetishise action and gunfire with copious amounts of explosions and slow motion towards the end of Transformers as that becomes even more empty headed; vacuous and nonsensical than it already was, and Stallone can offer nothing bar mere break-neck action as the baddies get their comeuppance toward the conclusion of Rambo IV, Boll shows us that war is, in fact, Hell and war-zones are places you really don't ever want to be. The two respective films have high IMDb ratings close to '7'; Tunnel Rats has something bordering on '4' – looks like that Boll-hate bandwagon is in full runaway mode, whereas the Stallone/Bay-love bandwagon is on an equally slick streak. How sad.\n",
-      "================================================================================\n",
-      "I went to Vieshow around 7:30pm and saw the schedule of Lust Caution was lighting in red all the way to midnight. It meant full house the whole night. It's kind of rare in my memory. Only summer blockbusters could have this strong performance, yet their ratings were not restricted! I didn't worry about my ticket. I already ordered on-line. Ang Lee, do make yourself at home. We all love you. <br /><br />And I love the sex scenes. On bed, they use their body languages to show their emotions. Lust and caution are the basic tones, the skin, and what hidden beneath are hatred, anger, revenge, loneliness, redemption, and love. I have never seen so many emotions in scenes of sexual intercourse or lovemaking, whatever you call them. <br /><br />I felt tense during the sex scenes which are indispensable for the whole dramatic arc. I didn't enjoy the lust part, and the caution undercurrent had my heart dangling. If I want to enjoy sex on movie, I would just go to watch porn. People who want to go for that very reason, be prepared to get disappointed. <br /><br />I was also moved by those young patriotic students. By them, Ang Lee tells us he was once like them, and still is now, sending message to the audience through art, through culture, and with passion. <br /><br />Ang Lee seems like to make movie with a lot of metaphors. You can see that judging from the movie titles. Crouching Tiger, Hidden Dragon is not only an idiom, but also meant for the characters and more; Brokeback Mountain is a lost paradise as well; and Lust Caution, for that foreign audience would miss it again, by its Chinese title 色戒 we realize 戒 is also a pun. 戒 is Caution and the diamond ring Mr. Yee gives to Wang too. <br /><br />The diamond ring, when the secretary returns back and says, \"it's yours.\" \"No,\" Mr. Yee says, \"it's not mine.\" <br /><br />I guess it means the diamond ring belongs to Wang. So does his love to her. For the first time, I didn't feel a diamond ring is so superficial like in the TV commercials.<br /><br />From some reviews and news, I noticed Ang Lee and the crew changed Eileen Chang's assassination scene? If so, that is really smart. Anyway, I am going to read Eileen Chang's short story. I am always interested in comparing his films and the original stories or movie scripts. No exception. It's kind of Lust Caution intercourse I enjoy between Ang Lee and me.\n",
-      "================================================================================\n",
-      "Obviously, this film is not for everyone. It is quite a postmodern movie with lots of background meanings. There are hidden connections not only to the movies \"Sukiyaki\" is a remake of, but also to the ancient Japanese \"Heike Monogatari\" (The Tale of the Heike) and even Shakespeare's \"Henry VI\". So, this work could be compared to the Japanese poetic genre \"haikai no renga\", а comic though quite emotional poem. This movie is not a deep philosophical opus, it is quite light and easy to watch, though it is for \"those, who know\". I am quite sure \"Sukiyaki\" will take a deserved place in the connoisseurs' movie library like some other \"experimental\" movies (i.e. \"Rosencrantz & Guildenstern Are Dead\").\n",
-      "================================================================================\n",
-      "The movie is a true Balkan Love story - Sex, Passion, Love, Tragedy. If you watch some of the American romantic comedies before watching Karaula, you'll see the diffеrences. And there's a second line of the story (that is a bit difficult for people outside the Balkans to understand - which is the reason for the words on the screen at the end of the film) - all the main characters are from different ex-Yugoslavian countries, but they are no different as people. So when you start watching the film you get the feeling of a happy story, maybe even comedy, but the film goes deeper and deeper. Pretty good film, makes you think. Actors are perfect, maybe because it's a true story and it's not very different from what happens in their real life. So go and watch it if you have the chance, you wouldn't regret it.\n",
-      "================================================================================\n",
-      "Brought to you by the following among others:<br /><br />1- Yigal Carmon (Hebrew יגאל כרמון) is the president and founder of the Middle East Media Research Institute (MEMRI)<br /><br />Yigal's Career: <br /><br />Colonel, Israeli Army Intelligence from 1968-88 Acting head and adviser on Arab affairs, Civil Administration in Judea and Samaria, 1977-1982<br /><br />2- Raphael Shore is an Israeli-Canadian film writer, producer, and Rabbi employed full time by Aish HaTorah. He is the founder of The Clarion Fund, a non-profit organization that seeks to advance the idea that the United States faces a threat of radical Islam. Shore is also a regular critic of the media coverage on the Israeli-Palestinian conflict, coverage which he alleges is regularly anti-Israel. (LMAO)<br /><br />3- Anti-Defamation League (ADL) Funny how ADL supports this hateful propaganda. You can never tell by reading their \"Anti-Defamation\" name title.<br /><br />Use your mind and see how objective these people are. They have their own agenda!<br /><br />I think, therefore I am.\n",
-      "================================================================================\n",
-      "i guess if they are not brother this film will became very common. how long were they can keep this ? if we were part,what should they do?so natural feelings,so plain and barren words.But I almost cried last night blood relationship brotherhood love knot film.in another word,the elder brother is very cute.if they are not brothers,they won't have so many forbidden factors，from the family、society、friends、even hearts of their own at the very beginning.The elder brother is doubtful of whether he is coming out or not at the beginning .maybe the little brother being so long time with his brother and even can't got any praise from his father，this made him very upset and even sad,maybe this is a key blasting fuse let him feel there were no one in the world loving him except his beloved brother. and i want to say ,this is a so human-natural feeling ,there is nothing to be shamed,you may fell in love your mother、brother、sister.Just a frail heart looking for backbone to rely on\n",
-      "================================================================================\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "\n",
     "# print(dataset)\n",
@@ -125,8 +116,8 @@
     "\n",
     "# check for any non-english characters\n",
     "\n",
-    "def is_not_english(text):       # returns true for unicode > U+007E, which are not used in English\n",
-    "    return bool(re.search(r'[\\u0386-\\u2010\\u2E7F-\\uD7FC]',  text))      \n",
+    "def is_not_english(text):       # returns true for unicode > U+0386, which are not used in English\n",
+    "    return re.sub(r'[\\u0386-\\u2010\\u2E7F-\\uD7FC]', '',  text)     \n",
     "\n",
     "def print_non_english_texts(text_set):\n",
     "    for text in text_set:\n",
@@ -134,9 +125,9 @@
     "            print(text)\n",
     "            print('=' * 80)\n",
     "\n",
-    "print_non_english_texts(dataset[\"unsupervised\"][\"text\"])\n",
-    "print_non_english_texts(dataset[\"test\"][\"text\"])\n",
-    "print_non_english_texts(dataset[\"train\"][\"text\"])"
+    "# print_non_english_texts(dataset[\"unsupervised\"][\"text\"])\n",
+    "# print_non_english_texts(dataset[\"test\"][\"text\"])\n",
+    "# print_non_english_texts(dataset[\"train\"][\"text\"])\n"
    ]
   },
   {
@@ -159,7 +150,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -173,32 +164,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Checking if texts are not in English: 100%|██████████| 50000/50000 [31:18<00:00, 26.62it/s] \n",
-      "Checking if texts are not in English: 100%|██████████| 25000/25000 [15:10<00:00, 27.46it/s]\n",
-      "Checking if texts are not in English: 100%|██████████| 25000/25000 [14:56<00:00, 27.88it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Amount of non-english texts in unsupervised: 2\n",
-      "Amount of non-english texts in train: 0\n",
-      "Amount of non-english texts in test: 1\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+     "ename": "NameError",
+     "evalue": "name 'spacy' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[12], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# load the English model\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m nlp \u001b[38;5;241m=\u001b[39m \u001b[43mspacy\u001b[49m\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124men_core_web_sm\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# create pandas dataframe of text only\u001b[39;00m\n\u001b[0;32m      5\u001b[0m texts_unsupervised \u001b[38;5;241m=\u001b[39m df_unsupervised[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mtolist()\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'spacy' is not defined"
      ]
     }
    ],
@@ -210,10 +187,13 @@
     "texts_unsupervised = df_unsupervised[\"text\"].tolist()\n",
     "texts_train = df_train[\"text\"].tolist()\n",
     "texts_test = df_test[\"text\"].tolist()\n",
+    "texts_total = df_total[\"text\"].tolist()\n",
+    "\n",
     "\n",
     "texts_unsupervised = [str(text) for text in texts_unsupervised]\n",
     "texts_train = [str(text) for text in texts_train]\n",
     "texts_test = [str(text) for text in texts_test]\n",
+    "texts_total = [str(text) for text in texts_total]\n",
     "\n",
     "# Custom language detector factory function\n",
     "@Language.factory(\"language_detector\") \n",
@@ -234,6 +214,7 @@
     "english_checks_unsupervised = [is_not_english(text) for text in tqdm(texts_unsupervised, desc=\"Checking if texts are not in English\")] \n",
     "english_checks_train = [is_not_english(text) for text in tqdm(texts_train, desc=\"Checking if texts are not in English\")] \n",
     "english_checks_test = [is_not_english(text) for text in tqdm(texts_test, desc=\"Checking if texts are not in English\")] \n",
+    "english_checks_total = [is_not_english(text) for text in tqdm(texts_test, desc=\"Checking if texts are not in English\")] \n",
     "\n",
     "# Print the amount of non-english texts:\n",
     "print(\"Amount of non-english texts in unsupervised: \" + str(sum(english_checks_unsupervised)))\n",
@@ -241,6 +222,11 @@
     "print(\"Amount of non-english texts in test: \" + str(sum(english_checks_test)))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 12,
@@ -306,21 +292,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Map:   0%|          | 0/50000 [00:00<?, ? examples/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Map: 100%|██████████| 50000/50000 [00:02<00:00, 17292.43 examples/s]"
+      "Map: 100%|██████████| 50000/50000 [00:03<00:00, 16119.02 examples/s]\n"
      ]
     },
     {
@@ -329,13 +308,6 @@
      "text": [
       "['This is just a precious little diamond. The play, the script are excellent. I cant compare this movie with anything else, maybe except the movie \"Leon\" wonderfully played by Jean Reno and Natalie Portman. But... What can I say about this one? This is the best movie Anne Parillaud has ever played in (See please \"Frankie Starlight\", she\\'s speaking English there) to see what I mean. The story of young punk girl Nikita, taken into the depraved world of the secret government forces has been exceptionally over used by Americans. Never mind the \"Point of no return\" and especially the \"La femme Nikita\" TV series. They cannot compare the original believe me! Trash these videos. Buy this one, do not rent it, BUY it. BTW beware of the subtitles of the LA company which \"translate\" the US release. What a disgrace! If you cant understand French, get a dubbed version. But you\\'ll regret later :)', 'When I say this is my favourite film of all time, that comment is not to be taken lightly. I probably watch far too many films than is healthy for me, and have loved quite a few of them. I first saw \"La Femme Nikita\" nearly ten years ago, and it still manages to be my absolute favourite. Why?This is more than an incredibly stylish and sexy thriller. Luc Besson\\'s great flair for impeccable direction, fashion, and appropriate usage of music makes this a very watchable film. But it is Anne Parillaud\\'s perfect rendering of a complex character who transforms from a heartless killer into a compassionate, vibrant young woman that makes this film beautiful. I can\\'t keep my eyes off of her when she is on screen.I have seen several of Luc Besson\\'s films including \"Subway\", \"The Professional\", and the irritating \"Fifth Element\", and \"Nikita\" is without a doubt, far superior to any of these. Although this film has tragic elements, it is ultimately extremely hopeful. It is the story of a person who is cruel and merciless, who ultimately comes to realize her own humanity and her own personal power. That, to me is extremely inspiring. If there is hope for Nikita, there is hope for all of us.', 'I saw this movie because I am a huge fan of the TV series of the same name starring Roy Dupuis and Pet Wilson. The movie was really good and I saw how the TV show is based on the movie. A few episodes of the TV series came directly from the movie and their similarity was amazing. To keep things short, any fan of the movie has to watch the series and any fan of the series must see the original Nikita.', \"Being that the only foreign films I usually like star a Japanese person in a rubber suit who crushes little tiny buildings and tanks, I had high hopes for this movie. I thought that this was a movie that wouldn't put me to sleep. WRONG! Starts off with a bang, okay, now she's in training, alright, she's an assassin, I'm still with you, oh, now she's having this moral dilemma and she can't decide if she loves her boyfriend or her controller, zzzzz.... Oh well, back to Gamera!\", \"After seeing Point of No Return (a great movie) and being told that the original was better, I was certainly thrilled to see that one of the indie film channels was running La Femme Nikita. Then I saw the movie. Ouch! This was a major let-down.Nikita herself reminds me of Jar Jar Binks more than any other character I've seen recently. She comes across entirely as comic relief. The movie simply has nothing to recommend it besides the core concept of an evil, inhuman character paradoxically learning to be human while training as an assassin, and that concept failed miserably in Nikita due to the poor writing of the title role.\", 'Stylish thriller? Hardly! This French action film from director Luc Besson is more confused and unsure about itself than anything else I have seen recently.The opening action sequence shows some promise, and the following action sequences are good. However they are sadly too few and far between. The drama is dull and erratic, and the characters are generally unlikeable. Anne Parillaud and the support cast are unable to involve the audience emotionally.\"La Femme Nikita\" ends up more like a try hard, B-grade action thriller. Very average!Saturday, April 28, 1991 - Balwyn Cinema', \"A Luc Besson (Léon, The Fith Element, The Messenger: The Story of Joan of Arc. Angel-A) film with Anne Parillaud (Innocent Blood), Tchéky Karyo(The Messenger: The Story of Joan of Arc, The Patriot, Kiss of the Dragon) and Jean Reno (Léon , The Crimson Rivers, Godzilla). It doesn't get better than this.The photography by Thierry Arbogast was perfect, especially the blue lighting at the pharmacy scene. It is easy to see why Besson uses him for his films. He is a master.Parillaud was absolutely magnificent in a character that was a strong woman, and a little girl at the same time. She was a profession, but she was also human. To see this instead of the comic book characters we are so familiar with is refreshing.Reno was a perfect cleaner. Hard and professional. Get the mission done.Jean-Hugues Anglade added just the right touch of sensitivity. Outstanding! There are just not enough superlatives to use in describing the acting and action in the film.\", 'This isn\\'t my typical movie genre. I am more interested in character drama than action movies. This is able to span both genres in a way that Hollywood hasn\\'t managed to accomplish. The plot was interesting and the acting by Anne Parillaud was exceptional.I had seen the American remake, called \"Point of No Return\" and found it to be an OK movie, but I wanted to see the original. I am glad that I did. The technical aspects of direction and editing are very good. The plot was interesting - revolving around the timeless theme of redemption. The acting made this an exceptional movie.For fans of the action genre, they will be quite happy. For those like myself who are more interested in character and plot-driven movies may find this to be a pleasant surprise as well.', \"La Femme Nikita (1990) was a great movie from genre director Luc Besson. The story is about a drugged out young woman (Anne Parillund) who's hung out with the wrong crowd one too many times. During a botched drug store robbery and subsequent shoot out with the police, NIkita is the only survivor. She's tried, sentenced and executed. But, she wakes up in a strange room and greeted by a quiet and subdued individual (Tcheky Kayro) he tells her that she's been given a second chance at life. But will she do what needs to be done in order for her to receive. Will Nikita take this opportunity at a second life or will she throw it all away? Watch LA FEMME NIKITA to find out.The always imitated by never properly duplicated chic thriller has become a fixture in pop culture. Not only was this film remade several times but it also spawned an unrelated television series. Anne Parillund is super hot in her role as Nikita. Watch out for Luc Besson regular Jean Reno as Victor the Cleaner.Highly recommended.\", 'Luc Besson has been behind some of the most provocative and intriguing action films of the last 15 years, and his introduction to mainstream American cinema-goers began with 1991\\'s \"La Femme Nikita.\" Beginning with a drug store shootout that leaves three junkie punkers and a few policemen dead, this film is rapidly engaging and shows off Besson\\'s violent chic. In the fray, only young Nikita (Anne Parillaud) survives, after offing a policeman who had let his guard down at a crucial moment. Facing certain death in France\\'s court of law, she is given two choices: lethal injection, or have her execution faked, and then assume a new life as a female super-assassin/spy. A wild-raised feral child of sorts, Nikita had picked up certain skills on the street that eventually prove to be quite valuable to her government handlers, as she makes things difficult for them and especially her chief mentor Bob (Tcheky Karyo). No secret is made of his affections for her, but he tries to mold Nikita into a valuable person, and he\\'s able to do it with the help of another agent, played by actress Jeanne Moreau, who teaches her the finer ways of womanhood. By the end of her training, Nikita has been modeled into a fine, sexy female assassin. Besson makes no attempts to sacrifice his film in the ways of Hollywood convention, and he proves that the days of female-driven action cinema are not yet dead. Parillaud is compelling, as at first, she almost seems unaware of her identity as a woman. When she isn\\'t taking out high-powered political officials, she\\'s romancing a kind and gentle shop clerk, who falls madly in love with her as she does with him. Lastly, \"La Femme Nikita\" has some of the most stylized and violent action scenes of any movie I\\'ve seen in a while. Luc Besson, with this film and much of his subsequent work, has proved he is one of cinema\\'s most valuable personalities on the European, Asian, and American action film circuits. Also, be aware of a cameo by future Besson \"Leon\" (1994) star Jean Reno as Victor the Cleaner, in what is pretty much an early prototype of his role from that star-making breakthrough film.8/10']\n"
      ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
     }
    ],
    "source": [
@@ -349,7 +321,7 @@
     "    return dataset\n",
     "\n",
     "dataset[\"unsupervised\"] = dataset[\"unsupervised\"].map(remove_html_elements_dataset)\n",
-    "print(dataset[\"unsupervised\"][\"text\"][:10])"
+    "print(dataset[\"unsupervised\"][\"text\"][:10])\n"
    ]
   },
   {
@@ -363,14 +335,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Map: 100%|██████████| 50000/50000 [00:02<00:00, 21841.08 examples/s]\n"
+      "Map: 100%|██████████| 50000/50000 [00:02<00:00, 17272.17 examples/s]\n"
      ]
     },
     {
@@ -392,7 +364,80 @@
     "    return dataset\n",
     "\n",
     "dataset[\"unsupervised\"] = dataset[\"unsupervised\"].map(lower_texts_dataset)\n",
-    "print(dataset[\"unsupervised\"][\"text\"][:10])"
+    "print(dataset[\"unsupervised\"][\"text\"][:10])\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Finalizing dataset\n",
+    "Now that we know all the functions are working correctly we can apply them to the dataset."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "New amount of duplicates in dataframe: 0\n",
+      "              label\n",
+      "count  49580.000000\n",
+      "mean       0.501876\n",
+      "std        0.500002\n",
+      "min        0.000000\n",
+      "25%        0.000000\n",
+      "50%        1.000000\n",
+      "75%        1.000000\n",
+      "max        1.000000\n",
+      "Dataset({\n",
+      "    features: ['text', 'label'],\n",
+      "    num_rows: 49580\n",
+      "})\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Saving the dataset (1/1 shards): 100%|██████████| 49580/49580 [00:00<00:00, 668070.33 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Remove duplicates\n",
+    "df_total = df_total.drop_duplicates()\n",
+    "\n",
+    "print(\"New amount of duplicates in dataframe: \" + str(df_total.duplicated().sum()))\n",
+    "\n",
+    "# remove non-english characters\n",
+    "df_total[\"text\"] = df_total[\"text\"].apply(is_not_english)\n",
+    "\n",
+    "# Remove HTML elements\n",
+    "df_total[\"text\"] = df_total[\"text\"].apply(remove_html_element)\n",
+    "\n",
+    "# Remove the row that contains the many amounts of \"blah\", since applying the Spacy Libary takes a lot of time, we have to filter out the text manualy. Inspired by: https://saturncloud.io/blog/how-to-remove-rows-from-pandas-data-frame-that-contains-any-string-in-a-particular-column/#:~:text=To%20remove%20rows%20from%20a%20pandas%20data%20frame%20that%20contains,a%20specified%20substring%20or%20not.\n",
+    "substring = 'blahblahblahblahblahblah'\n",
+    "filter = df_total['text'].str.contains(substring)\n",
+    "df_total = df_total[~filter]\n",
+    "\n",
+    "\n",
+    "# Set all texts to lowercase\n",
+    "df_total[\"text\"] = df_total[\"text\"].apply(lowercase_text)\n",
+    "\n",
+    "print(df_total.describe())\n",
+    "\n",
+    "df_total = df_total.reset_index(drop=True)\n",
+    "new_dataset = Dataset.from_pandas(df_total)\n",
+    "\n",
+    "print(new_dataset)\n",
+    "\n",
+    "# save dataset to disk, such that it can be used in training\n",
+    "new_dataset.save_to_disk(\"preprocessed_dataset\")\n"
    ]
   }
  ],