Upload 10 files

.gitattributes

@@ -36,3 +36,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 Fast_text_50_dim/shona_fasttext_vectors_50d.kv filter=lfs diff=lfs merge=lfs -text
 Fast_text_100_dim/shona_corpus_E.txt filter=lfs diff=lfs merge=lfs -text
 Fast_text_100_dim/shona_fasttext_vectors_100d.kv filter=lfs diff=lfs merge=lfs -text
+Fast_text_300_dim/shona_corpus_E.txt filter=lfs diff=lfs merge=lfs -text
+Fast_text_300_dim/shona_fasttext_vectors_300d.kv filter=lfs diff=lfs merge=lfs -text

Fast_text_300_dim/.ipynb_checkpoints/FAST_TEXT -300-checkpoint.ipynb

@@ -0,0 +1,324 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from gensim.models import FastText\n",
+    "import regex as re\n",
+    "import time\n",
+    "import os\n",
+    "from gensim.utils import simple_preprocess\n",
+    "from gensim.models import FastText\n",
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def preprocess_text(text):\n",
+    "    text = text.lower()  # Lowercase\n",
+    "    text = re.sub(r'[^\\w\\s]', '', text)  # Remove punctuation\n",
+    "    return simple_preprocess(text)\n",
+    "\n",
+    "def read_corpus(file_path):\n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        for line in file:\n",
+    "            yield preprocess_text(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_file_path = 'shona_corpus_E.txt'\n",
+    "# Read and preprocess the corpus\n",
+    "sentences = list(read_corpus(corpus_file_path))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[['mavambo',\n",
+       "  'kusikwa',\n",
+       "  'kwezvinhu',\n",
+       "  'zvose',\n",
+       "  'pakutanga',\n",
+       "  'mwari',\n",
+       "  'akasika',\n",
+       "  'denga',\n",
+       "  'nepasi'],\n",
+       " ['zvino',\n",
+       "  'rakanga',\n",
+       "  'risina',\n",
+       "  'chiumbo',\n",
+       "  'risina',\n",
+       "  'uye',\n",
+       "  'rakanga',\n",
+       "  'riri',\n",
+       "  'pamusoro',\n",
+       "  'pehwenje'],\n",
+       " ['mweya', 'wamwari', 'wakanga', 'uchidzengerera', 'pamusoro', 'pemvura']]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sentences[:3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start_time = time.time()\n",
+    "\n",
+    "# Train FastText model\n",
+    "model = FastText(\n",
+    "    sentences, \n",
+    "    vector_size=300,  # Higher dimension for better performance\n",
+    "    window=7, \n",
+    "    min_count=5, \n",
+    "    workers=4, \n",
+    "    sg=1,  # Skip-gram model\n",
+    "    epochs=100,  # More epochs for thorough training\n",
+    "    bucket=2000000,  # Large bucket size for handling subwords\n",
+    "    min_n=3,  # Minimum length of char n-grams\n",
+    "    max_n=6   # Maximum length of char n-grams\n",
+    ")\n",
+    "end_time = time.time()\n",
+    "# Calculate the elapsed time\n",
+    "elapsed_time = end_time - start_time\n",
+    "print(\"Time taken:\", elapsed_time, \"minutes\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save the model\n",
+    "model.save(\"shona_fasttext_300d.model\")\n",
+    "model.wv.save(\"shona_fasttext_vectors_300d.kv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def evaluate_similarity(model, word_pairs):\n",
+    "    similarity_scores = []\n",
+    "    for word1, word2, score in word_pairs:\n",
+    "        similarity_score = model.wv.similarity(word1, word2)\n",
+    "        similarity_scores.append((word1, word2, score, similarity_score))\n",
+    "    print(\"Similarity task evaluation:\")\n",
+    "    for word1, word2, human_score, model_score in similarity_scores:\n",
+    "        print(f\"{word1}-{word2}: Human score = {human_score}, Model score = {model_score}\")\n",
+    "\n",
+    "# Example similarity word pairs\n",
+    "similarity_word_pairs = [(\"murume\", \"mukadzi\", 0.8), (\"mwana\", \"mukomana\", 0.6)]\n",
+    "evaluate_similarity(model, similarity_word_pairs)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def perform_analogical_reasoning(model, a, b, c, topn=5):\n",
+    "    d = model.wv[b] - model.wv[a] + model.wv[c]\n",
+    "    closest_words = model.wv.similar_by_vector(d, topn=topn + 3)  # Add extra to ensure we get at least topn unique words\n",
+    "    result_words = [word for word, _ in closest_words if word not in [a, b, c]]\n",
+    "    return result_words[:topn]\n",
+    "\n",
+    "# Example usage\n",
+    "a = \"mukomana\"  # man\n",
+    "b = \"amai\"   # king\n",
+    "c = \"musikana\" # woman\n",
+    "\n",
+    "predicted_words = perform_analogical_reasoning(model, a, b, c)\n",
+    "if predicted_words:\n",
+    "    print(f\"{a} is to {b} as {c} is to: {', '.join(predicted_words)}\")\n",
+    "else:\n",
+    "    print(\"No suitable words found.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Perform Analogical Reasoning\n",
+    "def perform_analogical_reasoning(model, a, b, c, topn=5):\n",
+    "    # Calculate the vector d as b - a + c\n",
+    "    d = model.wv[b] - model.wv[a] + model.wv[c]\n",
+    "    \n",
+    "    # Find the words that best complete the analogy\n",
+    "    closest_words = model.wv.similar_by_vector(d, topn=topn + 3)  # Add extra to ensure we get at least topn unique words\n",
+    "    result_words = [word for word, _ in closest_words if word not in [a, b, c]]\n",
+    "    \n",
+    "    # Ensure we return exactly 'topn' words\n",
+    "    return result_words[:topn]\n",
+    "\n",
+    "# Example usage\n",
+    "a = \"murume\"  # man\n",
+    "b = \"sekuru\"   # king\n",
+    "c = \"mukadzi\" # woman\n",
+    "\n",
+    "predicted_words = perform_analogical_reasoning(model, a, b, c)\n",
+    "if predicted_words:\n",
+    "    print(f\"{a} is to {b} as {c} is to: {', '.join(predicted_words)}\")\n",
+    "else:\n",
+    "    print(\"No suitable words found.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Test similarity\n",
+    "similar_words = model.wv.most_similar(\"kudzidza\", topn=10)\n",
+    "print(similar_words)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

Fast_text_300_dim/FAST_TEXT -300.ipynb

@@ -0,0 +1,346 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from gensim.models import FastText\n",
+    "import regex as re\n",
+    "import time\n",
+    "import os\n",
+    "from gensim.utils import simple_preprocess\n",
+    "from gensim.models import FastText\n",
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def preprocess_text(text):\n",
+    "    text = text.lower()  # Lowercase\n",
+    "    text = re.sub(r'[^\\w\\s]', '', text)  # Remove punctuation\n",
+    "    return simple_preprocess(text)\n",
+    "\n",
+    "def read_corpus(file_path):\n",
+    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
+    "        for line in file:\n",
+    "            yield preprocess_text(line)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_file_path = 'shona_corpus_E.txt'\n",
+    "# Read and preprocess the corpus\n",
+    "sentences = list(read_corpus(corpus_file_path))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[['mavambo',\n",
+       "  'kusikwa',\n",
+       "  'kwezvinhu',\n",
+       "  'zvose',\n",
+       "  'pakutanga',\n",
+       "  'mwari',\n",
+       "  'akasika',\n",
+       "  'denga',\n",
+       "  'nepasi'],\n",
+       " ['zvino',\n",
+       "  'rakanga',\n",
+       "  'risina',\n",
+       "  'chiumbo',\n",
+       "  'risina',\n",
+       "  'uye',\n",
+       "  'rakanga',\n",
+       "  'riri',\n",
+       "  'pamusoro',\n",
+       "  'pehwenje'],\n",
+       " ['mweya', 'wamwari', 'wakanga', 'uchidzengerera', 'pamusoro', 'pemvura']]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sentences[:3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Time taken: 6643.198479413986 minutes\n"
+     ]
+    }
+   ],
+   "source": [
+    "start_time = time.time()\n",
+    "\n",
+    "# Train FastText model\n",
+    "model = FastText(\n",
+    "    sentences, \n",
+    "    vector_size=300,  # Higher dimension for better performance\n",
+    "    window=7, \n",
+    "    min_count=5, \n",
+    "    workers=4, \n",
+    "    sg=1,  # Skip-gram model\n",
+    "    epochs=100,  # More epochs for thorough training\n",
+    "    bucket=2000000,  # Large bucket size for handling subwords\n",
+    "    min_n=3,  # Minimum length of char n-grams\n",
+    "    max_n=6   # Maximum length of char n-grams\n",
+    ")\n",
+    "end_time = time.time()\n",
+    "# Calculate the elapsed time\n",
+    "elapsed_time = end_time - start_time\n",
+    "print(\"Time taken:\", elapsed_time, \"minutes\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save the model\n",
+    "model.save(\"shona_fasttext_300d.model\")\n",
+    "model.wv.save(\"shona_fasttext_vectors_300d.kv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "FastText(vocab=107228, vector_size=300, alpha=0.025)\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Similarity task evaluation:\n",
+      "murume-mukadzi: Human score = 0.8, Model score = 0.6584572196006775\n",
+      "mwana-mukomana: Human score = 0.6, Model score = 0.49458423256874084\n"
+     ]
+    }
+   ],
+   "source": [
+    "def evaluate_similarity(model, word_pairs):\n",
+    "    similarity_scores = []\n",
+    "    for word1, word2, score in word_pairs:\n",
+    "        similarity_score = model.wv.similarity(word1, word2)\n",
+    "        similarity_scores.append((word1, word2, score, similarity_score))\n",
+    "    print(\"Similarity task evaluation:\")\n",
+    "    for word1, word2, human_score, model_score in similarity_scores:\n",
+    "        print(f\"{word1}-{word2}: Human score = {human_score}, Model score = {model_score}\")\n",
+    "\n",
+    "# Example similarity word pairs\n",
+    "similarity_word_pairs = [(\"murume\", \"mukadzi\", 0.8), (\"mwana\", \"mukomana\", 0.6)]\n",
+    "evaluate_similarity(model, similarity_word_pairs)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "mukomana is to baba as musikana is to: bab, babavo, babayo, babangu, babawee\n"
+     ]
+    }
+   ],
+   "source": [
+    "def perform_analogical_reasoning(model, a, b, c, topn=5):\n",
+    "    d = model.wv[b] - model.wv[a] + model.wv[c]\n",
+    "    closest_words = model.wv.similar_by_vector(d, topn=topn + 3)  # Add extra to ensure we get at least topn unique words\n",
+    "    result_words = [word for word, _ in closest_words if word not in [a, b, c]]\n",
+    "    return result_words[:topn]\n",
+    "\n",
+    "# Example usage\n",
+    "a = \"mukomana\"  # man\n",
+    "b = \"baba\"   # king\n",
+    "c = \"musikana\" # woman\n",
+    "\n",
+    "predicted_words = perform_analogical_reasoning(model, a, b, c)\n",
+    "if predicted_words:\n",
+    "    print(f\"{a} is to {b} as {c} is to: {', '.join(predicted_words)}\")\n",
+    "else:\n",
+    "    print(\"No suitable words found.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "murume is to sekuru as mukadzi is to: sasekuru, ambuya, kwavasekuru, sekuruwo, raambuya\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Perform Analogical Reasoning\n",
+    "def perform_analogical_reasoning(model, a, b, c, topn=5):\n",
+    "    # Calculate the vector d as b - a + c\n",
+    "    d = model.wv[b] - model.wv[a] + model.wv[c]\n",
+    "    \n",
+    "    # Find the words that best complete the analogy\n",
+    "    closest_words = model.wv.similar_by_vector(d, topn=topn + 3)  # Add extra to ensure we get at least topn unique words\n",
+    "    result_words = [word for word, _ in closest_words if word not in [a, b, c]]\n",
+    "    \n",
+    "    # Ensure we return exactly 'topn' words\n",
+    "    return result_words[:topn]\n",
+    "\n",
+    "# Example usage\n",
+    "a = \"murume\"  # man\n",
+    "b = \"sekuru\"   # king\n",
+    "c = \"mukadzi\" # woman\n",
+    "\n",
+    "predicted_words = perform_analogical_reasoning(model, a, b, c)\n",
+    "if predicted_words:\n",
+    "    print(f\"{a} is to {b} as {c} is to: {', '.join(predicted_words)}\")\n",
+    "else:\n",
+    "    print(\"No suitable words found.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('nezvemakuhwa', 0.4936619699001312), ('nesu', 0.48648443818092346), ('taurirana', 0.4829455614089966), ('tauraka', 0.46879878640174866), ('ndapota', 0.4648963510990143), ('utaure', 0.45030686259269714), ('chitaura', 0.44970065355300903), ('taurira', 0.4396206736564636), ('tapota', 0.42348620295524597), ('itstechschool', 0.4228824973106384)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Test similarity\n",
+    "similar_words = model.wv.most_similar(\"taura\", topn=10)\n",
+    "print(similar_words)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

Fast_text_300_dim/shona_corpus_E.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b8a3674c729ea64dc6cdf21ad9567b12cfc396f53f19111abb94f022cb4c619
+size 98750355

Fast_text_300_dim/shona_fasttext_300d.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec2d9641ef97d862a30d2fee58dace395c1615208d03b214711fdfcef04b5a7d
+size 3506557

Fast_text_300_dim/shona_fasttext_300d.model.syn1neg.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5d8e3b07cb97ac53e08b80e3b2079e6301906dcb48e8e16b5117a1d4cae8987
+size 128673728

Fast_text_300_dim/shona_fasttext_300d.model.wv.vectors_ngrams.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71b6718b417033cc31b9711e93be71d222683a6250bb50d55f84d4364f024f67
+size 2400000128

Fast_text_300_dim/shona_fasttext_300d.model.wv.vectors_vocab.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:787a8a1d2d0b98cf017f268c455909c477a8a64c5b74c328521656dd9d0ec6e1
+size 128673728

Fast_text_300_dim/shona_fasttext_vectors_300d.kv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332dd5bcb075a04d8d080c6b9e1629710bc5e6567b04c994f6c618fad27508d0
+size 3501803

Fast_text_300_dim/shona_fasttext_vectors_300d.kv.vectors_ngrams.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71b6718b417033cc31b9711e93be71d222683a6250bb50d55f84d4364f024f67
+size 2400000128

Fast_text_300_dim/shona_fasttext_vectors_300d.kv.vectors_vocab.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:787a8a1d2d0b98cf017f268c455909c477a8a64c5b74c328521656dd9d0ec6e1
+size 128673728