{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "163a0ec0-2003-40f8-bf8a-63c1146f9130",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nimport os\\n\\n# 设置环境变量\\nos.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\\n\\n# 打印环境变量以确认设置成功\\nprint(os.environ.get('HF_ENDPOINT'))\\n\""
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import subprocess\n",
    "import os\n",
    "\n",
    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
    "output = result.stdout\n",
    "for line in output.splitlines():\n",
    "    if '=' in line:\n",
    "        var, value = line.split('=', 1)\n",
    "        os.environ[var] = value\n",
    "\n",
    "\"\"\"\n",
    "import os\n",
    "\n",
    "# 设置环境变量\n",
    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
    "\n",
    "# 打印环境变量以确认设置成功\n",
    "print(os.environ.get('HF_ENDPOINT'))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4aa15036-4be6-44ac-be6d-83389ae07c5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from datasets import load_dataset\n",
    "\n",
    "# # 加载特定语言的 Wikipedia 语料\n",
    "# langs = [\"20220301.en\", \"20220301.fr\", \"20220301.de\", \"20220301.zh\", \"20220301.ja\"]\n",
    "# datasets = {lang: load_dataset(\"wikimedia/wikipedia\", lang, split=\"train\") for lang in langs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02351414-fcad-44ba-964c-8ced0a88e609",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "408855cc406d47a3a47803d00829febd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset shards:   0%|          | 0/41 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 6458670\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "datasets_en = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.en\")\n",
    "datasets_en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f00a2e59-0aed-446e-9c7b-d21233bcdd1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"en_wiki.txt\",\"w\")\n",
    "for item in datasets_en[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce2039da-ae06-49de-87da-bc452ee1f252",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d240fe1bef53424e9ccbb8af29346f0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00015.parquet:   0%|          | 0.00/764M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "807912e6be304fcaa0786f2c863fa8ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00015.parquet:   0%|          | 0.00/416M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff2349e16ea44a1a9033f04096cf0fe4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00015.parquet:   0%|          | 0.00/342M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "428705e16387469abc2f70d8f01032c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00015.parquet:   0%|          | 0.00/306M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3edadedc0767441fa441450e541d0a41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00004-of-00015.parquet:   0%|          | 0.00/281M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "485c9747448e4fc792c748e3920d893c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00005-of-00015.parquet:   0%|          | 0.00/261M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32acc84b51494535aff3fe9edf17d0f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00006-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1626f0e6390542298f9c22e659e95d52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00007-of-00015.parquet:   0%|          | 0.00/210M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c9d6d941a564add86f644daa4710850",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00008-of-00015.parquet:   0%|          | 0.00/215M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13b0fe00bca644d9b971793375c2bc3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00009-of-00015.parquet:   0%|          | 0.00/211M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60ff775f33b74bf6b9be369f4b38e116",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00010-of-00015.parquet:   0%|          | 0.00/181M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "862390a8eff241b3b371e6db7404591a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00011-of-00015.parquet:   0%|          | 0.00/197M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6162884ffd804cffb1f0d1e6a57e940f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00012-of-00015.parquet:   0%|          | 0.00/180M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d8e1f181e4043019e43704b685851f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00013-of-00015.parquet:   0%|          | 0.00/219M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc7550b48dbe4d35b1c5e1c7ff52f0a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00014-of-00015.parquet:   0%|          | 0.00/220M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d84178aa296430cb2e949c37fbde0e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/2402095 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 2402095\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.fr\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ac610da1-9529-491a-a9f4-5d6f3a92cce7",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"fr_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8be4a828-affd-4b07-8821-16cbefb93bd6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2a4a8425-8615-45c5-84cf-fcfb6a54727a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ade008c4100486e9e47a52fadc1ece7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0/18 [00:00<?, ?files/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbe51753a90240d399d26dfd0bdb1aa6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00010-of-00018.parquet:   0%|          | 0.00/239M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "717f471eccca49e5a415f8b8b8c9c6e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00011-of-00018.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ea2609792144411afa2a855e2ff1c7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00005-of-00018.parquet:   0%|          | 0.00/283M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8c88a13fd5d475085649381fe2fbba7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00008-of-00018.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b63f5caf6c54225b35a3bdc915cc73e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00018.parquet:   0%|          | 0.00/773M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de0f6e9973ab427eaa937924b1134029",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00018.parquet:   0%|          | 0.00/446M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "beac8579da8244deadc2da23aec6ec6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00015-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "293b420636b44bab986752b92c91a1b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00009-of-00018.parquet:   0%|          | 0.00/236M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3d66b5d72ac472aa3ba20dae34f9695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00012-of-00018.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3be8b392a6614472b005ba9b9b1e1d59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00006-of-00018.parquet:   0%|          | 0.00/277M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b371f42b410e4cbf9137666b4bac2647",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00007-of-00018.parquet:   0%|          | 0.00/256M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9ab370c0f534b289fdac4ed3cddff3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00018.parquet:   0%|          | 0.00/369M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dce437fe46e04c82b2f8ab7a3121592b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00018.parquet:   0%|          | 0.00/291M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bec466db34b4daa926a6eef48273817",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00014-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e47f796b69749ef8f3bf255300553ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00004-of-00018.parquet:   0%|          | 0.00/300M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6dd9bf441d24ffa886996c210b4607f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00013-of-00018.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fa5351d1be542b88a3024b900b0ef19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00016-of-00018.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1452efc3d5564ef99ae336dbcc36b22f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00017-of-00018.parquet:   0%|          | 0.00/222M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c12fa5a2d21241c6a31b4c889999ad0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/2665357 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca6b84a24ecf4e00bf8e550ba5eb5054",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset shards:   0%|          | 0/18 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 2665357\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"legacy-datasets/wikipedia\", \"20220301.de\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1c60e53d-3031-44e7-b1da-edc419079f13",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"de_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42100db-79d9-4cbe-9582-e43d58567173",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eee3698d-ccb8-4cec-9796-e93886fd6b1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a03b4c97410645eaaae45f25e1c03108",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00015.parquet:   0%|          | 0.00/612M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a03f25cd8354e3299b82554c0fc3287",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00015.parquet:   0%|          | 0.00/383M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a63110f3cf648eda442ee3177aee2f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00015.parquet:   0%|          | 0.00/305M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09ec9ce700084abcb944f6b6669ebf15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00015.parquet:   0%|          | 0.00/252M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "363a3b6272c24e23a0e708864c872310",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00004-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58b7f596d96246439e7cc21acc333fb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00005-of-00015.parquet:   0%|          | 0.00/240M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2872cc14f4944b4894bd05bbb256d892",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00006-of-00015.parquet:   0%|          | 0.00/233M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d65c3b76a02445e85d6723ea19565c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00007-of-00015.parquet:   0%|          | 0.00/218M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a77dd5822bd34896b4dbd4d22cb8331c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00008-of-00015.parquet:   0%|          | 0.00/224M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52ce12211c8743acac275af2c7ac049c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00009-of-00015.parquet:   0%|          | 0.00/235M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9da3788934f14b15a63e4add38e989b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00010-of-00015.parquet:   0%|          | 0.00/217M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06254bec73a24830b685468235260f0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00011-of-00015.parquet:   0%|          | 0.00/213M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d25a6d36b72446458525dd060d63102a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00012-of-00015.parquet:   0%|          | 0.00/201M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4da023af77894a72a1e6223c91820983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00013-of-00015.parquet:   0%|          | 0.00/193M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c6d63c3e4df46c69da6b42c8b7eae4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00014-of-00015.parquet:   0%|          | 0.00/176M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f98c43bad8a140cca38d7c00232f884a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/1389467 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 1389467\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ja\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4b61a5b0-85ca-4a0e-99b0-0ea2e3bc9e82",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"ja_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c56f7fd-8188-4d30-928d-e307764f23f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1668c12c-cbfe-4ea8-82c6-4e27607e9b67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f8cd7bb3992440c89087f809b96832d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00013.parquet:   0%|          | 0.00/688M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6c6f856ff3f4070a26e56bfacca6deb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00013.parquet:   0%|          | 0.00/376M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d90dc9bb27ef418e906109adf3e7fad3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00013.parquet:   0%|          | 0.00/287M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e72006a2edd24c3a9bd88a0e2ac57d7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00003-of-00013.parquet:   0%|          | 0.00/245M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df667cedb5374cd39e86e52c6cea3a7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00004-of-00013.parquet:   0%|          | 0.00/168M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b005307539743769e0ac974c2996d9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00005-of-00013.parquet:   0%|          | 0.00/178M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c749d7a330f24d2fa684857be0700cc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00006-of-00013.parquet:   0%|          | 0.00/216M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b07061ea4694ba8af395eeae31bd1fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00007-of-00013.parquet:   0%|          | 0.00/241M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e07a225a3f24d9a9c92ad9ee153b066",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00008-of-00013.parquet:   0%|          | 0.00/227M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3054fd1470a34bbab1c6dee28dcb2441",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00009-of-00013.parquet:   0%|          | 0.00/223M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90f7a09fb95b4c4a926711a258c32107",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00010-of-00013.parquet:   0%|          | 0.00/167M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2ceb836cfca47e2ba75659dacfc9739",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00011-of-00013.parquet:   0%|          | 0.00/254M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b20a7cea4fb146a5810cde198ef60ab6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00012-of-00013.parquet:   0%|          | 0.00/226M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba34e3fba4be480cbf145281eb834d64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/1841155 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 1841155\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.es\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9b3d2f1d-47c3-4a4d-8f60-cf5874ec643b",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"es_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d300900-2d36-4f21-a799-17b466239666",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1d20d238-3015-443f-956a-ca1a7c622001",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 1384748\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.zh\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "eac16b65-44d1-43ae-9aeb-c06a470daf0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"zh_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67c13fd3-ce11-41ce-8ca9-7d1d783310fb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8d4f223d-accc-44f5-acf3-5eca7621e960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b39607ab45b4449b96d05858de65c90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00003.parquet:   0%|          | 0.00/400M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c930ca1ff58a4ba2995158c1944335c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00001-of-00003.parquet:   0%|          | 0.00/205M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37654989816342af9a355a2cb6e770b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00002-of-00003.parquet:   0%|          | 0.00/177M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0dbd13bc9dc457eae92438fd63947ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/647897 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'url', 'title', 'text'],\n",
       "        num_rows: 647897\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets_wiki = load_dataset(\"wikimedia/wikipedia\", \"20231101.ko\")\n",
    "datasets_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1af1e848-559a-4882-a3ce-966a63edac44",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = open(\"ko_wiki.txt\",\"w\")\n",
    "for item in datasets_wiki[\"train\"]:\n",
    "    line = item[\"title\"] + \" \"+ item[\"text\"] + \"\\n\"\n",
    "    wiki.write(line)\n",
    "wiki.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4b3810a-f564-4a4e-8ccc-5f23f8541414",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}