Upload google_collab.ipynb
Browse files- google_collab.ipynb +608 -0
google_collab.ipynb
ADDED
|
@@ -0,0 +1,608 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"nbformat": 4,
|
| 3 |
+
"nbformat_minor": 0,
|
| 4 |
+
"metadata": {
|
| 5 |
+
"colab": {
|
| 6 |
+
"provenance": []
|
| 7 |
+
},
|
| 8 |
+
"kernelspec": {
|
| 9 |
+
"name": "python3",
|
| 10 |
+
"display_name": "Python 3"
|
| 11 |
+
},
|
| 12 |
+
"language_info": {
|
| 13 |
+
"name": "python"
|
| 14 |
+
}
|
| 15 |
+
},
|
| 16 |
+
"cells": [
|
| 17 |
+
{
|
| 18 |
+
"cell_type": "code",
|
| 19 |
+
"execution_count": null,
|
| 20 |
+
"metadata": {
|
| 21 |
+
"colab": {
|
| 22 |
+
"base_uri": "https://localhost:8080/",
|
| 23 |
+
"height": 73
|
| 24 |
+
},
|
| 25 |
+
"id": "FaSiqVnTItLq",
|
| 26 |
+
"outputId": "1ef8a78c-7421-41eb-8cf4-db8426edeed9"
|
| 27 |
+
},
|
| 28 |
+
"outputs": [
|
| 29 |
+
{
|
| 30 |
+
"output_type": "display_data",
|
| 31 |
+
"data": {
|
| 32 |
+
"text/plain": [
|
| 33 |
+
"<IPython.core.display.HTML object>"
|
| 34 |
+
],
|
| 35 |
+
"text/html": [
|
| 36 |
+
"\n",
|
| 37 |
+
" <input type=\"file\" id=\"files-a9b68a68-38ec-4a0a-8037-171b8cfec796\" name=\"files[]\" multiple disabled\n",
|
| 38 |
+
" style=\"border:none\" />\n",
|
| 39 |
+
" <output id=\"result-a9b68a68-38ec-4a0a-8037-171b8cfec796\">\n",
|
| 40 |
+
" Upload widget is only available when the cell has been executed in the\n",
|
| 41 |
+
" current browser session. Please rerun this cell to enable.\n",
|
| 42 |
+
" </output>\n",
|
| 43 |
+
" <script>// Copyright 2017 Google LLC\n",
|
| 44 |
+
"//\n",
|
| 45 |
+
"// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
| 46 |
+
"// you may not use this file except in compliance with the License.\n",
|
| 47 |
+
"// You may obtain a copy of the License at\n",
|
| 48 |
+
"//\n",
|
| 49 |
+
"// http://www.apache.org/licenses/LICENSE-2.0\n",
|
| 50 |
+
"//\n",
|
| 51 |
+
"// Unless required by applicable law or agreed to in writing, software\n",
|
| 52 |
+
"// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
| 53 |
+
"// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
| 54 |
+
"// See the License for the specific language governing permissions and\n",
|
| 55 |
+
"// limitations under the License.\n",
|
| 56 |
+
"\n",
|
| 57 |
+
"/**\n",
|
| 58 |
+
" * @fileoverview Helpers for google.colab Python module.\n",
|
| 59 |
+
" */\n",
|
| 60 |
+
"(function(scope) {\n",
|
| 61 |
+
"function span(text, styleAttributes = {}) {\n",
|
| 62 |
+
" const element = document.createElement('span');\n",
|
| 63 |
+
" element.textContent = text;\n",
|
| 64 |
+
" for (const key of Object.keys(styleAttributes)) {\n",
|
| 65 |
+
" element.style[key] = styleAttributes[key];\n",
|
| 66 |
+
" }\n",
|
| 67 |
+
" return element;\n",
|
| 68 |
+
"}\n",
|
| 69 |
+
"\n",
|
| 70 |
+
"// Max number of bytes which will be uploaded at a time.\n",
|
| 71 |
+
"const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
|
| 72 |
+
"\n",
|
| 73 |
+
"function _uploadFiles(inputId, outputId) {\n",
|
| 74 |
+
" const steps = uploadFilesStep(inputId, outputId);\n",
|
| 75 |
+
" const outputElement = document.getElementById(outputId);\n",
|
| 76 |
+
" // Cache steps on the outputElement to make it available for the next call\n",
|
| 77 |
+
" // to uploadFilesContinue from Python.\n",
|
| 78 |
+
" outputElement.steps = steps;\n",
|
| 79 |
+
"\n",
|
| 80 |
+
" return _uploadFilesContinue(outputId);\n",
|
| 81 |
+
"}\n",
|
| 82 |
+
"\n",
|
| 83 |
+
"// This is roughly an async generator (not supported in the browser yet),\n",
|
| 84 |
+
"// where there are multiple asynchronous steps and the Python side is going\n",
|
| 85 |
+
"// to poll for completion of each step.\n",
|
| 86 |
+
"// This uses a Promise to block the python side on completion of each step,\n",
|
| 87 |
+
"// then passes the result of the previous step as the input to the next step.\n",
|
| 88 |
+
"function _uploadFilesContinue(outputId) {\n",
|
| 89 |
+
" const outputElement = document.getElementById(outputId);\n",
|
| 90 |
+
" const steps = outputElement.steps;\n",
|
| 91 |
+
"\n",
|
| 92 |
+
" const next = steps.next(outputElement.lastPromiseValue);\n",
|
| 93 |
+
" return Promise.resolve(next.value.promise).then((value) => {\n",
|
| 94 |
+
" // Cache the last promise value to make it available to the next\n",
|
| 95 |
+
" // step of the generator.\n",
|
| 96 |
+
" outputElement.lastPromiseValue = value;\n",
|
| 97 |
+
" return next.value.response;\n",
|
| 98 |
+
" });\n",
|
| 99 |
+
"}\n",
|
| 100 |
+
"\n",
|
| 101 |
+
"/**\n",
|
| 102 |
+
" * Generator function which is called between each async step of the upload\n",
|
| 103 |
+
" * process.\n",
|
| 104 |
+
" * @param {string} inputId Element ID of the input file picker element.\n",
|
| 105 |
+
" * @param {string} outputId Element ID of the output display.\n",
|
| 106 |
+
" * @return {!Iterable<!Object>} Iterable of next steps.\n",
|
| 107 |
+
" */\n",
|
| 108 |
+
"function* uploadFilesStep(inputId, outputId) {\n",
|
| 109 |
+
" const inputElement = document.getElementById(inputId);\n",
|
| 110 |
+
" inputElement.disabled = false;\n",
|
| 111 |
+
"\n",
|
| 112 |
+
" const outputElement = document.getElementById(outputId);\n",
|
| 113 |
+
" outputElement.innerHTML = '';\n",
|
| 114 |
+
"\n",
|
| 115 |
+
" const pickedPromise = new Promise((resolve) => {\n",
|
| 116 |
+
" inputElement.addEventListener('change', (e) => {\n",
|
| 117 |
+
" resolve(e.target.files);\n",
|
| 118 |
+
" });\n",
|
| 119 |
+
" });\n",
|
| 120 |
+
"\n",
|
| 121 |
+
" const cancel = document.createElement('button');\n",
|
| 122 |
+
" inputElement.parentElement.appendChild(cancel);\n",
|
| 123 |
+
" cancel.textContent = 'Cancel upload';\n",
|
| 124 |
+
" const cancelPromise = new Promise((resolve) => {\n",
|
| 125 |
+
" cancel.onclick = () => {\n",
|
| 126 |
+
" resolve(null);\n",
|
| 127 |
+
" };\n",
|
| 128 |
+
" });\n",
|
| 129 |
+
"\n",
|
| 130 |
+
" // Wait for the user to pick the files.\n",
|
| 131 |
+
" const files = yield {\n",
|
| 132 |
+
" promise: Promise.race([pickedPromise, cancelPromise]),\n",
|
| 133 |
+
" response: {\n",
|
| 134 |
+
" action: 'starting',\n",
|
| 135 |
+
" }\n",
|
| 136 |
+
" };\n",
|
| 137 |
+
"\n",
|
| 138 |
+
" cancel.remove();\n",
|
| 139 |
+
"\n",
|
| 140 |
+
" // Disable the input element since further picks are not allowed.\n",
|
| 141 |
+
" inputElement.disabled = true;\n",
|
| 142 |
+
"\n",
|
| 143 |
+
" if (!files) {\n",
|
| 144 |
+
" return {\n",
|
| 145 |
+
" response: {\n",
|
| 146 |
+
" action: 'complete',\n",
|
| 147 |
+
" }\n",
|
| 148 |
+
" };\n",
|
| 149 |
+
" }\n",
|
| 150 |
+
"\n",
|
| 151 |
+
" for (const file of files) {\n",
|
| 152 |
+
" const li = document.createElement('li');\n",
|
| 153 |
+
" li.append(span(file.name, {fontWeight: 'bold'}));\n",
|
| 154 |
+
" li.append(span(\n",
|
| 155 |
+
" `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
|
| 156 |
+
" `last modified: ${\n",
|
| 157 |
+
" file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
|
| 158 |
+
" 'n/a'} - `));\n",
|
| 159 |
+
" const percent = span('0% done');\n",
|
| 160 |
+
" li.appendChild(percent);\n",
|
| 161 |
+
"\n",
|
| 162 |
+
" outputElement.appendChild(li);\n",
|
| 163 |
+
"\n",
|
| 164 |
+
" const fileDataPromise = new Promise((resolve) => {\n",
|
| 165 |
+
" const reader = new FileReader();\n",
|
| 166 |
+
" reader.onload = (e) => {\n",
|
| 167 |
+
" resolve(e.target.result);\n",
|
| 168 |
+
" };\n",
|
| 169 |
+
" reader.readAsArrayBuffer(file);\n",
|
| 170 |
+
" });\n",
|
| 171 |
+
" // Wait for the data to be ready.\n",
|
| 172 |
+
" let fileData = yield {\n",
|
| 173 |
+
" promise: fileDataPromise,\n",
|
| 174 |
+
" response: {\n",
|
| 175 |
+
" action: 'continue',\n",
|
| 176 |
+
" }\n",
|
| 177 |
+
" };\n",
|
| 178 |
+
"\n",
|
| 179 |
+
" // Use a chunked sending to avoid message size limits. See b/62115660.\n",
|
| 180 |
+
" let position = 0;\n",
|
| 181 |
+
" do {\n",
|
| 182 |
+
" const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
|
| 183 |
+
" const chunk = new Uint8Array(fileData, position, length);\n",
|
| 184 |
+
" position += length;\n",
|
| 185 |
+
"\n",
|
| 186 |
+
" const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
|
| 187 |
+
" yield {\n",
|
| 188 |
+
" response: {\n",
|
| 189 |
+
" action: 'append',\n",
|
| 190 |
+
" file: file.name,\n",
|
| 191 |
+
" data: base64,\n",
|
| 192 |
+
" },\n",
|
| 193 |
+
" };\n",
|
| 194 |
+
"\n",
|
| 195 |
+
" let percentDone = fileData.byteLength === 0 ?\n",
|
| 196 |
+
" 100 :\n",
|
| 197 |
+
" Math.round((position / fileData.byteLength) * 100);\n",
|
| 198 |
+
" percent.textContent = `${percentDone}% done`;\n",
|
| 199 |
+
"\n",
|
| 200 |
+
" } while (position < fileData.byteLength);\n",
|
| 201 |
+
" }\n",
|
| 202 |
+
"\n",
|
| 203 |
+
" // All done.\n",
|
| 204 |
+
" yield {\n",
|
| 205 |
+
" response: {\n",
|
| 206 |
+
" action: 'complete',\n",
|
| 207 |
+
" }\n",
|
| 208 |
+
" };\n",
|
| 209 |
+
"}\n",
|
| 210 |
+
"\n",
|
| 211 |
+
"scope.google = scope.google || {};\n",
|
| 212 |
+
"scope.google.colab = scope.google.colab || {};\n",
|
| 213 |
+
"scope.google.colab._files = {\n",
|
| 214 |
+
" _uploadFiles,\n",
|
| 215 |
+
" _uploadFilesContinue,\n",
|
| 216 |
+
"};\n",
|
| 217 |
+
"})(self);\n",
|
| 218 |
+
"</script> "
|
| 219 |
+
]
|
| 220 |
+
},
|
| 221 |
+
"metadata": {}
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"output_type": "stream",
|
| 225 |
+
"name": "stdout",
|
| 226 |
+
"text": [
|
| 227 |
+
"Saving words.txt to words (1).txt\n"
|
| 228 |
+
]
|
| 229 |
+
}
|
| 230 |
+
],
|
| 231 |
+
"source": [
|
| 232 |
+
"# Cell 1\n",
|
| 233 |
+
"from google.colab import files\n",
|
| 234 |
+
"uploaded = files.upload() # select words.txt from your PC\n"
|
| 235 |
+
]
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"cell_type": "code",
|
| 239 |
+
"source": [
|
| 240 |
+
"\"\"\"\n",
|
| 241 |
+
"=============================================================================\n",
|
| 242 |
+
" FULL PERMUTATION MISSPELLINGS GENERATOR (Google Colab Edition)\n",
|
| 243 |
+
"=============================================================================\n",
|
| 244 |
+
"\n",
|
| 245 |
+
"Purpose:\n",
|
| 246 |
+
" Generate ALL possible letter permutations of each word from words.txt\n",
|
| 247 |
+
" and write them as misspelling=correction pairs.\n",
|
| 248 |
+
"\n",
|
| 249 |
+
"β οΈ WARNING β READ BEFORE RUNNING β οΈ\n",
|
| 250 |
+
" This is computationally EXTREME. A single 10-letter word has 3,628,800\n",
|
| 251 |
+
" permutations. A 12-letter word has 479,001,600. For 466k words, the full\n",
|
| 252 |
+
" output could be PETABYTES. You WILL need to limit word length.\n",
|
| 253 |
+
"\n",
|
| 254 |
+
"=============================================================================\n",
|
| 255 |
+
" HOW TO USE ON GOOGLE COLAB\n",
|
| 256 |
+
"=============================================================================\n",
|
| 257 |
+
"\n",
|
| 258 |
+
"1. Open Google Colab β https://colab.research.google.com\n",
|
| 259 |
+
"2. Create a new notebook (Python 3)\n",
|
| 260 |
+
"\n",
|
| 261 |
+
"3. Upload your words.txt:\n",
|
| 262 |
+
" βββββββββββββββββββββββββββββββββββββ\n",
|
| 263 |
+
" # CELL 1: Upload words.txt\n",
|
| 264 |
+
" from google.colab import files\n",
|
| 265 |
+
" uploaded = files.upload() # click \"Choose Files\" β select words.txt\n",
|
| 266 |
+
" βββββββββββββββββββββββββββββββββββββ\n",
|
| 267 |
+
"\n",
|
| 268 |
+
"4. Copy-paste this ENTIRE script into a new cell and run it.\n",
|
| 269 |
+
"\n",
|
| 270 |
+
"5. Download the result:\n",
|
| 271 |
+
" βββββββββββββββββββββββββββββββββββββ\n",
|
| 272 |
+
" # CELL 3: Download the output\n",
|
| 273 |
+
" files.download('misspellings_permutations.txt')\n",
|
| 274 |
+
" βββββββββββββββββββββββββββββββββββββ\n",
|
| 275 |
+
"\n",
|
| 276 |
+
"=============================================================================\n",
|
| 277 |
+
" OR: Use Google Drive for large files\n",
|
| 278 |
+
"=============================================================================\n",
|
| 279 |
+
"\n",
|
| 280 |
+
" # Mount Google Drive (you get 15 GB free)\n",
|
| 281 |
+
" from google.colab import drive\n",
|
| 282 |
+
" drive.mount('/content/drive')\n",
|
| 283 |
+
"\n",
|
| 284 |
+
" # Then set OUTPUT_PATH below to:\n",
|
| 285 |
+
" OUTPUT_PATH = '/content/drive/MyDrive/misspellings_permutations.txt'\n",
|
| 286 |
+
"\n",
|
| 287 |
+
"=============================================================================\n",
|
| 288 |
+
" CONFIGURATION β Adjust these before running!\n",
|
| 289 |
+
"=============================================================================\n",
|
| 290 |
+
"\"\"\"\n",
|
| 291 |
+
"\n",
|
| 292 |
+
"import os\n",
|
| 293 |
+
"import sys\n",
|
| 294 |
+
"import time\n",
|
| 295 |
+
"import math\n",
|
| 296 |
+
"from itertools import permutations\n",
|
| 297 |
+
"\n",
|
| 298 |
+
"# ββ CONFIGURATION βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 299 |
+
"\n",
|
| 300 |
+
"WORDS_PATH = 'words.txt' # path to your words.txt\n",
|
| 301 |
+
"OUTPUT_PATH = 'misspellings_permutations.txt' # output file path\n",
|
| 302 |
+
"\n",
|
| 303 |
+
"MIN_WORD_LEN = 3 # skip words shorter than this\n",
|
| 304 |
+
"MAX_WORD_LEN = 7 # β οΈ CRITICAL: max word length to permute\n",
|
| 305 |
+
" # 7 β max 5,040 perms/word (manageable)\n",
|
| 306 |
+
" # 8 β max 40,320 perms/word (large)\n",
|
| 307 |
+
" # 9 β max 362,880 perms/word (very large)\n",
|
| 308 |
+
" # 10 β max 3,628,800 perms/word (EXTREME)\n",
|
| 309 |
+
" # Increase at your own risk!\n",
|
| 310 |
+
"\n",
|
| 311 |
+
"ONLY_ALPHA = True # only process pure-alphabetical words\n",
|
| 312 |
+
"BATCH_LOG = 5000 # print progress every N words\n",
|
| 313 |
+
"\n",
|
| 314 |
+
"# ββ ESTIMATION TABLE ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 315 |
+
"# Here's roughly how big the output gets at each MAX_WORD_LEN setting,\n",
|
| 316 |
+
"# assuming ~200k qualifying words at each length bracket:\n",
|
| 317 |
+
"#\n",
|
| 318 |
+
"# MAX_WORD_LEN β Perms per word (worst) β Rough output size\n",
|
| 319 |
+
"# ββββββββββββββΌβββββββββββββββββββββββββΌββββββββββββββββββ\n",
|
| 320 |
+
"# 5 β 120 β ~200 MB\n",
|
| 321 |
+
"# 6 β 720 β ~1-2 GB\n",
|
| 322 |
+
"# 7 β 5,040 β ~5-15 GB\n",
|
| 323 |
+
"# 8 β 40,320 β ~50-150 GB\n",
|
| 324 |
+
"# 9 β 362,880 β ~500 GB - 1 TB\n",
|
| 325 |
+
"# 10 β 3,628,800 β ~5-50 TB β won't fit anywhere\n",
|
| 326 |
+
"#\n",
|
| 327 |
+
"# Google Colab free tier gives you:\n",
|
| 328 |
+
"# β’ ~78 GB disk on the VM (temporary, lost on disconnect)\n",
|
| 329 |
+
"# β’ 15 GB Google Drive (persistent)\n",
|
| 330 |
+
"# β’ Colab Pro: 225 GB disk, longer runtimes\n",
|
| 331 |
+
"#\n",
|
| 332 |
+
"# RECOMMENDATION: Start with MAX_WORD_LEN = 6 or 7, see the size,\n",
|
| 333 |
+
"# then increase if you have space.\n",
|
| 334 |
+
"# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 335 |
+
"\n",
|
| 336 |
+
"\n",
|
| 337 |
+
"def estimate_output(words):\n",
|
| 338 |
+
" \"\"\"Estimate total permutations and file size before generating.\"\"\"\n",
|
| 339 |
+
" total_perms = 0\n",
|
| 340 |
+
" for w in words:\n",
|
| 341 |
+
" n = len(w)\n",
|
| 342 |
+
" # Account for duplicate letters: n! / (c1! * c2! * ...)\n",
|
| 343 |
+
" freq = {}\n",
|
| 344 |
+
" for ch in w.lower():\n",
|
| 345 |
+
" freq[ch] = freq.get(ch, 0) + 1\n",
|
| 346 |
+
" unique_perms = math.factorial(n)\n",
|
| 347 |
+
" for count in freq.values():\n",
|
| 348 |
+
" unique_perms //= math.factorial(count)\n",
|
| 349 |
+
" total_perms += unique_perms - 1 # subtract the original word\n",
|
| 350 |
+
"\n",
|
| 351 |
+
" # Estimate ~15 bytes per line (avg) β \"typo=word\\n\"\n",
|
| 352 |
+
" avg_bytes_per_line = 15\n",
|
| 353 |
+
" est_bytes = total_perms * avg_bytes_per_line\n",
|
| 354 |
+
" est_gb = est_bytes / (1024 ** 3)\n",
|
| 355 |
+
"\n",
|
| 356 |
+
" return total_perms, est_gb\n",
|
| 357 |
+
"\n",
|
| 358 |
+
"\n",
|
| 359 |
+
"def generate_unique_permutations(word):\n",
|
| 360 |
+
" \"\"\"\n",
|
| 361 |
+
" Generate all unique permutations of a word's letters,\n",
|
| 362 |
+
" excluding the original word itself.\n",
|
| 363 |
+
"\n",
|
| 364 |
+
" Uses set() to deduplicate (handles repeated letters efficiently).\n",
|
| 365 |
+
" \"\"\"\n",
|
| 366 |
+
" lower = word.lower()\n",
|
| 367 |
+
" perms = set(''.join(p) for p in permutations(lower))\n",
|
| 368 |
+
" perms.discard(lower) # remove the correctly-spelled word\n",
|
| 369 |
+
" return perms\n",
|
| 370 |
+
"\n",
|
| 371 |
+
"\n",
|
| 372 |
+
"def is_pure_alpha(word):\n",
|
| 373 |
+
" return word.isalpha()\n",
|
| 374 |
+
"\n",
|
| 375 |
+
"\n",
|
| 376 |
+
"def main():\n",
|
| 377 |
+
" if not os.path.exists(WORDS_PATH):\n",
|
| 378 |
+
" print(f\"ERROR: '{WORDS_PATH}' not found!\")\n",
|
| 379 |
+
" print(\"Make sure you uploaded words.txt or set WORDS_PATH correctly.\")\n",
|
| 380 |
+
" sys.exit(1)\n",
|
| 381 |
+
"\n",
|
| 382 |
+
" # ββ Read words ββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 383 |
+
" print(f\"Reading words from: {WORDS_PATH}\")\n",
|
| 384 |
+
" with open(WORDS_PATH, 'r', encoding='utf-8', errors='replace') as f:\n",
|
| 385 |
+
" raw_words = [line.strip() for line in f if line.strip()]\n",
|
| 386 |
+
"\n",
|
| 387 |
+
" print(f\"Total raw entries: {len(raw_words):,}\")\n",
|
| 388 |
+
"\n",
|
| 389 |
+
" # Filter\n",
|
| 390 |
+
" words = []\n",
|
| 391 |
+
" for w in raw_words:\n",
|
| 392 |
+
" if ONLY_ALPHA and not is_pure_alpha(w):\n",
|
| 393 |
+
" continue\n",
|
| 394 |
+
" if len(w) < MIN_WORD_LEN or len(w) > MAX_WORD_LEN:\n",
|
| 395 |
+
" continue\n",
|
| 396 |
+
" words.append(w)\n",
|
| 397 |
+
"\n",
|
| 398 |
+
" print(f\"Filtered to {len(words):,} words (alpha-only, len {MIN_WORD_LEN}-{MAX_WORD_LEN})\")\n",
|
| 399 |
+
"\n",
|
| 400 |
+
" if len(words) == 0:\n",
|
| 401 |
+
" print(\"No words matched the filter. Adjust MIN/MAX_WORD_LEN.\")\n",
|
| 402 |
+
" sys.exit(1)\n",
|
| 403 |
+
"\n",
|
| 404 |
+
" # ββ Estimate ββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 405 |
+
" print(\"\\nEstimating output size (this may take a moment)...\")\n",
|
| 406 |
+
" total_perms, est_gb = estimate_output(words)\n",
|
| 407 |
+
" print(f\" Estimated permutations : {total_perms:,}\")\n",
|
| 408 |
+
" print(f\" Estimated file size : {est_gb:.2f} GB\")\n",
|
| 409 |
+
"\n",
|
| 410 |
+
" # Safety check\n",
|
| 411 |
+
" if est_gb > 70:\n",
|
| 412 |
+
" print(f\"\\nβ οΈ WARNING: Estimated output ({est_gb:.1f} GB) exceeds Colab disk (~78 GB).\")\n",
|
| 413 |
+
" print(\" Reduce MAX_WORD_LEN or the script will crash when disk fills up.\")\n",
|
| 414 |
+
" print(\" Aborting. Set MAX_WORD_LEN lower and re-run.\")\n",
|
| 415 |
+
" sys.exit(1)\n",
|
| 416 |
+
"\n",
|
| 417 |
+
" print(f\"\\nProceeding with generation β {OUTPUT_PATH}\")\n",
|
| 418 |
+
" print(\"=\" * 60)\n",
|
| 419 |
+
"\n",
|
| 420 |
+
" # ββ Generate ββββββββββββββββββββββββββββββββββββββββββββββββ\n",
|
| 421 |
+
" start = time.time()\n",
|
| 422 |
+
" total_written = 0\n",
|
| 423 |
+
"\n",
|
| 424 |
+
" with open(OUTPUT_PATH, 'w', encoding='utf-8') as out:\n",
|
| 425 |
+
" out.write(\"# Auto-generated FULL PERMUTATION misspellings\\n\")\n",
|
| 426 |
+
" out.write(f\"# Config: word length {MIN_WORD_LEN}-{MAX_WORD_LEN}\\n\")\n",
|
| 427 |
+
" out.write(\"# Format: misspelling=correction\\n\\n\")\n",
|
| 428 |
+
"\n",
|
| 429 |
+
" for idx, word in enumerate(words):\n",
|
| 430 |
+
" perms = generate_unique_permutations(word)\n",
|
| 431 |
+
"\n",
|
| 432 |
+
" for typo in sorted(perms):\n",
|
| 433 |
+
" out.write(f\"{typo}={word}\\n\")\n",
|
| 434 |
+
" total_written += 1\n",
|
| 435 |
+
"\n",
|
| 436 |
+
" # Progress\n",
|
| 437 |
+
" if (idx + 1) % BATCH_LOG == 0:\n",
|
| 438 |
+
" elapsed = time.time() - start\n",
|
| 439 |
+
" pct = (idx + 1) / len(words) * 100\n",
|
| 440 |
+
" rate = (idx + 1) / elapsed if elapsed > 0 else 0\n",
|
| 441 |
+
" cur_size = os.path.getsize(OUTPUT_PATH) / (1024 ** 3)\n",
|
| 442 |
+
" print(f\" [{pct:5.1f}%] {idx+1:>7,}/{len(words):,} words |\"\n",
|
| 443 |
+
" f\" {total_written:>12,} lines | {cur_size:.2f} GB |\"\n",
|
| 444 |
+
" f\" {rate:.0f} words/sec\")\n",
|
| 445 |
+
"\n",
|
| 446 |
+
" elapsed = time.time() - start\n",
|
| 447 |
+
" final_size = os.path.getsize(OUTPUT_PATH) / (1024 ** 3)\n",
|
| 448 |
+
"\n",
|
| 449 |
+
" print()\n",
|
| 450 |
+
" print(\"=\" * 60)\n",
|
| 451 |
+
" print(f\" β
DONE in {elapsed:.1f}s ({elapsed/60:.1f} min)\")\n",
|
| 452 |
+
" print(f\" Words processed : {len(words):,}\")\n",
|
| 453 |
+
" print(f\" Lines written : {total_written:,}\")\n",
|
| 454 |
+
" print(f\" Output file : {OUTPUT_PATH}\")\n",
|
| 455 |
+
" print(f\" File size : {final_size:.2f} GB\")\n",
|
| 456 |
+
" print(\"=\" * 60)\n",
|
| 457 |
+
"\n",
|
| 458 |
+
"\n",
|
| 459 |
+
"if __name__ == '__main__':\n",
|
| 460 |
+
" main()\n"
|
| 461 |
+
],
|
| 462 |
+
"metadata": {
|
| 463 |
+
"colab": {
|
| 464 |
+
"base_uri": "https://localhost:8080/"
|
| 465 |
+
},
|
| 466 |
+
"id": "Et0QfIxpJz_5",
|
| 467 |
+
"outputId": "e7e72965-f709-45c0-ae56-abf76b89d714"
|
| 468 |
+
},
|
| 469 |
+
"execution_count": null,
|
| 470 |
+
"outputs": [
|
| 471 |
+
{
|
| 472 |
+
"output_type": "stream",
|
| 473 |
+
"name": "stdout",
|
| 474 |
+
"text": [
|
| 475 |
+
"Reading words from: words.txt\n",
|
| 476 |
+
"Total raw entries: 466,550\n",
|
| 477 |
+
"Filtered to 125,414 words (alpha-only, len 3-7)\n",
|
| 478 |
+
"\n",
|
| 479 |
+
"Estimating output size (this may take a moment)...\n",
|
| 480 |
+
" Estimated permutations : 173,110,626\n",
|
| 481 |
+
" Estimated file size : 2.42 GB\n",
|
| 482 |
+
"\n",
|
| 483 |
+
"Proceeding with generation β misspellings_permutations.txt\n",
|
| 484 |
+
"============================================================\n",
|
| 485 |
+
" [ 4.0%] 5,000/125,414 words | 5,810,553 lines | 0.08 GB | 898 words/sec\n",
|
| 486 |
+
" [ 8.0%] 10,000/125,414 words | 11,972,245 lines | 0.18 GB | 781 words/sec\n",
|
| 487 |
+
" [ 12.0%] 15,000/125,414 words | 19,094,747 lines | 0.28 GB | 775 words/sec\n",
|
| 488 |
+
" [ 15.9%] 20,000/125,414 words | 26,800,249 lines | 0.39 GB | 721 words/sec\n",
|
| 489 |
+
" [ 19.9%] 25,000/125,414 words | 35,047,153 lines | 0.51 GB | 690 words/sec\n",
|
| 490 |
+
" [ 23.9%] 30,000/125,414 words | 42,273,166 lines | 0.62 GB | 695 words/sec\n",
|
| 491 |
+
" [ 27.9%] 35,000/125,414 words | 48,702,338 lines | 0.71 GB | 692 words/sec\n",
|
| 492 |
+
" [ 31.9%] 40,000/125,414 words | 55,295,151 lines | 0.81 GB | 703 words/sec\n",
|
| 493 |
+
" [ 35.9%] 45,000/125,414 words | 62,710,327 lines | 0.92 GB | 690 words/sec\n",
|
| 494 |
+
" [ 39.9%] 50,000/125,414 words | 69,722,485 lines | 1.02 GB | 690 words/sec\n",
|
| 495 |
+
" [ 43.9%] 55,000/125,414 words | 76,146,526 lines | 1.12 GB | 674 words/sec\n",
|
| 496 |
+
" [ 47.8%] 60,000/125,414 words | 81,994,038 lines | 1.20 GB | 686 words/sec\n",
|
| 497 |
+
" [ 51.8%] 65,000/125,414 words | 88,058,594 lines | 1.29 GB | 683 words/sec\n",
|
| 498 |
+
" [ 55.8%] 70,000/125,414 words | 94,651,291 lines | 1.39 GB | 688 words/sec\n",
|
| 499 |
+
" [ 59.8%] 75,000/125,414 words | 101,636,647 lines | 1.49 GB | 679 words/sec\n",
|
| 500 |
+
" [ 63.8%] 80,000/125,414 words | 107,086,424 lines | 1.57 GB | 691 words/sec\n",
|
| 501 |
+
" [ 67.8%] 85,000/125,414 words | 114,898,717 lines | 1.68 GB | 678 words/sec\n",
|
| 502 |
+
" [ 71.8%] 90,000/125,414 words | 123,278,791 lines | 1.80 GB | 675 words/sec\n",
|
| 503 |
+
" [ 75.7%] 95,000/125,414 words | 129,821,900 lines | 1.90 GB | 669 words/sec\n",
|
| 504 |
+
" [ 79.7%] 100,000/125,414 words | 136,429,269 lines | 2.00 GB | 673 words/sec\n",
|
| 505 |
+
" [ 83.7%] 105,000/125,414 words | 143,342,171 lines | 2.10 GB | 667 words/sec\n",
|
| 506 |
+
" [ 87.7%] 110,000/125,414 words | 150,701,210 lines | 2.21 GB | 666 words/sec\n",
|
| 507 |
+
" [ 91.7%] 115,000/125,414 words | 157,479,616 lines | 2.31 GB | 665 words/sec\n",
|
| 508 |
+
" [ 95.7%] 120,000/125,414 words | 165,619,673 lines | 2.43 GB | 662 words/sec\n",
|
| 509 |
+
" [ 99.7%] 125,000/125,414 words | 172,558,768 lines | 2.53 GB | 661 words/sec\n",
|
| 510 |
+
"\n",
|
| 511 |
+
"============================================================\n",
|
| 512 |
+
" β
DONE in 189.5s (3.2 min)\n",
|
| 513 |
+
" Words processed : 125,414\n",
|
| 514 |
+
" Lines written : 173,110,626\n",
|
| 515 |
+
" Output file : misspellings_permutations.txt\n",
|
| 516 |
+
" File size : 2.53 GB\n",
|
| 517 |
+
"============================================================\n"
|
| 518 |
+
]
|
| 519 |
+
}
|
| 520 |
+
]
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"cell_type": "code",
|
| 524 |
+
"source": [
|
| 525 |
+
"# If saved to VM disk:\n",
|
| 526 |
+
"files.download('misspellings_permutations.txt')\n",
|
| 527 |
+
"\n",
|
| 528 |
+
"# If saved to Google Drive: just access it from drive.google.com\n",
|
| 529 |
+
"\n"
|
| 530 |
+
],
|
| 531 |
+
"metadata": {
|
| 532 |
+
"id": "y9jWxvv8LWoH",
|
| 533 |
+
"outputId": "d8d754d3-234e-4020-bcc7-a19f3fc5fb26",
|
| 534 |
+
"colab": {
|
| 535 |
+
"base_uri": "https://localhost:8080/",
|
| 536 |
+
"height": 34
|
| 537 |
+
}
|
| 538 |
+
},
|
| 539 |
+
"execution_count": null,
|
| 540 |
+
"outputs": [
|
| 541 |
+
{
|
| 542 |
+
"output_type": "display_data",
|
| 543 |
+
"data": {
|
| 544 |
+
"text/plain": [
|
| 545 |
+
"<IPython.core.display.Javascript object>"
|
| 546 |
+
],
|
| 547 |
+
"application/javascript": [
|
| 548 |
+
"\n",
|
| 549 |
+
" async function download(id, filename, size) {\n",
|
| 550 |
+
" if (!google.colab.kernel.accessAllowed) {\n",
|
| 551 |
+
" return;\n",
|
| 552 |
+
" }\n",
|
| 553 |
+
" const div = document.createElement('div');\n",
|
| 554 |
+
" const label = document.createElement('label');\n",
|
| 555 |
+
" label.textContent = `Downloading \"${filename}\": `;\n",
|
| 556 |
+
" div.appendChild(label);\n",
|
| 557 |
+
" const progress = document.createElement('progress');\n",
|
| 558 |
+
" progress.max = size;\n",
|
| 559 |
+
" div.appendChild(progress);\n",
|
| 560 |
+
" document.body.appendChild(div);\n",
|
| 561 |
+
"\n",
|
| 562 |
+
" const buffers = [];\n",
|
| 563 |
+
" let downloaded = 0;\n",
|
| 564 |
+
"\n",
|
| 565 |
+
" const channel = await google.colab.kernel.comms.open(id);\n",
|
| 566 |
+
" // Send a message to notify the kernel that we're ready.\n",
|
| 567 |
+
" channel.send({})\n",
|
| 568 |
+
"\n",
|
| 569 |
+
" for await (const message of channel.messages) {\n",
|
| 570 |
+
" // Send a message to notify the kernel that we're ready.\n",
|
| 571 |
+
" channel.send({})\n",
|
| 572 |
+
" if (message.buffers) {\n",
|
| 573 |
+
" for (const buffer of message.buffers) {\n",
|
| 574 |
+
" buffers.push(buffer);\n",
|
| 575 |
+
" downloaded += buffer.byteLength;\n",
|
| 576 |
+
" progress.value = downloaded;\n",
|
| 577 |
+
" }\n",
|
| 578 |
+
" }\n",
|
| 579 |
+
" }\n",
|
| 580 |
+
" const blob = new Blob(buffers, {type: 'application/binary'});\n",
|
| 581 |
+
" const a = document.createElement('a');\n",
|
| 582 |
+
" a.href = window.URL.createObjectURL(blob);\n",
|
| 583 |
+
" a.download = filename;\n",
|
| 584 |
+
" div.appendChild(a);\n",
|
| 585 |
+
" a.click();\n",
|
| 586 |
+
" div.remove();\n",
|
| 587 |
+
" }\n",
|
| 588 |
+
" "
|
| 589 |
+
]
|
| 590 |
+
},
|
| 591 |
+
"metadata": {}
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"output_type": "display_data",
|
| 595 |
+
"data": {
|
| 596 |
+
"text/plain": [
|
| 597 |
+
"<IPython.core.display.Javascript object>"
|
| 598 |
+
],
|
| 599 |
+
"application/javascript": [
|
| 600 |
+
"download(\"download_10941777-78c6-4833-b8e6-093feee02e11\", \"misspellings_permutations.txt\", 2721877361)"
|
| 601 |
+
]
|
| 602 |
+
},
|
| 603 |
+
"metadata": {}
|
| 604 |
+
}
|
| 605 |
+
]
|
| 606 |
+
}
|
| 607 |
+
]
|
| 608 |
+
}
|