algorembrant commited on
Commit
e518c34
Β·
verified Β·
1 Parent(s): eb7534e

Upload google_collab.ipynb

Browse files
Files changed (1) hide show
  1. google_collab.ipynb +608 -0
google_collab.ipynb ADDED
@@ -0,0 +1,608 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "nbformat": 4,
3
+ "nbformat_minor": 0,
4
+ "metadata": {
5
+ "colab": {
6
+ "provenance": []
7
+ },
8
+ "kernelspec": {
9
+ "name": "python3",
10
+ "display_name": "Python 3"
11
+ },
12
+ "language_info": {
13
+ "name": "python"
14
+ }
15
+ },
16
+ "cells": [
17
+ {
18
+ "cell_type": "code",
19
+ "execution_count": null,
20
+ "metadata": {
21
+ "colab": {
22
+ "base_uri": "https://localhost:8080/",
23
+ "height": 73
24
+ },
25
+ "id": "FaSiqVnTItLq",
26
+ "outputId": "1ef8a78c-7421-41eb-8cf4-db8426edeed9"
27
+ },
28
+ "outputs": [
29
+ {
30
+ "output_type": "display_data",
31
+ "data": {
32
+ "text/plain": [
33
+ "<IPython.core.display.HTML object>"
34
+ ],
35
+ "text/html": [
36
+ "\n",
37
+ " <input type=\"file\" id=\"files-a9b68a68-38ec-4a0a-8037-171b8cfec796\" name=\"files[]\" multiple disabled\n",
38
+ " style=\"border:none\" />\n",
39
+ " <output id=\"result-a9b68a68-38ec-4a0a-8037-171b8cfec796\">\n",
40
+ " Upload widget is only available when the cell has been executed in the\n",
41
+ " current browser session. Please rerun this cell to enable.\n",
42
+ " </output>\n",
43
+ " <script>// Copyright 2017 Google LLC\n",
44
+ "//\n",
45
+ "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
46
+ "// you may not use this file except in compliance with the License.\n",
47
+ "// You may obtain a copy of the License at\n",
48
+ "//\n",
49
+ "// http://www.apache.org/licenses/LICENSE-2.0\n",
50
+ "//\n",
51
+ "// Unless required by applicable law or agreed to in writing, software\n",
52
+ "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
53
+ "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
54
+ "// See the License for the specific language governing permissions and\n",
55
+ "// limitations under the License.\n",
56
+ "\n",
57
+ "/**\n",
58
+ " * @fileoverview Helpers for google.colab Python module.\n",
59
+ " */\n",
60
+ "(function(scope) {\n",
61
+ "function span(text, styleAttributes = {}) {\n",
62
+ " const element = document.createElement('span');\n",
63
+ " element.textContent = text;\n",
64
+ " for (const key of Object.keys(styleAttributes)) {\n",
65
+ " element.style[key] = styleAttributes[key];\n",
66
+ " }\n",
67
+ " return element;\n",
68
+ "}\n",
69
+ "\n",
70
+ "// Max number of bytes which will be uploaded at a time.\n",
71
+ "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
72
+ "\n",
73
+ "function _uploadFiles(inputId, outputId) {\n",
74
+ " const steps = uploadFilesStep(inputId, outputId);\n",
75
+ " const outputElement = document.getElementById(outputId);\n",
76
+ " // Cache steps on the outputElement to make it available for the next call\n",
77
+ " // to uploadFilesContinue from Python.\n",
78
+ " outputElement.steps = steps;\n",
79
+ "\n",
80
+ " return _uploadFilesContinue(outputId);\n",
81
+ "}\n",
82
+ "\n",
83
+ "// This is roughly an async generator (not supported in the browser yet),\n",
84
+ "// where there are multiple asynchronous steps and the Python side is going\n",
85
+ "// to poll for completion of each step.\n",
86
+ "// This uses a Promise to block the python side on completion of each step,\n",
87
+ "// then passes the result of the previous step as the input to the next step.\n",
88
+ "function _uploadFilesContinue(outputId) {\n",
89
+ " const outputElement = document.getElementById(outputId);\n",
90
+ " const steps = outputElement.steps;\n",
91
+ "\n",
92
+ " const next = steps.next(outputElement.lastPromiseValue);\n",
93
+ " return Promise.resolve(next.value.promise).then((value) => {\n",
94
+ " // Cache the last promise value to make it available to the next\n",
95
+ " // step of the generator.\n",
96
+ " outputElement.lastPromiseValue = value;\n",
97
+ " return next.value.response;\n",
98
+ " });\n",
99
+ "}\n",
100
+ "\n",
101
+ "/**\n",
102
+ " * Generator function which is called between each async step of the upload\n",
103
+ " * process.\n",
104
+ " * @param {string} inputId Element ID of the input file picker element.\n",
105
+ " * @param {string} outputId Element ID of the output display.\n",
106
+ " * @return {!Iterable<!Object>} Iterable of next steps.\n",
107
+ " */\n",
108
+ "function* uploadFilesStep(inputId, outputId) {\n",
109
+ " const inputElement = document.getElementById(inputId);\n",
110
+ " inputElement.disabled = false;\n",
111
+ "\n",
112
+ " const outputElement = document.getElementById(outputId);\n",
113
+ " outputElement.innerHTML = '';\n",
114
+ "\n",
115
+ " const pickedPromise = new Promise((resolve) => {\n",
116
+ " inputElement.addEventListener('change', (e) => {\n",
117
+ " resolve(e.target.files);\n",
118
+ " });\n",
119
+ " });\n",
120
+ "\n",
121
+ " const cancel = document.createElement('button');\n",
122
+ " inputElement.parentElement.appendChild(cancel);\n",
123
+ " cancel.textContent = 'Cancel upload';\n",
124
+ " const cancelPromise = new Promise((resolve) => {\n",
125
+ " cancel.onclick = () => {\n",
126
+ " resolve(null);\n",
127
+ " };\n",
128
+ " });\n",
129
+ "\n",
130
+ " // Wait for the user to pick the files.\n",
131
+ " const files = yield {\n",
132
+ " promise: Promise.race([pickedPromise, cancelPromise]),\n",
133
+ " response: {\n",
134
+ " action: 'starting',\n",
135
+ " }\n",
136
+ " };\n",
137
+ "\n",
138
+ " cancel.remove();\n",
139
+ "\n",
140
+ " // Disable the input element since further picks are not allowed.\n",
141
+ " inputElement.disabled = true;\n",
142
+ "\n",
143
+ " if (!files) {\n",
144
+ " return {\n",
145
+ " response: {\n",
146
+ " action: 'complete',\n",
147
+ " }\n",
148
+ " };\n",
149
+ " }\n",
150
+ "\n",
151
+ " for (const file of files) {\n",
152
+ " const li = document.createElement('li');\n",
153
+ " li.append(span(file.name, {fontWeight: 'bold'}));\n",
154
+ " li.append(span(\n",
155
+ " `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
156
+ " `last modified: ${\n",
157
+ " file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
158
+ " 'n/a'} - `));\n",
159
+ " const percent = span('0% done');\n",
160
+ " li.appendChild(percent);\n",
161
+ "\n",
162
+ " outputElement.appendChild(li);\n",
163
+ "\n",
164
+ " const fileDataPromise = new Promise((resolve) => {\n",
165
+ " const reader = new FileReader();\n",
166
+ " reader.onload = (e) => {\n",
167
+ " resolve(e.target.result);\n",
168
+ " };\n",
169
+ " reader.readAsArrayBuffer(file);\n",
170
+ " });\n",
171
+ " // Wait for the data to be ready.\n",
172
+ " let fileData = yield {\n",
173
+ " promise: fileDataPromise,\n",
174
+ " response: {\n",
175
+ " action: 'continue',\n",
176
+ " }\n",
177
+ " };\n",
178
+ "\n",
179
+ " // Use a chunked sending to avoid message size limits. See b/62115660.\n",
180
+ " let position = 0;\n",
181
+ " do {\n",
182
+ " const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
183
+ " const chunk = new Uint8Array(fileData, position, length);\n",
184
+ " position += length;\n",
185
+ "\n",
186
+ " const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
187
+ " yield {\n",
188
+ " response: {\n",
189
+ " action: 'append',\n",
190
+ " file: file.name,\n",
191
+ " data: base64,\n",
192
+ " },\n",
193
+ " };\n",
194
+ "\n",
195
+ " let percentDone = fileData.byteLength === 0 ?\n",
196
+ " 100 :\n",
197
+ " Math.round((position / fileData.byteLength) * 100);\n",
198
+ " percent.textContent = `${percentDone}% done`;\n",
199
+ "\n",
200
+ " } while (position < fileData.byteLength);\n",
201
+ " }\n",
202
+ "\n",
203
+ " // All done.\n",
204
+ " yield {\n",
205
+ " response: {\n",
206
+ " action: 'complete',\n",
207
+ " }\n",
208
+ " };\n",
209
+ "}\n",
210
+ "\n",
211
+ "scope.google = scope.google || {};\n",
212
+ "scope.google.colab = scope.google.colab || {};\n",
213
+ "scope.google.colab._files = {\n",
214
+ " _uploadFiles,\n",
215
+ " _uploadFilesContinue,\n",
216
+ "};\n",
217
+ "})(self);\n",
218
+ "</script> "
219
+ ]
220
+ },
221
+ "metadata": {}
222
+ },
223
+ {
224
+ "output_type": "stream",
225
+ "name": "stdout",
226
+ "text": [
227
+ "Saving words.txt to words (1).txt\n"
228
+ ]
229
+ }
230
+ ],
231
+ "source": [
232
+ "# Cell 1\n",
233
+ "from google.colab import files\n",
234
+ "uploaded = files.upload() # select words.txt from your PC\n"
235
+ ]
236
+ },
237
+ {
238
+ "cell_type": "code",
239
+ "source": [
240
+ "\"\"\"\n",
241
+ "=============================================================================\n",
242
+ " FULL PERMUTATION MISSPELLINGS GENERATOR (Google Colab Edition)\n",
243
+ "=============================================================================\n",
244
+ "\n",
245
+ "Purpose:\n",
246
+ " Generate ALL possible letter permutations of each word from words.txt\n",
247
+ " and write them as misspelling=correction pairs.\n",
248
+ "\n",
249
+ "⚠️ WARNING β€” READ BEFORE RUNNING ⚠️\n",
250
+ " This is computationally EXTREME. A single 10-letter word has 3,628,800\n",
251
+ " permutations. A 12-letter word has 479,001,600. For 466k words, the full\n",
252
+ " output could be PETABYTES. You WILL need to limit word length.\n",
253
+ "\n",
254
+ "=============================================================================\n",
255
+ " HOW TO USE ON GOOGLE COLAB\n",
256
+ "=============================================================================\n",
257
+ "\n",
258
+ "1. Open Google Colab β†’ https://colab.research.google.com\n",
259
+ "2. Create a new notebook (Python 3)\n",
260
+ "\n",
261
+ "3. Upload your words.txt:\n",
262
+ " ─────────────────────────────────────\n",
263
+ " # CELL 1: Upload words.txt\n",
264
+ " from google.colab import files\n",
265
+ " uploaded = files.upload() # click \"Choose Files\" β†’ select words.txt\n",
266
+ " ─────────────────────────────────────\n",
267
+ "\n",
268
+ "4. Copy-paste this ENTIRE script into a new cell and run it.\n",
269
+ "\n",
270
+ "5. Download the result:\n",
271
+ " ─────────────────────────────────────\n",
272
+ " # CELL 3: Download the output\n",
273
+ " files.download('misspellings_permutations.txt')\n",
274
+ " ─────────────────────────────────────\n",
275
+ "\n",
276
+ "=============================================================================\n",
277
+ " OR: Use Google Drive for large files\n",
278
+ "=============================================================================\n",
279
+ "\n",
280
+ " # Mount Google Drive (you get 15 GB free)\n",
281
+ " from google.colab import drive\n",
282
+ " drive.mount('/content/drive')\n",
283
+ "\n",
284
+ " # Then set OUTPUT_PATH below to:\n",
285
+ " OUTPUT_PATH = '/content/drive/MyDrive/misspellings_permutations.txt'\n",
286
+ "\n",
287
+ "=============================================================================\n",
288
+ " CONFIGURATION β€” Adjust these before running!\n",
289
+ "=============================================================================\n",
290
+ "\"\"\"\n",
291
+ "\n",
292
+ "import os\n",
293
+ "import sys\n",
294
+ "import time\n",
295
+ "import math\n",
296
+ "from itertools import permutations\n",
297
+ "\n",
298
+ "# ── CONFIGURATION ───────────────────────────────────────────────────────────\n",
299
+ "\n",
300
+ "WORDS_PATH = 'words.txt' # path to your words.txt\n",
301
+ "OUTPUT_PATH = 'misspellings_permutations.txt' # output file path\n",
302
+ "\n",
303
+ "MIN_WORD_LEN = 3 # skip words shorter than this\n",
304
+ "MAX_WORD_LEN = 7 # ⚠️ CRITICAL: max word length to permute\n",
305
+ " # 7 β†’ max 5,040 perms/word (manageable)\n",
306
+ " # 8 β†’ max 40,320 perms/word (large)\n",
307
+ " # 9 β†’ max 362,880 perms/word (very large)\n",
308
+ " # 10 β†’ max 3,628,800 perms/word (EXTREME)\n",
309
+ " # Increase at your own risk!\n",
310
+ "\n",
311
+ "ONLY_ALPHA = True # only process pure-alphabetical words\n",
312
+ "BATCH_LOG = 5000 # print progress every N words\n",
313
+ "\n",
314
+ "# ── ESTIMATION TABLE ────────────────────────────────────────────────────────\n",
315
+ "# Here's roughly how big the output gets at each MAX_WORD_LEN setting,\n",
316
+ "# assuming ~200k qualifying words at each length bracket:\n",
317
+ "#\n",
318
+ "# MAX_WORD_LEN β”‚ Perms per word (worst) β”‚ Rough output size\n",
319
+ "# ─────────────┼────────────────────────┼──────────────────\n",
320
+ "# 5 β”‚ 120 β”‚ ~200 MB\n",
321
+ "# 6 β”‚ 720 β”‚ ~1-2 GB\n",
322
+ "# 7 β”‚ 5,040 β”‚ ~5-15 GB\n",
323
+ "# 8 β”‚ 40,320 β”‚ ~50-150 GB\n",
324
+ "# 9 β”‚ 362,880 β”‚ ~500 GB - 1 TB\n",
325
+ "# 10 β”‚ 3,628,800 β”‚ ~5-50 TB ← won't fit anywhere\n",
326
+ "#\n",
327
+ "# Google Colab free tier gives you:\n",
328
+ "# β€’ ~78 GB disk on the VM (temporary, lost on disconnect)\n",
329
+ "# β€’ 15 GB Google Drive (persistent)\n",
330
+ "# β€’ Colab Pro: 225 GB disk, longer runtimes\n",
331
+ "#\n",
332
+ "# RECOMMENDATION: Start with MAX_WORD_LEN = 6 or 7, see the size,\n",
333
+ "# then increase if you have space.\n",
334
+ "# ────────────────────────────────────────────────────────────────────────────\n",
335
+ "\n",
336
+ "\n",
337
+ "def estimate_output(words):\n",
338
+ " \"\"\"Estimate total permutations and file size before generating.\"\"\"\n",
339
+ " total_perms = 0\n",
340
+ " for w in words:\n",
341
+ " n = len(w)\n",
342
+ " # Account for duplicate letters: n! / (c1! * c2! * ...)\n",
343
+ " freq = {}\n",
344
+ " for ch in w.lower():\n",
345
+ " freq[ch] = freq.get(ch, 0) + 1\n",
346
+ " unique_perms = math.factorial(n)\n",
347
+ " for count in freq.values():\n",
348
+ " unique_perms //= math.factorial(count)\n",
349
+ " total_perms += unique_perms - 1 # subtract the original word\n",
350
+ "\n",
351
+ " # Estimate ~15 bytes per line (avg) β†’ \"typo=word\\n\"\n",
352
+ " avg_bytes_per_line = 15\n",
353
+ " est_bytes = total_perms * avg_bytes_per_line\n",
354
+ " est_gb = est_bytes / (1024 ** 3)\n",
355
+ "\n",
356
+ " return total_perms, est_gb\n",
357
+ "\n",
358
+ "\n",
359
+ "def generate_unique_permutations(word):\n",
360
+ " \"\"\"\n",
361
+ " Generate all unique permutations of a word's letters,\n",
362
+ " excluding the original word itself.\n",
363
+ "\n",
364
+ " Uses set() to deduplicate (handles repeated letters efficiently).\n",
365
+ " \"\"\"\n",
366
+ " lower = word.lower()\n",
367
+ " perms = set(''.join(p) for p in permutations(lower))\n",
368
+ " perms.discard(lower) # remove the correctly-spelled word\n",
369
+ " return perms\n",
370
+ "\n",
371
+ "\n",
372
+ "def is_pure_alpha(word):\n",
373
+ " return word.isalpha()\n",
374
+ "\n",
375
+ "\n",
376
+ "def main():\n",
377
+ " if not os.path.exists(WORDS_PATH):\n",
378
+ " print(f\"ERROR: '{WORDS_PATH}' not found!\")\n",
379
+ " print(\"Make sure you uploaded words.txt or set WORDS_PATH correctly.\")\n",
380
+ " sys.exit(1)\n",
381
+ "\n",
382
+ " # ── Read words ──────────────────────────────────────────────\n",
383
+ " print(f\"Reading words from: {WORDS_PATH}\")\n",
384
+ " with open(WORDS_PATH, 'r', encoding='utf-8', errors='replace') as f:\n",
385
+ " raw_words = [line.strip() for line in f if line.strip()]\n",
386
+ "\n",
387
+ " print(f\"Total raw entries: {len(raw_words):,}\")\n",
388
+ "\n",
389
+ " # Filter\n",
390
+ " words = []\n",
391
+ " for w in raw_words:\n",
392
+ " if ONLY_ALPHA and not is_pure_alpha(w):\n",
393
+ " continue\n",
394
+ " if len(w) < MIN_WORD_LEN or len(w) > MAX_WORD_LEN:\n",
395
+ " continue\n",
396
+ " words.append(w)\n",
397
+ "\n",
398
+ " print(f\"Filtered to {len(words):,} words (alpha-only, len {MIN_WORD_LEN}-{MAX_WORD_LEN})\")\n",
399
+ "\n",
400
+ " if len(words) == 0:\n",
401
+ " print(\"No words matched the filter. Adjust MIN/MAX_WORD_LEN.\")\n",
402
+ " sys.exit(1)\n",
403
+ "\n",
404
+ " # ── Estimate ────────────────────────────────────────────────\n",
405
+ " print(\"\\nEstimating output size (this may take a moment)...\")\n",
406
+ " total_perms, est_gb = estimate_output(words)\n",
407
+ " print(f\" Estimated permutations : {total_perms:,}\")\n",
408
+ " print(f\" Estimated file size : {est_gb:.2f} GB\")\n",
409
+ "\n",
410
+ " # Safety check\n",
411
+ " if est_gb > 70:\n",
412
+ " print(f\"\\n⚠️ WARNING: Estimated output ({est_gb:.1f} GB) exceeds Colab disk (~78 GB).\")\n",
413
+ " print(\" Reduce MAX_WORD_LEN or the script will crash when disk fills up.\")\n",
414
+ " print(\" Aborting. Set MAX_WORD_LEN lower and re-run.\")\n",
415
+ " sys.exit(1)\n",
416
+ "\n",
417
+ " print(f\"\\nProceeding with generation β†’ {OUTPUT_PATH}\")\n",
418
+ " print(\"=\" * 60)\n",
419
+ "\n",
420
+ " # ── Generate ────────────────────────────────────────────────\n",
421
+ " start = time.time()\n",
422
+ " total_written = 0\n",
423
+ "\n",
424
+ " with open(OUTPUT_PATH, 'w', encoding='utf-8') as out:\n",
425
+ " out.write(\"# Auto-generated FULL PERMUTATION misspellings\\n\")\n",
426
+ " out.write(f\"# Config: word length {MIN_WORD_LEN}-{MAX_WORD_LEN}\\n\")\n",
427
+ " out.write(\"# Format: misspelling=correction\\n\\n\")\n",
428
+ "\n",
429
+ " for idx, word in enumerate(words):\n",
430
+ " perms = generate_unique_permutations(word)\n",
431
+ "\n",
432
+ " for typo in sorted(perms):\n",
433
+ " out.write(f\"{typo}={word}\\n\")\n",
434
+ " total_written += 1\n",
435
+ "\n",
436
+ " # Progress\n",
437
+ " if (idx + 1) % BATCH_LOG == 0:\n",
438
+ " elapsed = time.time() - start\n",
439
+ " pct = (idx + 1) / len(words) * 100\n",
440
+ " rate = (idx + 1) / elapsed if elapsed > 0 else 0\n",
441
+ " cur_size = os.path.getsize(OUTPUT_PATH) / (1024 ** 3)\n",
442
+ " print(f\" [{pct:5.1f}%] {idx+1:>7,}/{len(words):,} words |\"\n",
443
+ " f\" {total_written:>12,} lines | {cur_size:.2f} GB |\"\n",
444
+ " f\" {rate:.0f} words/sec\")\n",
445
+ "\n",
446
+ " elapsed = time.time() - start\n",
447
+ " final_size = os.path.getsize(OUTPUT_PATH) / (1024 ** 3)\n",
448
+ "\n",
449
+ " print()\n",
450
+ " print(\"=\" * 60)\n",
451
+ " print(f\" βœ… DONE in {elapsed:.1f}s ({elapsed/60:.1f} min)\")\n",
452
+ " print(f\" Words processed : {len(words):,}\")\n",
453
+ " print(f\" Lines written : {total_written:,}\")\n",
454
+ " print(f\" Output file : {OUTPUT_PATH}\")\n",
455
+ " print(f\" File size : {final_size:.2f} GB\")\n",
456
+ " print(\"=\" * 60)\n",
457
+ "\n",
458
+ "\n",
459
+ "if __name__ == '__main__':\n",
460
+ " main()\n"
461
+ ],
462
+ "metadata": {
463
+ "colab": {
464
+ "base_uri": "https://localhost:8080/"
465
+ },
466
+ "id": "Et0QfIxpJz_5",
467
+ "outputId": "e7e72965-f709-45c0-ae56-abf76b89d714"
468
+ },
469
+ "execution_count": null,
470
+ "outputs": [
471
+ {
472
+ "output_type": "stream",
473
+ "name": "stdout",
474
+ "text": [
475
+ "Reading words from: words.txt\n",
476
+ "Total raw entries: 466,550\n",
477
+ "Filtered to 125,414 words (alpha-only, len 3-7)\n",
478
+ "\n",
479
+ "Estimating output size (this may take a moment)...\n",
480
+ " Estimated permutations : 173,110,626\n",
481
+ " Estimated file size : 2.42 GB\n",
482
+ "\n",
483
+ "Proceeding with generation β†’ misspellings_permutations.txt\n",
484
+ "============================================================\n",
485
+ " [ 4.0%] 5,000/125,414 words | 5,810,553 lines | 0.08 GB | 898 words/sec\n",
486
+ " [ 8.0%] 10,000/125,414 words | 11,972,245 lines | 0.18 GB | 781 words/sec\n",
487
+ " [ 12.0%] 15,000/125,414 words | 19,094,747 lines | 0.28 GB | 775 words/sec\n",
488
+ " [ 15.9%] 20,000/125,414 words | 26,800,249 lines | 0.39 GB | 721 words/sec\n",
489
+ " [ 19.9%] 25,000/125,414 words | 35,047,153 lines | 0.51 GB | 690 words/sec\n",
490
+ " [ 23.9%] 30,000/125,414 words | 42,273,166 lines | 0.62 GB | 695 words/sec\n",
491
+ " [ 27.9%] 35,000/125,414 words | 48,702,338 lines | 0.71 GB | 692 words/sec\n",
492
+ " [ 31.9%] 40,000/125,414 words | 55,295,151 lines | 0.81 GB | 703 words/sec\n",
493
+ " [ 35.9%] 45,000/125,414 words | 62,710,327 lines | 0.92 GB | 690 words/sec\n",
494
+ " [ 39.9%] 50,000/125,414 words | 69,722,485 lines | 1.02 GB | 690 words/sec\n",
495
+ " [ 43.9%] 55,000/125,414 words | 76,146,526 lines | 1.12 GB | 674 words/sec\n",
496
+ " [ 47.8%] 60,000/125,414 words | 81,994,038 lines | 1.20 GB | 686 words/sec\n",
497
+ " [ 51.8%] 65,000/125,414 words | 88,058,594 lines | 1.29 GB | 683 words/sec\n",
498
+ " [ 55.8%] 70,000/125,414 words | 94,651,291 lines | 1.39 GB | 688 words/sec\n",
499
+ " [ 59.8%] 75,000/125,414 words | 101,636,647 lines | 1.49 GB | 679 words/sec\n",
500
+ " [ 63.8%] 80,000/125,414 words | 107,086,424 lines | 1.57 GB | 691 words/sec\n",
501
+ " [ 67.8%] 85,000/125,414 words | 114,898,717 lines | 1.68 GB | 678 words/sec\n",
502
+ " [ 71.8%] 90,000/125,414 words | 123,278,791 lines | 1.80 GB | 675 words/sec\n",
503
+ " [ 75.7%] 95,000/125,414 words | 129,821,900 lines | 1.90 GB | 669 words/sec\n",
504
+ " [ 79.7%] 100,000/125,414 words | 136,429,269 lines | 2.00 GB | 673 words/sec\n",
505
+ " [ 83.7%] 105,000/125,414 words | 143,342,171 lines | 2.10 GB | 667 words/sec\n",
506
+ " [ 87.7%] 110,000/125,414 words | 150,701,210 lines | 2.21 GB | 666 words/sec\n",
507
+ " [ 91.7%] 115,000/125,414 words | 157,479,616 lines | 2.31 GB | 665 words/sec\n",
508
+ " [ 95.7%] 120,000/125,414 words | 165,619,673 lines | 2.43 GB | 662 words/sec\n",
509
+ " [ 99.7%] 125,000/125,414 words | 172,558,768 lines | 2.53 GB | 661 words/sec\n",
510
+ "\n",
511
+ "============================================================\n",
512
+ " βœ… DONE in 189.5s (3.2 min)\n",
513
+ " Words processed : 125,414\n",
514
+ " Lines written : 173,110,626\n",
515
+ " Output file : misspellings_permutations.txt\n",
516
+ " File size : 2.53 GB\n",
517
+ "============================================================\n"
518
+ ]
519
+ }
520
+ ]
521
+ },
522
+ {
523
+ "cell_type": "code",
524
+ "source": [
525
+ "# If saved to VM disk:\n",
526
+ "files.download('misspellings_permutations.txt')\n",
527
+ "\n",
528
+ "# If saved to Google Drive: just access it from drive.google.com\n",
529
+ "\n"
530
+ ],
531
+ "metadata": {
532
+ "id": "y9jWxvv8LWoH",
533
+ "outputId": "d8d754d3-234e-4020-bcc7-a19f3fc5fb26",
534
+ "colab": {
535
+ "base_uri": "https://localhost:8080/",
536
+ "height": 34
537
+ }
538
+ },
539
+ "execution_count": null,
540
+ "outputs": [
541
+ {
542
+ "output_type": "display_data",
543
+ "data": {
544
+ "text/plain": [
545
+ "<IPython.core.display.Javascript object>"
546
+ ],
547
+ "application/javascript": [
548
+ "\n",
549
+ " async function download(id, filename, size) {\n",
550
+ " if (!google.colab.kernel.accessAllowed) {\n",
551
+ " return;\n",
552
+ " }\n",
553
+ " const div = document.createElement('div');\n",
554
+ " const label = document.createElement('label');\n",
555
+ " label.textContent = `Downloading \"${filename}\": `;\n",
556
+ " div.appendChild(label);\n",
557
+ " const progress = document.createElement('progress');\n",
558
+ " progress.max = size;\n",
559
+ " div.appendChild(progress);\n",
560
+ " document.body.appendChild(div);\n",
561
+ "\n",
562
+ " const buffers = [];\n",
563
+ " let downloaded = 0;\n",
564
+ "\n",
565
+ " const channel = await google.colab.kernel.comms.open(id);\n",
566
+ " // Send a message to notify the kernel that we're ready.\n",
567
+ " channel.send({})\n",
568
+ "\n",
569
+ " for await (const message of channel.messages) {\n",
570
+ " // Send a message to notify the kernel that we're ready.\n",
571
+ " channel.send({})\n",
572
+ " if (message.buffers) {\n",
573
+ " for (const buffer of message.buffers) {\n",
574
+ " buffers.push(buffer);\n",
575
+ " downloaded += buffer.byteLength;\n",
576
+ " progress.value = downloaded;\n",
577
+ " }\n",
578
+ " }\n",
579
+ " }\n",
580
+ " const blob = new Blob(buffers, {type: 'application/binary'});\n",
581
+ " const a = document.createElement('a');\n",
582
+ " a.href = window.URL.createObjectURL(blob);\n",
583
+ " a.download = filename;\n",
584
+ " div.appendChild(a);\n",
585
+ " a.click();\n",
586
+ " div.remove();\n",
587
+ " }\n",
588
+ " "
589
+ ]
590
+ },
591
+ "metadata": {}
592
+ },
593
+ {
594
+ "output_type": "display_data",
595
+ "data": {
596
+ "text/plain": [
597
+ "<IPython.core.display.Javascript object>"
598
+ ],
599
+ "application/javascript": [
600
+ "download(\"download_10941777-78c6-4833-b8e6-093feee02e11\", \"misspellings_permutations.txt\", 2721877361)"
601
+ ]
602
+ },
603
+ "metadata": {}
604
+ }
605
+ ]
606
+ }
607
+ ]
608
+ }