Hugging Face's logo

codeShare
/
JupyterNotebooks

Model card Files
xet
Community
codeShare commited on
Commit
fa58912
·
verified ·
1 Parent(s): abc0b46

Upload sd_token_similarity_calculator.ipynb

Browse files
Files changed (1) hide show
  1. sd_token_similarity_calculator.ipynb +63 -2217
sd_token_similarity_calculator.ipynb CHANGED
@@ -11,1720 +11,6 @@
11
  },
12
  "language_info": {
13
  "name": "python"
14
- },
15
- "widgets": {
16
- "application/vnd.jupyter.widget-state+json": {
17
- "7b073d9c166a45539c651ecd0b27285c": {
18
- "model_module": "@jupyter-widgets/controls",
19
- "model_name": "HBoxModel",
20
- "model_module_version": "1.5.0",
21
- "state": {
22
- "_dom_classes": [],
23
- "_model_module": "@jupyter-widgets/controls",
24
- "_model_module_version": "1.5.0",
25
- "_model_name": "HBoxModel",
26
- "_view_count": null,
27
- "_view_module": "@jupyter-widgets/controls",
28
- "_view_module_version": "1.5.0",
29
- "_view_name": "HBoxView",
30
- "box_style": "",
31
- "children": [
32
- "IPY_MODEL_ffba5f49ba214984b8329b4e06e40dcf",
33
- "IPY_MODEL_e69b98b68527488cbb7c1ee5f6011a92",
34
- "IPY_MODEL_983145e29ab84636b1e266c73eeb6889"
35
- ],
36
- "layout": "IPY_MODEL_278c2c021faf4f2696c03070fc3511bf"
37
- }
38
- },
39
- "ffba5f49ba214984b8329b4e06e40dcf": {
40
- "model_module": "@jupyter-widgets/controls",
41
- "model_name": "HTMLModel",
42
- "model_module_version": "1.5.0",
43
- "state": {
44
- "_dom_classes": [],
45
- "_model_module": "@jupyter-widgets/controls",
46
- "_model_module_version": "1.5.0",
47
- "_model_name": "HTMLModel",
48
- "_view_count": null,
49
- "_view_module": "@jupyter-widgets/controls",
50
- "_view_module_version": "1.5.0",
51
- "_view_name": "HTMLView",
52
- "description": "",
53
- "description_tooltip": null,
54
- "layout": "IPY_MODEL_3e9df4ac699843df8ae361aba1337b06",
55
- "placeholder": "​",
56
- "style": "IPY_MODEL_8e9cc6914fc14e56945716902fb11fe9",
57
- "value": "tokenizer_config.json: 100%"
58
- }
59
- },
60
- "e69b98b68527488cbb7c1ee5f6011a92": {
61
- "model_module": "@jupyter-widgets/controls",
62
- "model_name": "FloatProgressModel",
63
- "model_module_version": "1.5.0",
64
- "state": {
65
- "_dom_classes": [],
66
- "_model_module": "@jupyter-widgets/controls",
67
- "_model_module_version": "1.5.0",
68
- "_model_name": "FloatProgressModel",
69
- "_view_count": null,
70
- "_view_module": "@jupyter-widgets/controls",
71
- "_view_module_version": "1.5.0",
72
- "_view_name": "ProgressView",
73
- "bar_style": "success",
74
- "description": "",
75
- "description_tooltip": null,
76
- "layout": "IPY_MODEL_0ca945d6f8a5457d9706c211869983c8",
77
- "max": 905,
78
- "min": 0,
79
- "orientation": "horizontal",
80
- "style": "IPY_MODEL_69cf2c7a4ecf4c3188586c86c7056518",
81
- "value": 905
82
- }
83
- },
84
- "983145e29ab84636b1e266c73eeb6889": {
85
- "model_module": "@jupyter-widgets/controls",
86
- "model_name": "HTMLModel",
87
- "model_module_version": "1.5.0",
88
- "state": {
89
- "_dom_classes": [],
90
- "_model_module": "@jupyter-widgets/controls",
91
- "_model_module_version": "1.5.0",
92
- "_model_name": "HTMLModel",
93
- "_view_count": null,
94
- "_view_module": "@jupyter-widgets/controls",
95
- "_view_module_version": "1.5.0",
96
- "_view_name": "HTMLView",
97
- "description": "",
98
- "description_tooltip": null,
99
- "layout": "IPY_MODEL_1d60ac5b02de4cc7b0d51ed57165ff64",
100
- "placeholder": "​",
101
- "style": "IPY_MODEL_e34074fc893d47b8bf75a8089cd8df37",
102
- "value": " 905/905 [00:00<00:00, 32.4kB/s]"
103
- }
104
- },
105
- "278c2c021faf4f2696c03070fc3511bf": {
106
- "model_module": "@jupyter-widgets/base",
107
- "model_name": "LayoutModel",
108
- "model_module_version": "1.2.0",
109
- "state": {
110
- "_model_module": "@jupyter-widgets/base",
111
- "_model_module_version": "1.2.0",
112
- "_model_name": "LayoutModel",
113
- "_view_count": null,
114
- "_view_module": "@jupyter-widgets/base",
115
- "_view_module_version": "1.2.0",
116
- "_view_name": "LayoutView",
117
- "align_content": null,
118
- "align_items": null,
119
- "align_self": null,
120
- "border": null,
121
- "bottom": null,
122
- "display": null,
123
- "flex": null,
124
- "flex_flow": null,
125
- "grid_area": null,
126
- "grid_auto_columns": null,
127
- "grid_auto_flow": null,
128
- "grid_auto_rows": null,
129
- "grid_column": null,
130
- "grid_gap": null,
131
- "grid_row": null,
132
- "grid_template_areas": null,
133
- "grid_template_columns": null,
134
- "grid_template_rows": null,
135
- "height": null,
136
- "justify_content": null,
137
- "justify_items": null,
138
- "left": null,
139
- "margin": null,
140
- "max_height": null,
141
- "max_width": null,
142
- "min_height": null,
143
- "min_width": null,
144
- "object_fit": null,
145
- "object_position": null,
146
- "order": null,
147
- "overflow": null,
148
- "overflow_x": null,
149
- "overflow_y": null,
150
- "padding": null,
151
- "right": null,
152
- "top": null,
153
- "visibility": null,
154
- "width": null
155
- }
156
- },
157
- "3e9df4ac699843df8ae361aba1337b06": {
158
- "model_module": "@jupyter-widgets/base",
159
- "model_name": "LayoutModel",
160
- "model_module_version": "1.2.0",
161
- "state": {
162
- "_model_module": "@jupyter-widgets/base",
163
- "_model_module_version": "1.2.0",
164
- "_model_name": "LayoutModel",
165
- "_view_count": null,
166
- "_view_module": "@jupyter-widgets/base",
167
- "_view_module_version": "1.2.0",
168
- "_view_name": "LayoutView",
169
- "align_content": null,
170
- "align_items": null,
171
- "align_self": null,
172
- "border": null,
173
- "bottom": null,
174
- "display": null,
175
- "flex": null,
176
- "flex_flow": null,
177
- "grid_area": null,
178
- "grid_auto_columns": null,
179
- "grid_auto_flow": null,
180
- "grid_auto_rows": null,
181
- "grid_column": null,
182
- "grid_gap": null,
183
- "grid_row": null,
184
- "grid_template_areas": null,
185
- "grid_template_columns": null,
186
- "grid_template_rows": null,
187
- "height": null,
188
- "justify_content": null,
189
- "justify_items": null,
190
- "left": null,
191
- "margin": null,
192
- "max_height": null,
193
- "max_width": null,
194
- "min_height": null,
195
- "min_width": null,
196
- "object_fit": null,
197
- "object_position": null,
198
- "order": null,
199
- "overflow": null,
200
- "overflow_x": null,
201
- "overflow_y": null,
202
- "padding": null,
203
- "right": null,
204
- "top": null,
205
- "visibility": null,
206
- "width": null
207
- }
208
- },
209
- "8e9cc6914fc14e56945716902fb11fe9": {
210
- "model_module": "@jupyter-widgets/controls",
211
- "model_name": "DescriptionStyleModel",
212
- "model_module_version": "1.5.0",
213
- "state": {
214
- "_model_module": "@jupyter-widgets/controls",
215
- "_model_module_version": "1.5.0",
216
- "_model_name": "DescriptionStyleModel",
217
- "_view_count": null,
218
- "_view_module": "@jupyter-widgets/base",
219
- "_view_module_version": "1.2.0",
220
- "_view_name": "StyleView",
221
- "description_width": ""
222
- }
223
- },
224
- "0ca945d6f8a5457d9706c211869983c8": {
225
- "model_module": "@jupyter-widgets/base",
226
- "model_name": "LayoutModel",
227
- "model_module_version": "1.2.0",
228
- "state": {
229
- "_model_module": "@jupyter-widgets/base",
230
- "_model_module_version": "1.2.0",
231
- "_model_name": "LayoutModel",
232
- "_view_count": null,
233
- "_view_module": "@jupyter-widgets/base",
234
- "_view_module_version": "1.2.0",
235
- "_view_name": "LayoutView",
236
- "align_content": null,
237
- "align_items": null,
238
- "align_self": null,
239
- "border": null,
240
- "bottom": null,
241
- "display": null,
242
- "flex": null,
243
- "flex_flow": null,
244
- "grid_area": null,
245
- "grid_auto_columns": null,
246
- "grid_auto_flow": null,
247
- "grid_auto_rows": null,
248
- "grid_column": null,
249
- "grid_gap": null,
250
- "grid_row": null,
251
- "grid_template_areas": null,
252
- "grid_template_columns": null,
253
- "grid_template_rows": null,
254
- "height": null,
255
- "justify_content": null,
256
- "justify_items": null,
257
- "left": null,
258
- "margin": null,
259
- "max_height": null,
260
- "max_width": null,
261
- "min_height": null,
262
- "min_width": null,
263
- "object_fit": null,
264
- "object_position": null,
265
- "order": null,
266
- "overflow": null,
267
- "overflow_x": null,
268
- "overflow_y": null,
269
- "padding": null,
270
- "right": null,
271
- "top": null,
272
- "visibility": null,
273
- "width": null
274
- }
275
- },
276
- "69cf2c7a4ecf4c3188586c86c7056518": {
277
- "model_module": "@jupyter-widgets/controls",
278
- "model_name": "ProgressStyleModel",
279
- "model_module_version": "1.5.0",
280
- "state": {
281
- "_model_module": "@jupyter-widgets/controls",
282
- "_model_module_version": "1.5.0",
283
- "_model_name": "ProgressStyleModel",
284
- "_view_count": null,
285
- "_view_module": "@jupyter-widgets/base",
286
- "_view_module_version": "1.2.0",
287
- "_view_name": "StyleView",
288
- "bar_color": null,
289
- "description_width": ""
290
- }
291
- },
292
- "1d60ac5b02de4cc7b0d51ed57165ff64": {
293
- "model_module": "@jupyter-widgets/base",
294
- "model_name": "LayoutModel",
295
- "model_module_version": "1.2.0",
296
- "state": {
297
- "_model_module": "@jupyter-widgets/base",
298
- "_model_module_version": "1.2.0",
299
- "_model_name": "LayoutModel",
300
- "_view_count": null,
301
- "_view_module": "@jupyter-widgets/base",
302
- "_view_module_version": "1.2.0",
303
- "_view_name": "LayoutView",
304
- "align_content": null,
305
- "align_items": null,
306
- "align_self": null,
307
- "border": null,
308
- "bottom": null,
309
- "display": null,
310
- "flex": null,
311
- "flex_flow": null,
312
- "grid_area": null,
313
- "grid_auto_columns": null,
314
- "grid_auto_flow": null,
315
- "grid_auto_rows": null,
316
- "grid_column": null,
317
- "grid_gap": null,
318
- "grid_row": null,
319
- "grid_template_areas": null,
320
- "grid_template_columns": null,
321
- "grid_template_rows": null,
322
- "height": null,
323
- "justify_content": null,
324
- "justify_items": null,
325
- "left": null,
326
- "margin": null,
327
- "max_height": null,
328
- "max_width": null,
329
- "min_height": null,
330
- "min_width": null,
331
- "object_fit": null,
332
- "object_position": null,
333
- "order": null,
334
- "overflow": null,
335
- "overflow_x": null,
336
- "overflow_y": null,
337
- "padding": null,
338
- "right": null,
339
- "top": null,
340
- "visibility": null,
341
- "width": null
342
- }
343
- },
344
- "e34074fc893d47b8bf75a8089cd8df37": {
345
- "model_module": "@jupyter-widgets/controls",
346
- "model_name": "DescriptionStyleModel",
347
- "model_module_version": "1.5.0",
348
- "state": {
349
- "_model_module": "@jupyter-widgets/controls",
350
- "_model_module_version": "1.5.0",
351
- "_model_name": "DescriptionStyleModel",
352
- "_view_count": null,
353
- "_view_module": "@jupyter-widgets/base",
354
- "_view_module_version": "1.2.0",
355
- "_view_name": "StyleView",
356
- "description_width": ""
357
- }
358
- },
359
- "26cb34af09544ea1b17526891fd319e3": {
360
- "model_module": "@jupyter-widgets/controls",
361
- "model_name": "HBoxModel",
362
- "model_module_version": "1.5.0",
363
- "state": {
364
- "_dom_classes": [],
365
- "_model_module": "@jupyter-widgets/controls",
366
- "_model_module_version": "1.5.0",
367
- "_model_name": "HBoxModel",
368
- "_view_count": null,
369
- "_view_module": "@jupyter-widgets/controls",
370
- "_view_module_version": "1.5.0",
371
- "_view_name": "HBoxView",
372
- "box_style": "",
373
- "children": [
374
- "IPY_MODEL_71627d67c8ec4993a8b1e4a93d53e8db",
375
- "IPY_MODEL_ec11e5b722dd4d69bcd758775edd61b6",
376
- "IPY_MODEL_63f760d16a1249b6a80f7a73989299ac"
377
- ],
378
- "layout": "IPY_MODEL_ba3dae52e6bc4a48945ebde7464a5277"
379
- }
380
- },
381
- "71627d67c8ec4993a8b1e4a93d53e8db": {
382
- "model_module": "@jupyter-widgets/controls",
383
- "model_name": "HTMLModel",
384
- "model_module_version": "1.5.0",
385
- "state": {
386
- "_dom_classes": [],
387
- "_model_module": "@jupyter-widgets/controls",
388
- "_model_module_version": "1.5.0",
389
- "_model_name": "HTMLModel",
390
- "_view_count": null,
391
- "_view_module": "@jupyter-widgets/controls",
392
- "_view_module_version": "1.5.0",
393
- "_view_name": "HTMLView",
394
- "description": "",
395
- "description_tooltip": null,
396
- "layout": "IPY_MODEL_e914f0b293b4410a82e02f990b8a9a1d",
397
- "placeholder": "​",
398
- "style": "IPY_MODEL_7e18f5165da84f60bea9d9a1819e17df",
399
- "value": "vocab.json: 100%"
400
- }
401
- },
402
- "ec11e5b722dd4d69bcd758775edd61b6": {
403
- "model_module": "@jupyter-widgets/controls",
404
- "model_name": "FloatProgressModel",
405
- "model_module_version": "1.5.0",
406
- "state": {
407
- "_dom_classes": [],
408
- "_model_module": "@jupyter-widgets/controls",
409
- "_model_module_version": "1.5.0",
410
- "_model_name": "FloatProgressModel",
411
- "_view_count": null,
412
- "_view_module": "@jupyter-widgets/controls",
413
- "_view_module_version": "1.5.0",
414
- "_view_name": "ProgressView",
415
- "bar_style": "success",
416
- "description": "",
417
- "description_tooltip": null,
418
- "layout": "IPY_MODEL_ba1b3dc7d77b4fee9addea98f60277bc",
419
- "max": 961143,
420
- "min": 0,
421
- "orientation": "horizontal",
422
- "style": "IPY_MODEL_b40d206858bd46e19ef99aa1de302060",
423
- "value": 961143
424
- }
425
- },
426
- "63f760d16a1249b6a80f7a73989299ac": {
427
- "model_module": "@jupyter-widgets/controls",
428
- "model_name": "HTMLModel",
429
- "model_module_version": "1.5.0",
430
- "state": {
431
- "_dom_classes": [],
432
- "_model_module": "@jupyter-widgets/controls",
433
- "_model_module_version": "1.5.0",
434
- "_model_name": "HTMLModel",
435
- "_view_count": null,
436
- "_view_module": "@jupyter-widgets/controls",
437
- "_view_module_version": "1.5.0",
438
- "_view_name": "HTMLView",
439
- "description": "",
440
- "description_tooltip": null,
441
- "layout": "IPY_MODEL_92113b912eea41b4b2bfadd3f65cd835",
442
- "placeholder": "​",
443
- "style": "IPY_MODEL_bfca51dacc6a4762810309006fc6eca5",
444
- "value": " 961k/961k [00:00<00:00, 5.41MB/s]"
445
- }
446
- },
447
- "ba3dae52e6bc4a48945ebde7464a5277": {
448
- "model_module": "@jupyter-widgets/base",
449
- "model_name": "LayoutModel",
450
- "model_module_version": "1.2.0",
451
- "state": {
452
- "_model_module": "@jupyter-widgets/base",
453
- "_model_module_version": "1.2.0",
454
- "_model_name": "LayoutModel",
455
- "_view_count": null,
456
- "_view_module": "@jupyter-widgets/base",
457
- "_view_module_version": "1.2.0",
458
- "_view_name": "LayoutView",
459
- "align_content": null,
460
- "align_items": null,
461
- "align_self": null,
462
- "border": null,
463
- "bottom": null,
464
- "display": null,
465
- "flex": null,
466
- "flex_flow": null,
467
- "grid_area": null,
468
- "grid_auto_columns": null,
469
- "grid_auto_flow": null,
470
- "grid_auto_rows": null,
471
- "grid_column": null,
472
- "grid_gap": null,
473
- "grid_row": null,
474
- "grid_template_areas": null,
475
- "grid_template_columns": null,
476
- "grid_template_rows": null,
477
- "height": null,
478
- "justify_content": null,
479
- "justify_items": null,
480
- "left": null,
481
- "margin": null,
482
- "max_height": null,
483
- "max_width": null,
484
- "min_height": null,
485
- "min_width": null,
486
- "object_fit": null,
487
- "object_position": null,
488
- "order": null,
489
- "overflow": null,
490
- "overflow_x": null,
491
- "overflow_y": null,
492
- "padding": null,
493
- "right": null,
494
- "top": null,
495
- "visibility": null,
496
- "width": null
497
- }
498
- },
499
- "e914f0b293b4410a82e02f990b8a9a1d": {
500
- "model_module": "@jupyter-widgets/base",
501
- "model_name": "LayoutModel",
502
- "model_module_version": "1.2.0",
503
- "state": {
504
- "_model_module": "@jupyter-widgets/base",
505
- "_model_module_version": "1.2.0",
506
- "_model_name": "LayoutModel",
507
- "_view_count": null,
508
- "_view_module": "@jupyter-widgets/base",
509
- "_view_module_version": "1.2.0",
510
- "_view_name": "LayoutView",
511
- "align_content": null,
512
- "align_items": null,
513
- "align_self": null,
514
- "border": null,
515
- "bottom": null,
516
- "display": null,
517
- "flex": null,
518
- "flex_flow": null,
519
- "grid_area": null,
520
- "grid_auto_columns": null,
521
- "grid_auto_flow": null,
522
- "grid_auto_rows": null,
523
- "grid_column": null,
524
- "grid_gap": null,
525
- "grid_row": null,
526
- "grid_template_areas": null,
527
- "grid_template_columns": null,
528
- "grid_template_rows": null,
529
- "height": null,
530
- "justify_content": null,
531
- "justify_items": null,
532
- "left": null,
533
- "margin": null,
534
- "max_height": null,
535
- "max_width": null,
536
- "min_height": null,
537
- "min_width": null,
538
- "object_fit": null,
539
- "object_position": null,
540
- "order": null,
541
- "overflow": null,
542
- "overflow_x": null,
543
- "overflow_y": null,
544
- "padding": null,
545
- "right": null,
546
- "top": null,
547
- "visibility": null,
548
- "width": null
549
- }
550
- },
551
- "7e18f5165da84f60bea9d9a1819e17df": {
552
- "model_module": "@jupyter-widgets/controls",
553
- "model_name": "DescriptionStyleModel",
554
- "model_module_version": "1.5.0",
555
- "state": {
556
- "_model_module": "@jupyter-widgets/controls",
557
- "_model_module_version": "1.5.0",
558
- "_model_name": "DescriptionStyleModel",
559
- "_view_count": null,
560
- "_view_module": "@jupyter-widgets/base",
561
- "_view_module_version": "1.2.0",
562
- "_view_name": "StyleView",
563
- "description_width": ""
564
- }
565
- },
566
- "ba1b3dc7d77b4fee9addea98f60277bc": {
567
- "model_module": "@jupyter-widgets/base",
568
- "model_name": "LayoutModel",
569
- "model_module_version": "1.2.0",
570
- "state": {
571
- "_model_module": "@jupyter-widgets/base",
572
- "_model_module_version": "1.2.0",
573
- "_model_name": "LayoutModel",
574
- "_view_count": null,
575
- "_view_module": "@jupyter-widgets/base",
576
- "_view_module_version": "1.2.0",
577
- "_view_name": "LayoutView",
578
- "align_content": null,
579
- "align_items": null,
580
- "align_self": null,
581
- "border": null,
582
- "bottom": null,
583
- "display": null,
584
- "flex": null,
585
- "flex_flow": null,
586
- "grid_area": null,
587
- "grid_auto_columns": null,
588
- "grid_auto_flow": null,
589
- "grid_auto_rows": null,
590
- "grid_column": null,
591
- "grid_gap": null,
592
- "grid_row": null,
593
- "grid_template_areas": null,
594
- "grid_template_columns": null,
595
- "grid_template_rows": null,
596
- "height": null,
597
- "justify_content": null,
598
- "justify_items": null,
599
- "left": null,
600
- "margin": null,
601
- "max_height": null,
602
- "max_width": null,
603
- "min_height": null,
604
- "min_width": null,
605
- "object_fit": null,
606
- "object_position": null,
607
- "order": null,
608
- "overflow": null,
609
- "overflow_x": null,
610
- "overflow_y": null,
611
- "padding": null,
612
- "right": null,
613
- "top": null,
614
- "visibility": null,
615
- "width": null
616
- }
617
- },
618
- "b40d206858bd46e19ef99aa1de302060": {
619
- "model_module": "@jupyter-widgets/controls",
620
- "model_name": "ProgressStyleModel",
621
- "model_module_version": "1.5.0",
622
- "state": {
623
- "_model_module": "@jupyter-widgets/controls",
624
- "_model_module_version": "1.5.0",
625
- "_model_name": "ProgressStyleModel",
626
- "_view_count": null,
627
- "_view_module": "@jupyter-widgets/base",
628
- "_view_module_version": "1.2.0",
629
- "_view_name": "StyleView",
630
- "bar_color": null,
631
- "description_width": ""
632
- }
633
- },
634
- "92113b912eea41b4b2bfadd3f65cd835": {
635
- "model_module": "@jupyter-widgets/base",
636
- "model_name": "LayoutModel",
637
- "model_module_version": "1.2.0",
638
- "state": {
639
- "_model_module": "@jupyter-widgets/base",
640
- "_model_module_version": "1.2.0",
641
- "_model_name": "LayoutModel",
642
- "_view_count": null,
643
- "_view_module": "@jupyter-widgets/base",
644
- "_view_module_version": "1.2.0",
645
- "_view_name": "LayoutView",
646
- "align_content": null,
647
- "align_items": null,
648
- "align_self": null,
649
- "border": null,
650
- "bottom": null,
651
- "display": null,
652
- "flex": null,
653
- "flex_flow": null,
654
- "grid_area": null,
655
- "grid_auto_columns": null,
656
- "grid_auto_flow": null,
657
- "grid_auto_rows": null,
658
- "grid_column": null,
659
- "grid_gap": null,
660
- "grid_row": null,
661
- "grid_template_areas": null,
662
- "grid_template_columns": null,
663
- "grid_template_rows": null,
664
- "height": null,
665
- "justify_content": null,
666
- "justify_items": null,
667
- "left": null,
668
- "margin": null,
669
- "max_height": null,
670
- "max_width": null,
671
- "min_height": null,
672
- "min_width": null,
673
- "object_fit": null,
674
- "object_position": null,
675
- "order": null,
676
- "overflow": null,
677
- "overflow_x": null,
678
- "overflow_y": null,
679
- "padding": null,
680
- "right": null,
681
- "top": null,
682
- "visibility": null,
683
- "width": null
684
- }
685
- },
686
- "bfca51dacc6a4762810309006fc6eca5": {
687
- "model_module": "@jupyter-widgets/controls",
688
- "model_name": "DescriptionStyleModel",
689
- "model_module_version": "1.5.0",
690
- "state": {
691
- "_model_module": "@jupyter-widgets/controls",
692
- "_model_module_version": "1.5.0",
693
- "_model_name": "DescriptionStyleModel",
694
- "_view_count": null,
695
- "_view_module": "@jupyter-widgets/base",
696
- "_view_module_version": "1.2.0",
697
- "_view_name": "StyleView",
698
- "description_width": ""
699
- }
700
- },
701
- "a7a66681148748ae94788ee276925bec": {
702
- "model_module": "@jupyter-widgets/controls",
703
- "model_name": "HBoxModel",
704
- "model_module_version": "1.5.0",
705
- "state": {
706
- "_dom_classes": [],
707
- "_model_module": "@jupyter-widgets/controls",
708
- "_model_module_version": "1.5.0",
709
- "_model_name": "HBoxModel",
710
- "_view_count": null,
711
- "_view_module": "@jupyter-widgets/controls",
712
- "_view_module_version": "1.5.0",
713
- "_view_name": "HBoxView",
714
- "box_style": "",
715
- "children": [
716
- "IPY_MODEL_438a2db97b8649cfbd5b6d87aeb4b13a",
717
- "IPY_MODEL_8608253bb3cc4e93a379b05648e1e35c",
718
- "IPY_MODEL_aa84ccf805b3476ea155e64ed31edef6"
719
- ],
720
- "layout": "IPY_MODEL_851b1acc7d524ec1aabb04012c469432"
721
- }
722
- },
723
- "438a2db97b8649cfbd5b6d87aeb4b13a": {
724
- "model_module": "@jupyter-widgets/controls",
725
- "model_name": "HTMLModel",
726
- "model_module_version": "1.5.0",
727
- "state": {
728
- "_dom_classes": [],
729
- "_model_module": "@jupyter-widgets/controls",
730
- "_model_module_version": "1.5.0",
731
- "_model_name": "HTMLModel",
732
- "_view_count": null,
733
- "_view_module": "@jupyter-widgets/controls",
734
- "_view_module_version": "1.5.0",
735
- "_view_name": "HTMLView",
736
- "description": "",
737
- "description_tooltip": null,
738
- "layout": "IPY_MODEL_2277a0b29cd841a2a1ad5526f39c28a6",
739
- "placeholder": "​",
740
- "style": "IPY_MODEL_5f7a4ea2308d4cb2b6e4a0f2f925e2b1",
741
- "value": "merges.txt: 100%"
742
- }
743
- },
744
- "8608253bb3cc4e93a379b05648e1e35c": {
745
- "model_module": "@jupyter-widgets/controls",
746
- "model_name": "FloatProgressModel",
747
- "model_module_version": "1.5.0",
748
- "state": {
749
- "_dom_classes": [],
750
- "_model_module": "@jupyter-widgets/controls",
751
- "_model_module_version": "1.5.0",
752
- "_model_name": "FloatProgressModel",
753
- "_view_count": null,
754
- "_view_module": "@jupyter-widgets/controls",
755
- "_view_module_version": "1.5.0",
756
- "_view_name": "ProgressView",
757
- "bar_style": "success",
758
- "description": "",
759
- "description_tooltip": null,
760
- "layout": "IPY_MODEL_dc23a6ac81894fe484247273be9d6a83",
761
- "max": 524619,
762
- "min": 0,
763
- "orientation": "horizontal",
764
- "style": "IPY_MODEL_d37f42e8924a4b0fafd2e407b1db865b",
765
- "value": 524619
766
- }
767
- },
768
- "aa84ccf805b3476ea155e64ed31edef6": {
769
- "model_module": "@jupyter-widgets/controls",
770
- "model_name": "HTMLModel",
771
- "model_module_version": "1.5.0",
772
- "state": {
773
- "_dom_classes": [],
774
- "_model_module": "@jupyter-widgets/controls",
775
- "_model_module_version": "1.5.0",
776
- "_model_name": "HTMLModel",
777
- "_view_count": null,
778
- "_view_module": "@jupyter-widgets/controls",
779
- "_view_module_version": "1.5.0",
780
- "_view_name": "HTMLView",
781
- "description": "",
782
- "description_tooltip": null,
783
- "layout": "IPY_MODEL_03fa9da8e5084d688bc7b0f4d8ba3fb5",
784
- "placeholder": "​",
785
- "style": "IPY_MODEL_613fa346a05441828356976ea941a7c7",
786
- "value": " 525k/525k [00:00<00:00, 13.6MB/s]"
787
- }
788
- },
789
- "851b1acc7d524ec1aabb04012c469432": {
790
- "model_module": "@jupyter-widgets/base",
791
- "model_name": "LayoutModel",
792
- "model_module_version": "1.2.0",
793
- "state": {
794
- "_model_module": "@jupyter-widgets/base",
795
- "_model_module_version": "1.2.0",
796
- "_model_name": "LayoutModel",
797
- "_view_count": null,
798
- "_view_module": "@jupyter-widgets/base",
799
- "_view_module_version": "1.2.0",
800
- "_view_name": "LayoutView",
801
- "align_content": null,
802
- "align_items": null,
803
- "align_self": null,
804
- "border": null,
805
- "bottom": null,
806
- "display": null,
807
- "flex": null,
808
- "flex_flow": null,
809
- "grid_area": null,
810
- "grid_auto_columns": null,
811
- "grid_auto_flow": null,
812
- "grid_auto_rows": null,
813
- "grid_column": null,
814
- "grid_gap": null,
815
- "grid_row": null,
816
- "grid_template_areas": null,
817
- "grid_template_columns": null,
818
- "grid_template_rows": null,
819
- "height": null,
820
- "justify_content": null,
821
- "justify_items": null,
822
- "left": null,
823
- "margin": null,
824
- "max_height": null,
825
- "max_width": null,
826
- "min_height": null,
827
- "min_width": null,
828
- "object_fit": null,
829
- "object_position": null,
830
- "order": null,
831
- "overflow": null,
832
- "overflow_x": null,
833
- "overflow_y": null,
834
- "padding": null,
835
- "right": null,
836
- "top": null,
837
- "visibility": null,
838
- "width": null
839
- }
840
- },
841
- "2277a0b29cd841a2a1ad5526f39c28a6": {
842
- "model_module": "@jupyter-widgets/base",
843
- "model_name": "LayoutModel",
844
- "model_module_version": "1.2.0",
845
- "state": {
846
- "_model_module": "@jupyter-widgets/base",
847
- "_model_module_version": "1.2.0",
848
- "_model_name": "LayoutModel",
849
- "_view_count": null,
850
- "_view_module": "@jupyter-widgets/base",
851
- "_view_module_version": "1.2.0",
852
- "_view_name": "LayoutView",
853
- "align_content": null,
854
- "align_items": null,
855
- "align_self": null,
856
- "border": null,
857
- "bottom": null,
858
- "display": null,
859
- "flex": null,
860
- "flex_flow": null,
861
- "grid_area": null,
862
- "grid_auto_columns": null,
863
- "grid_auto_flow": null,
864
- "grid_auto_rows": null,
865
- "grid_column": null,
866
- "grid_gap": null,
867
- "grid_row": null,
868
- "grid_template_areas": null,
869
- "grid_template_columns": null,
870
- "grid_template_rows": null,
871
- "height": null,
872
- "justify_content": null,
873
- "justify_items": null,
874
- "left": null,
875
- "margin": null,
876
- "max_height": null,
877
- "max_width": null,
878
- "min_height": null,
879
- "min_width": null,
880
- "object_fit": null,
881
- "object_position": null,
882
- "order": null,
883
- "overflow": null,
884
- "overflow_x": null,
885
- "overflow_y": null,
886
- "padding": null,
887
- "right": null,
888
- "top": null,
889
- "visibility": null,
890
- "width": null
891
- }
892
- },
893
- "5f7a4ea2308d4cb2b6e4a0f2f925e2b1": {
894
- "model_module": "@jupyter-widgets/controls",
895
- "model_name": "DescriptionStyleModel",
896
- "model_module_version": "1.5.0",
897
- "state": {
898
- "_model_module": "@jupyter-widgets/controls",
899
- "_model_module_version": "1.5.0",
900
- "_model_name": "DescriptionStyleModel",
901
- "_view_count": null,
902
- "_view_module": "@jupyter-widgets/base",
903
- "_view_module_version": "1.2.0",
904
- "_view_name": "StyleView",
905
- "description_width": ""
906
- }
907
- },
908
- "dc23a6ac81894fe484247273be9d6a83": {
909
- "model_module": "@jupyter-widgets/base",
910
- "model_name": "LayoutModel",
911
- "model_module_version": "1.2.0",
912
- "state": {
913
- "_model_module": "@jupyter-widgets/base",
914
- "_model_module_version": "1.2.0",
915
- "_model_name": "LayoutModel",
916
- "_view_count": null,
917
- "_view_module": "@jupyter-widgets/base",
918
- "_view_module_version": "1.2.0",
919
- "_view_name": "LayoutView",
920
- "align_content": null,
921
- "align_items": null,
922
- "align_self": null,
923
- "border": null,
924
- "bottom": null,
925
- "display": null,
926
- "flex": null,
927
- "flex_flow": null,
928
- "grid_area": null,
929
- "grid_auto_columns": null,
930
- "grid_auto_flow": null,
931
- "grid_auto_rows": null,
932
- "grid_column": null,
933
- "grid_gap": null,
934
- "grid_row": null,
935
- "grid_template_areas": null,
936
- "grid_template_columns": null,
937
- "grid_template_rows": null,
938
- "height": null,
939
- "justify_content": null,
940
- "justify_items": null,
941
- "left": null,
942
- "margin": null,
943
- "max_height": null,
944
- "max_width": null,
945
- "min_height": null,
946
- "min_width": null,
947
- "object_fit": null,
948
- "object_position": null,
949
- "order": null,
950
- "overflow": null,
951
- "overflow_x": null,
952
- "overflow_y": null,
953
- "padding": null,
954
- "right": null,
955
- "top": null,
956
- "visibility": null,
957
- "width": null
958
- }
959
- },
960
- "d37f42e8924a4b0fafd2e407b1db865b": {
961
- "model_module": "@jupyter-widgets/controls",
962
- "model_name": "ProgressStyleModel",
963
- "model_module_version": "1.5.0",
964
- "state": {
965
- "_model_module": "@jupyter-widgets/controls",
966
- "_model_module_version": "1.5.0",
967
- "_model_name": "ProgressStyleModel",
968
- "_view_count": null,
969
- "_view_module": "@jupyter-widgets/base",
970
- "_view_module_version": "1.2.0",
971
- "_view_name": "StyleView",
972
- "bar_color": null,
973
- "description_width": ""
974
- }
975
- },
976
- "03fa9da8e5084d688bc7b0f4d8ba3fb5": {
977
- "model_module": "@jupyter-widgets/base",
978
- "model_name": "LayoutModel",
979
- "model_module_version": "1.2.0",
980
- "state": {
981
- "_model_module": "@jupyter-widgets/base",
982
- "_model_module_version": "1.2.0",
983
- "_model_name": "LayoutModel",
984
- "_view_count": null,
985
- "_view_module": "@jupyter-widgets/base",
986
- "_view_module_version": "1.2.0",
987
- "_view_name": "LayoutView",
988
- "align_content": null,
989
- "align_items": null,
990
- "align_self": null,
991
- "border": null,
992
- "bottom": null,
993
- "display": null,
994
- "flex": null,
995
- "flex_flow": null,
996
- "grid_area": null,
997
- "grid_auto_columns": null,
998
- "grid_auto_flow": null,
999
- "grid_auto_rows": null,
1000
- "grid_column": null,
1001
- "grid_gap": null,
1002
- "grid_row": null,
1003
- "grid_template_areas": null,
1004
- "grid_template_columns": null,
1005
- "grid_template_rows": null,
1006
- "height": null,
1007
- "justify_content": null,
1008
- "justify_items": null,
1009
- "left": null,
1010
- "margin": null,
1011
- "max_height": null,
1012
- "max_width": null,
1013
- "min_height": null,
1014
- "min_width": null,
1015
- "object_fit": null,
1016
- "object_position": null,
1017
- "order": null,
1018
- "overflow": null,
1019
- "overflow_x": null,
1020
- "overflow_y": null,
1021
- "padding": null,
1022
- "right": null,
1023
- "top": null,
1024
- "visibility": null,
1025
- "width": null
1026
- }
1027
- },
1028
- "613fa346a05441828356976ea941a7c7": {
1029
- "model_module": "@jupyter-widgets/controls",
1030
- "model_name": "DescriptionStyleModel",
1031
- "model_module_version": "1.5.0",
1032
- "state": {
1033
- "_model_module": "@jupyter-widgets/controls",
1034
- "_model_module_version": "1.5.0",
1035
- "_model_name": "DescriptionStyleModel",
1036
- "_view_count": null,
1037
- "_view_module": "@jupyter-widgets/base",
1038
- "_view_module_version": "1.2.0",
1039
- "_view_name": "StyleView",
1040
- "description_width": ""
1041
- }
1042
- },
1043
- "f580a1015d334b398d024357adf898cf": {
1044
- "model_module": "@jupyter-widgets/controls",
1045
- "model_name": "HBoxModel",
1046
- "model_module_version": "1.5.0",
1047
- "state": {
1048
- "_dom_classes": [],
1049
- "_model_module": "@jupyter-widgets/controls",
1050
- "_model_module_version": "1.5.0",
1051
- "_model_name": "HBoxModel",
1052
- "_view_count": null,
1053
- "_view_module": "@jupyter-widgets/controls",
1054
- "_view_module_version": "1.5.0",
1055
- "_view_name": "HBoxView",
1056
- "box_style": "",
1057
- "children": [
1058
- "IPY_MODEL_866f75c147c747f5b9f3d2993440823e",
1059
- "IPY_MODEL_fe4842b3b96a4c1db17abe9a90e026c3",
1060
- "IPY_MODEL_69561cfd572c4e0fbc107e25a1e50654"
1061
- ],
1062
- "layout": "IPY_MODEL_00bc783a0f344176af0a760569c25ffd"
1063
- }
1064
- },
1065
- "866f75c147c747f5b9f3d2993440823e": {
1066
- "model_module": "@jupyter-widgets/controls",
1067
- "model_name": "HTMLModel",
1068
- "model_module_version": "1.5.0",
1069
- "state": {
1070
- "_dom_classes": [],
1071
- "_model_module": "@jupyter-widgets/controls",
1072
- "_model_module_version": "1.5.0",
1073
- "_model_name": "HTMLModel",
1074
- "_view_count": null,
1075
- "_view_module": "@jupyter-widgets/controls",
1076
- "_view_module_version": "1.5.0",
1077
- "_view_name": "HTMLView",
1078
- "description": "",
1079
- "description_tooltip": null,
1080
- "layout": "IPY_MODEL_dd0a347823244e3a8dbd2f8005f378ac",
1081
- "placeholder": "​",
1082
- "style": "IPY_MODEL_84b3abe67c4745fab0d67831a7a3c3d7",
1083
- "value": "tokenizer.json: 100%"
1084
- }
1085
- },
1086
- "fe4842b3b96a4c1db17abe9a90e026c3": {
1087
- "model_module": "@jupyter-widgets/controls",
1088
- "model_name": "FloatProgressModel",
1089
- "model_module_version": "1.5.0",
1090
- "state": {
1091
- "_dom_classes": [],
1092
- "_model_module": "@jupyter-widgets/controls",
1093
- "_model_module_version": "1.5.0",
1094
- "_model_name": "FloatProgressModel",
1095
- "_view_count": null,
1096
- "_view_module": "@jupyter-widgets/controls",
1097
- "_view_module_version": "1.5.0",
1098
- "_view_name": "ProgressView",
1099
- "bar_style": "success",
1100
- "description": "",
1101
- "description_tooltip": null,
1102
- "layout": "IPY_MODEL_e4bf0e2ea3904cc68b5dcdabf38169b4",
1103
- "max": 2224003,
1104
- "min": 0,
1105
- "orientation": "horizontal",
1106
- "style": "IPY_MODEL_8f338304e7e44b5f90c156dddc1a129b",
1107
- "value": 2224003
1108
- }
1109
- },
1110
- "69561cfd572c4e0fbc107e25a1e50654": {
1111
- "model_module": "@jupyter-widgets/controls",
1112
- "model_name": "HTMLModel",
1113
- "model_module_version": "1.5.0",
1114
- "state": {
1115
- "_dom_classes": [],
1116
- "_model_module": "@jupyter-widgets/controls",
1117
- "_model_module_version": "1.5.0",
1118
- "_model_name": "HTMLModel",
1119
- "_view_count": null,
1120
- "_view_module": "@jupyter-widgets/controls",
1121
- "_view_module_version": "1.5.0",
1122
- "_view_name": "HTMLView",
1123
- "description": "",
1124
- "description_tooltip": null,
1125
- "layout": "IPY_MODEL_c314dafdd3e1472789af4307ed170085",
1126
- "placeholder": "​",
1127
- "style": "IPY_MODEL_3311af848aac451e8e9b21fae5e89422",
1128
- "value": " 2.22M/2.22M [00:00<00:00, 16.3MB/s]"
1129
- }
1130
- },
1131
- "00bc783a0f344176af0a760569c25ffd": {
1132
- "model_module": "@jupyter-widgets/base",
1133
- "model_name": "LayoutModel",
1134
- "model_module_version": "1.2.0",
1135
- "state": {
1136
- "_model_module": "@jupyter-widgets/base",
1137
- "_model_module_version": "1.2.0",
1138
- "_model_name": "LayoutModel",
1139
- "_view_count": null,
1140
- "_view_module": "@jupyter-widgets/base",
1141
- "_view_module_version": "1.2.0",
1142
- "_view_name": "LayoutView",
1143
- "align_content": null,
1144
- "align_items": null,
1145
- "align_self": null,
1146
- "border": null,
1147
- "bottom": null,
1148
- "display": null,
1149
- "flex": null,
1150
- "flex_flow": null,
1151
- "grid_area": null,
1152
- "grid_auto_columns": null,
1153
- "grid_auto_flow": null,
1154
- "grid_auto_rows": null,
1155
- "grid_column": null,
1156
- "grid_gap": null,
1157
- "grid_row": null,
1158
- "grid_template_areas": null,
1159
- "grid_template_columns": null,
1160
- "grid_template_rows": null,
1161
- "height": null,
1162
- "justify_content": null,
1163
- "justify_items": null,
1164
- "left": null,
1165
- "margin": null,
1166
- "max_height": null,
1167
- "max_width": null,
1168
- "min_height": null,
1169
- "min_width": null,
1170
- "object_fit": null,
1171
- "object_position": null,
1172
- "order": null,
1173
- "overflow": null,
1174
- "overflow_x": null,
1175
- "overflow_y": null,
1176
- "padding": null,
1177
- "right": null,
1178
- "top": null,
1179
- "visibility": null,
1180
- "width": null
1181
- }
1182
- },
1183
- "dd0a347823244e3a8dbd2f8005f378ac": {
1184
- "model_module": "@jupyter-widgets/base",
1185
- "model_name": "LayoutModel",
1186
- "model_module_version": "1.2.0",
1187
- "state": {
1188
- "_model_module": "@jupyter-widgets/base",
1189
- "_model_module_version": "1.2.0",
1190
- "_model_name": "LayoutModel",
1191
- "_view_count": null,
1192
- "_view_module": "@jupyter-widgets/base",
1193
- "_view_module_version": "1.2.0",
1194
- "_view_name": "LayoutView",
1195
- "align_content": null,
1196
- "align_items": null,
1197
- "align_self": null,
1198
- "border": null,
1199
- "bottom": null,
1200
- "display": null,
1201
- "flex": null,
1202
- "flex_flow": null,
1203
- "grid_area": null,
1204
- "grid_auto_columns": null,
1205
- "grid_auto_flow": null,
1206
- "grid_auto_rows": null,
1207
- "grid_column": null,
1208
- "grid_gap": null,
1209
- "grid_row": null,
1210
- "grid_template_areas": null,
1211
- "grid_template_columns": null,
1212
- "grid_template_rows": null,
1213
- "height": null,
1214
- "justify_content": null,
1215
- "justify_items": null,
1216
- "left": null,
1217
- "margin": null,
1218
- "max_height": null,
1219
- "max_width": null,
1220
- "min_height": null,
1221
- "min_width": null,
1222
- "object_fit": null,
1223
- "object_position": null,
1224
- "order": null,
1225
- "overflow": null,
1226
- "overflow_x": null,
1227
- "overflow_y": null,
1228
- "padding": null,
1229
- "right": null,
1230
- "top": null,
1231
- "visibility": null,
1232
- "width": null
1233
- }
1234
- },
1235
- "84b3abe67c4745fab0d67831a7a3c3d7": {
1236
- "model_module": "@jupyter-widgets/controls",
1237
- "model_name": "DescriptionStyleModel",
1238
- "model_module_version": "1.5.0",
1239
- "state": {
1240
- "_model_module": "@jupyter-widgets/controls",
1241
- "_model_module_version": "1.5.0",
1242
- "_model_name": "DescriptionStyleModel",
1243
- "_view_count": null,
1244
- "_view_module": "@jupyter-widgets/base",
1245
- "_view_module_version": "1.2.0",
1246
- "_view_name": "StyleView",
1247
- "description_width": ""
1248
- }
1249
- },
1250
- "e4bf0e2ea3904cc68b5dcdabf38169b4": {
1251
- "model_module": "@jupyter-widgets/base",
1252
- "model_name": "LayoutModel",
1253
- "model_module_version": "1.2.0",
1254
- "state": {
1255
- "_model_module": "@jupyter-widgets/base",
1256
- "_model_module_version": "1.2.0",
1257
- "_model_name": "LayoutModel",
1258
- "_view_count": null,
1259
- "_view_module": "@jupyter-widgets/base",
1260
- "_view_module_version": "1.2.0",
1261
- "_view_name": "LayoutView",
1262
- "align_content": null,
1263
- "align_items": null,
1264
- "align_self": null,
1265
- "border": null,
1266
- "bottom": null,
1267
- "display": null,
1268
- "flex": null,
1269
- "flex_flow": null,
1270
- "grid_area": null,
1271
- "grid_auto_columns": null,
1272
- "grid_auto_flow": null,
1273
- "grid_auto_rows": null,
1274
- "grid_column": null,
1275
- "grid_gap": null,
1276
- "grid_row": null,
1277
- "grid_template_areas": null,
1278
- "grid_template_columns": null,
1279
- "grid_template_rows": null,
1280
- "height": null,
1281
- "justify_content": null,
1282
- "justify_items": null,
1283
- "left": null,
1284
- "margin": null,
1285
- "max_height": null,
1286
- "max_width": null,
1287
- "min_height": null,
1288
- "min_width": null,
1289
- "object_fit": null,
1290
- "object_position": null,
1291
- "order": null,
1292
- "overflow": null,
1293
- "overflow_x": null,
1294
- "overflow_y": null,
1295
- "padding": null,
1296
- "right": null,
1297
- "top": null,
1298
- "visibility": null,
1299
- "width": null
1300
- }
1301
- },
1302
- "8f338304e7e44b5f90c156dddc1a129b": {
1303
- "model_module": "@jupyter-widgets/controls",
1304
- "model_name": "ProgressStyleModel",
1305
- "model_module_version": "1.5.0",
1306
- "state": {
1307
- "_model_module": "@jupyter-widgets/controls",
1308
- "_model_module_version": "1.5.0",
1309
- "_model_name": "ProgressStyleModel",
1310
- "_view_count": null,
1311
- "_view_module": "@jupyter-widgets/base",
1312
- "_view_module_version": "1.2.0",
1313
- "_view_name": "StyleView",
1314
- "bar_color": null,
1315
- "description_width": ""
1316
- }
1317
- },
1318
- "c314dafdd3e1472789af4307ed170085": {
1319
- "model_module": "@jupyter-widgets/base",
1320
- "model_name": "LayoutModel",
1321
- "model_module_version": "1.2.0",
1322
- "state": {
1323
- "_model_module": "@jupyter-widgets/base",
1324
- "_model_module_version": "1.2.0",
1325
- "_model_name": "LayoutModel",
1326
- "_view_count": null,
1327
- "_view_module": "@jupyter-widgets/base",
1328
- "_view_module_version": "1.2.0",
1329
- "_view_name": "LayoutView",
1330
- "align_content": null,
1331
- "align_items": null,
1332
- "align_self": null,
1333
- "border": null,
1334
- "bottom": null,
1335
- "display": null,
1336
- "flex": null,
1337
- "flex_flow": null,
1338
- "grid_area": null,
1339
- "grid_auto_columns": null,
1340
- "grid_auto_flow": null,
1341
- "grid_auto_rows": null,
1342
- "grid_column": null,
1343
- "grid_gap": null,
1344
- "grid_row": null,
1345
- "grid_template_areas": null,
1346
- "grid_template_columns": null,
1347
- "grid_template_rows": null,
1348
- "height": null,
1349
- "justify_content": null,
1350
- "justify_items": null,
1351
- "left": null,
1352
- "margin": null,
1353
- "max_height": null,
1354
- "max_width": null,
1355
- "min_height": null,
1356
- "min_width": null,
1357
- "object_fit": null,
1358
- "object_position": null,
1359
- "order": null,
1360
- "overflow": null,
1361
- "overflow_x": null,
1362
- "overflow_y": null,
1363
- "padding": null,
1364
- "right": null,
1365
- "top": null,
1366
- "visibility": null,
1367
- "width": null
1368
- }
1369
- },
1370
- "3311af848aac451e8e9b21fae5e89422": {
1371
- "model_module": "@jupyter-widgets/controls",
1372
- "model_name": "DescriptionStyleModel",
1373
- "model_module_version": "1.5.0",
1374
- "state": {
1375
- "_model_module": "@jupyter-widgets/controls",
1376
- "_model_module_version": "1.5.0",
1377
- "_model_name": "DescriptionStyleModel",
1378
- "_view_count": null,
1379
- "_view_module": "@jupyter-widgets/base",
1380
- "_view_module_version": "1.2.0",
1381
- "_view_name": "StyleView",
1382
- "description_width": ""
1383
- }
1384
- },
1385
- "f5dbb938e5f542ef8f2a4a92c98b6ed7": {
1386
- "model_module": "@jupyter-widgets/controls",
1387
- "model_name": "HBoxModel",
1388
- "model_module_version": "1.5.0",
1389
- "state": {
1390
- "_dom_classes": [],
1391
- "_model_module": "@jupyter-widgets/controls",
1392
- "_model_module_version": "1.5.0",
1393
- "_model_name": "HBoxModel",
1394
- "_view_count": null,
1395
- "_view_module": "@jupyter-widgets/controls",
1396
- "_view_module_version": "1.5.0",
1397
- "_view_name": "HBoxView",
1398
- "box_style": "",
1399
- "children": [
1400
- "IPY_MODEL_ee0d678d5b694781a419c4277306204b",
1401
- "IPY_MODEL_c2d90be349784a16975a07e8aace7567",
1402
- "IPY_MODEL_a7e7ba1a8e954d71a27210b91456c760"
1403
- ],
1404
- "layout": "IPY_MODEL_3b341c3a37414a10832b1dc9fad77298"
1405
- }
1406
- },
1407
- "ee0d678d5b694781a419c4277306204b": {
1408
- "model_module": "@jupyter-widgets/controls",
1409
- "model_name": "HTMLModel",
1410
- "model_module_version": "1.5.0",
1411
- "state": {
1412
- "_dom_classes": [],
1413
- "_model_module": "@jupyter-widgets/controls",
1414
- "_model_module_version": "1.5.0",
1415
- "_model_name": "HTMLModel",
1416
- "_view_count": null,
1417
- "_view_module": "@jupyter-widgets/controls",
1418
- "_view_module_version": "1.5.0",
1419
- "_view_name": "HTMLView",
1420
- "description": "",
1421
- "description_tooltip": null,
1422
- "layout": "IPY_MODEL_2790e327bfd94b66b1ef411489f21d6f",
1423
- "placeholder": "​",
1424
- "style": "IPY_MODEL_88a5e37f2eb24ded9e3ec4a7835c34c4",
1425
- "value": "special_tokens_map.json: 100%"
1426
- }
1427
- },
1428
- "c2d90be349784a16975a07e8aace7567": {
1429
- "model_module": "@jupyter-widgets/controls",
1430
- "model_name": "FloatProgressModel",
1431
- "model_module_version": "1.5.0",
1432
- "state": {
1433
- "_dom_classes": [],
1434
- "_model_module": "@jupyter-widgets/controls",
1435
- "_model_module_version": "1.5.0",
1436
- "_model_name": "FloatProgressModel",
1437
- "_view_count": null,
1438
- "_view_module": "@jupyter-widgets/controls",
1439
- "_view_module_version": "1.5.0",
1440
- "_view_name": "ProgressView",
1441
- "bar_style": "success",
1442
- "description": "",
1443
- "description_tooltip": null,
1444
- "layout": "IPY_MODEL_b541e9cead9d43179b72d3f966d156bc",
1445
- "max": 389,
1446
- "min": 0,
1447
- "orientation": "horizontal",
1448
- "style": "IPY_MODEL_9d8b4e12f6fa478eba0624457d8b7804",
1449
- "value": 389
1450
- }
1451
- },
1452
- "a7e7ba1a8e954d71a27210b91456c760": {
1453
- "model_module": "@jupyter-widgets/controls",
1454
- "model_name": "HTMLModel",
1455
- "model_module_version": "1.5.0",
1456
- "state": {
1457
- "_dom_classes": [],
1458
- "_model_module": "@jupyter-widgets/controls",
1459
- "_model_module_version": "1.5.0",
1460
- "_model_name": "HTMLModel",
1461
- "_view_count": null,
1462
- "_view_module": "@jupyter-widgets/controls",
1463
- "_view_module_version": "1.5.0",
1464
- "_view_name": "HTMLView",
1465
- "description": "",
1466
- "description_tooltip": null,
1467
- "layout": "IPY_MODEL_21311ed9e97e42eab18d6eb58497f125",
1468
- "placeholder": "​",
1469
- "style": "IPY_MODEL_65c49dcd741b4101a349752fb9b2db49",
1470
- "value": " 389/389 [00:00<00:00, 5.78kB/s]"
1471
- }
1472
- },
1473
- "3b341c3a37414a10832b1dc9fad77298": {
1474
- "model_module": "@jupyter-widgets/base",
1475
- "model_name": "LayoutModel",
1476
- "model_module_version": "1.2.0",
1477
- "state": {
1478
- "_model_module": "@jupyter-widgets/base",
1479
- "_model_module_version": "1.2.0",
1480
- "_model_name": "LayoutModel",
1481
- "_view_count": null,
1482
- "_view_module": "@jupyter-widgets/base",
1483
- "_view_module_version": "1.2.0",
1484
- "_view_name": "LayoutView",
1485
- "align_content": null,
1486
- "align_items": null,
1487
- "align_self": null,
1488
- "border": null,
1489
- "bottom": null,
1490
- "display": null,
1491
- "flex": null,
1492
- "flex_flow": null,
1493
- "grid_area": null,
1494
- "grid_auto_columns": null,
1495
- "grid_auto_flow": null,
1496
- "grid_auto_rows": null,
1497
- "grid_column": null,
1498
- "grid_gap": null,
1499
- "grid_row": null,
1500
- "grid_template_areas": null,
1501
- "grid_template_columns": null,
1502
- "grid_template_rows": null,
1503
- "height": null,
1504
- "justify_content": null,
1505
- "justify_items": null,
1506
- "left": null,
1507
- "margin": null,
1508
- "max_height": null,
1509
- "max_width": null,
1510
- "min_height": null,
1511
- "min_width": null,
1512
- "object_fit": null,
1513
- "object_position": null,
1514
- "order": null,
1515
- "overflow": null,
1516
- "overflow_x": null,
1517
- "overflow_y": null,
1518
- "padding": null,
1519
- "right": null,
1520
- "top": null,
1521
- "visibility": null,
1522
- "width": null
1523
- }
1524
- },
1525
- "2790e327bfd94b66b1ef411489f21d6f": {
1526
- "model_module": "@jupyter-widgets/base",
1527
- "model_name": "LayoutModel",
1528
- "model_module_version": "1.2.0",
1529
- "state": {
1530
- "_model_module": "@jupyter-widgets/base",
1531
- "_model_module_version": "1.2.0",
1532
- "_model_name": "LayoutModel",
1533
- "_view_count": null,
1534
- "_view_module": "@jupyter-widgets/base",
1535
- "_view_module_version": "1.2.0",
1536
- "_view_name": "LayoutView",
1537
- "align_content": null,
1538
- "align_items": null,
1539
- "align_self": null,
1540
- "border": null,
1541
- "bottom": null,
1542
- "display": null,
1543
- "flex": null,
1544
- "flex_flow": null,
1545
- "grid_area": null,
1546
- "grid_auto_columns": null,
1547
- "grid_auto_flow": null,
1548
- "grid_auto_rows": null,
1549
- "grid_column": null,
1550
- "grid_gap": null,
1551
- "grid_row": null,
1552
- "grid_template_areas": null,
1553
- "grid_template_columns": null,
1554
- "grid_template_rows": null,
1555
- "height": null,
1556
- "justify_content": null,
1557
- "justify_items": null,
1558
- "left": null,
1559
- "margin": null,
1560
- "max_height": null,
1561
- "max_width": null,
1562
- "min_height": null,
1563
- "min_width": null,
1564
- "object_fit": null,
1565
- "object_position": null,
1566
- "order": null,
1567
- "overflow": null,
1568
- "overflow_x": null,
1569
- "overflow_y": null,
1570
- "padding": null,
1571
- "right": null,
1572
- "top": null,
1573
- "visibility": null,
1574
- "width": null
1575
- }
1576
- },
1577
- "88a5e37f2eb24ded9e3ec4a7835c34c4": {
1578
- "model_module": "@jupyter-widgets/controls",
1579
- "model_name": "DescriptionStyleModel",
1580
- "model_module_version": "1.5.0",
1581
- "state": {
1582
- "_model_module": "@jupyter-widgets/controls",
1583
- "_model_module_version": "1.5.0",
1584
- "_model_name": "DescriptionStyleModel",
1585
- "_view_count": null,
1586
- "_view_module": "@jupyter-widgets/base",
1587
- "_view_module_version": "1.2.0",
1588
- "_view_name": "StyleView",
1589
- "description_width": ""
1590
- }
1591
- },
1592
- "b541e9cead9d43179b72d3f966d156bc": {
1593
- "model_module": "@jupyter-widgets/base",
1594
- "model_name": "LayoutModel",
1595
- "model_module_version": "1.2.0",
1596
- "state": {
1597
- "_model_module": "@jupyter-widgets/base",
1598
- "_model_module_version": "1.2.0",
1599
- "_model_name": "LayoutModel",
1600
- "_view_count": null,
1601
- "_view_module": "@jupyter-widgets/base",
1602
- "_view_module_version": "1.2.0",
1603
- "_view_name": "LayoutView",
1604
- "align_content": null,
1605
- "align_items": null,
1606
- "align_self": null,
1607
- "border": null,
1608
- "bottom": null,
1609
- "display": null,
1610
- "flex": null,
1611
- "flex_flow": null,
1612
- "grid_area": null,
1613
- "grid_auto_columns": null,
1614
- "grid_auto_flow": null,
1615
- "grid_auto_rows": null,
1616
- "grid_column": null,
1617
- "grid_gap": null,
1618
- "grid_row": null,
1619
- "grid_template_areas": null,
1620
- "grid_template_columns": null,
1621
- "grid_template_rows": null,
1622
- "height": null,
1623
- "justify_content": null,
1624
- "justify_items": null,
1625
- "left": null,
1626
- "margin": null,
1627
- "max_height": null,
1628
- "max_width": null,
1629
- "min_height": null,
1630
- "min_width": null,
1631
- "object_fit": null,
1632
- "object_position": null,
1633
- "order": null,
1634
- "overflow": null,
1635
- "overflow_x": null,
1636
- "overflow_y": null,
1637
- "padding": null,
1638
- "right": null,
1639
- "top": null,
1640
- "visibility": null,
1641
- "width": null
1642
- }
1643
- },
1644
- "9d8b4e12f6fa478eba0624457d8b7804": {
1645
- "model_module": "@jupyter-widgets/controls",
1646
- "model_name": "ProgressStyleModel",
1647
- "model_module_version": "1.5.0",
1648
- "state": {
1649
- "_model_module": "@jupyter-widgets/controls",
1650
- "_model_module_version": "1.5.0",
1651
- "_model_name": "ProgressStyleModel",
1652
- "_view_count": null,
1653
- "_view_module": "@jupyter-widgets/base",
1654
- "_view_module_version": "1.2.0",
1655
- "_view_name": "StyleView",
1656
- "bar_color": null,
1657
- "description_width": ""
1658
- }
1659
- },
1660
- "21311ed9e97e42eab18d6eb58497f125": {
1661
- "model_module": "@jupyter-widgets/base",
1662
- "model_name": "LayoutModel",
1663
- "model_module_version": "1.2.0",
1664
- "state": {
1665
- "_model_module": "@jupyter-widgets/base",
1666
- "_model_module_version": "1.2.0",
1667
- "_model_name": "LayoutModel",
1668
- "_view_count": null,
1669
- "_view_module": "@jupyter-widgets/base",
1670
- "_view_module_version": "1.2.0",
1671
- "_view_name": "LayoutView",
1672
- "align_content": null,
1673
- "align_items": null,
1674
- "align_self": null,
1675
- "border": null,
1676
- "bottom": null,
1677
- "display": null,
1678
- "flex": null,
1679
- "flex_flow": null,
1680
- "grid_area": null,
1681
- "grid_auto_columns": null,
1682
- "grid_auto_flow": null,
1683
- "grid_auto_rows": null,
1684
- "grid_column": null,
1685
- "grid_gap": null,
1686
- "grid_row": null,
1687
- "grid_template_areas": null,
1688
- "grid_template_columns": null,
1689
- "grid_template_rows": null,
1690
- "height": null,
1691
- "justify_content": null,
1692
- "justify_items": null,
1693
- "left": null,
1694
- "margin": null,
1695
- "max_height": null,
1696
- "max_width": null,
1697
- "min_height": null,
1698
- "min_width": null,
1699
- "object_fit": null,
1700
- "object_position": null,
1701
- "order": null,
1702
- "overflow": null,
1703
- "overflow_x": null,
1704
- "overflow_y": null,
1705
- "padding": null,
1706
- "right": null,
1707
- "top": null,
1708
- "visibility": null,
1709
- "width": null
1710
- }
1711
- },
1712
- "65c49dcd741b4101a349752fb9b2db49": {
1713
- "model_module": "@jupyter-widgets/controls",
1714
- "model_name": "DescriptionStyleModel",
1715
- "model_module_version": "1.5.0",
1716
- "state": {
1717
- "_model_module": "@jupyter-widgets/controls",
1718
- "_model_module_version": "1.5.0",
1719
- "_model_name": "DescriptionStyleModel",
1720
- "_view_count": null,
1721
- "_view_module": "@jupyter-widgets/base",
1722
- "_view_module_version": "1.2.0",
1723
- "_view_name": "StyleView",
1724
- "description_width": ""
1725
- }
1726
- }
1727
- }
1728
  }
1729
  },
1730
  "cells": [
@@ -1787,34 +73,16 @@
1787
  "mix_method = \"None\""
1788
  ],
1789
  "metadata": {
1790
- "id": "Ch9puvwKH1s3",
1791
- "colab": {
1792
- "base_uri": "https://localhost:8080/"
1793
- },
1794
- "outputId": "46175dd6-a875-4208-a128-fddab8752406"
1795
  },
1796
- "execution_count": 1,
1797
- "outputs": [
1798
- {
1799
- "output_type": "stream",
1800
- "name": "stdout",
1801
- "text": [
1802
- "Cloning into 'sd_tokens'...\n",
1803
- "remote: Enumerating objects: 7, done.\u001b[K\n",
1804
- "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
1805
- "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
1806
- "remote: Total 7 (delta 0), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
1807
- "Unpacking objects: 100% (7/7), 305.93 KiB | 5.67 MiB/s, done.\n",
1808
- "/content/sd_tokens\n"
1809
- ]
1810
- }
1811
- ]
1812
  },
1813
  {
1814
  "cell_type": "code",
1815
  "source": [
1816
- "print(vocab[8922]) #the vocab item for ID 8922\n",
1817
- "print(token[8922].shape) #dimension of the token"
1818
  ],
1819
  "metadata": {
1820
  "id": "S_Yh9gH_OUA1"
@@ -1827,7 +95,7 @@
1827
  "source": [
1828
  "Get the IDs from a prompt text.\n",
1829
  "\n",
1830
- "The prompt will be enclosed with the <|start-of-text|> and <|end-of-text|> tokens"
1831
  ],
1832
  "metadata": {
1833
  "id": "f1-jS7YJApiO"
@@ -1838,164 +106,39 @@
1838
  "source": [
1839
  "from transformers import AutoTokenizer\n",
1840
  "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
 
1841
  "prompt= \"banana\" # @param {type:'string'}\n",
 
1842
  "tokenizer_output = tokenizer(text = prompt)\n",
1843
  "input_ids = tokenizer_output['input_ids']\n",
1844
  "print(input_ids)\n",
1845
  "id_A = input_ids[1]\n",
1846
  "A = token[id_A]\n",
1847
- "_A = LA.vector_norm(A, ord=2)"
 
 
 
 
 
 
 
 
 
 
 
 
 
1848
  ],
1849
  "metadata": {
1850
- "id": "RPdkYzT2_X85",
1851
- "colab": {
1852
- "base_uri": "https://localhost:8080/",
1853
- "height": 194,
1854
- "referenced_widgets": [
1855
- "7b073d9c166a45539c651ecd0b27285c",
1856
- "ffba5f49ba214984b8329b4e06e40dcf",
1857
- "e69b98b68527488cbb7c1ee5f6011a92",
1858
- "983145e29ab84636b1e266c73eeb6889",
1859
- "278c2c021faf4f2696c03070fc3511bf",
1860
- "3e9df4ac699843df8ae361aba1337b06",
1861
- "8e9cc6914fc14e56945716902fb11fe9",
1862
- "0ca945d6f8a5457d9706c211869983c8",
1863
- "69cf2c7a4ecf4c3188586c86c7056518",
1864
- "1d60ac5b02de4cc7b0d51ed57165ff64",
1865
- "e34074fc893d47b8bf75a8089cd8df37",
1866
- "26cb34af09544ea1b17526891fd319e3",
1867
- "71627d67c8ec4993a8b1e4a93d53e8db",
1868
- "ec11e5b722dd4d69bcd758775edd61b6",
1869
- "63f760d16a1249b6a80f7a73989299ac",
1870
- "ba3dae52e6bc4a48945ebde7464a5277",
1871
- "e914f0b293b4410a82e02f990b8a9a1d",
1872
- "7e18f5165da84f60bea9d9a1819e17df",
1873
- "ba1b3dc7d77b4fee9addea98f60277bc",
1874
- "b40d206858bd46e19ef99aa1de302060",
1875
- "92113b912eea41b4b2bfadd3f65cd835",
1876
- "bfca51dacc6a4762810309006fc6eca5",
1877
- "a7a66681148748ae94788ee276925bec",
1878
- "438a2db97b8649cfbd5b6d87aeb4b13a",
1879
- "8608253bb3cc4e93a379b05648e1e35c",
1880
- "aa84ccf805b3476ea155e64ed31edef6",
1881
- "851b1acc7d524ec1aabb04012c469432",
1882
- "2277a0b29cd841a2a1ad5526f39c28a6",
1883
- "5f7a4ea2308d4cb2b6e4a0f2f925e2b1",
1884
- "dc23a6ac81894fe484247273be9d6a83",
1885
- "d37f42e8924a4b0fafd2e407b1db865b",
1886
- "03fa9da8e5084d688bc7b0f4d8ba3fb5",
1887
- "613fa346a05441828356976ea941a7c7",
1888
- "f580a1015d334b398d024357adf898cf",
1889
- "866f75c147c747f5b9f3d2993440823e",
1890
- "fe4842b3b96a4c1db17abe9a90e026c3",
1891
- "69561cfd572c4e0fbc107e25a1e50654",
1892
- "00bc783a0f344176af0a760569c25ffd",
1893
- "dd0a347823244e3a8dbd2f8005f378ac",
1894
- "84b3abe67c4745fab0d67831a7a3c3d7",
1895
- "e4bf0e2ea3904cc68b5dcdabf38169b4",
1896
- "8f338304e7e44b5f90c156dddc1a129b",
1897
- "c314dafdd3e1472789af4307ed170085",
1898
- "3311af848aac451e8e9b21fae5e89422",
1899
- "f5dbb938e5f542ef8f2a4a92c98b6ed7",
1900
- "ee0d678d5b694781a419c4277306204b",
1901
- "c2d90be349784a16975a07e8aace7567",
1902
- "a7e7ba1a8e954d71a27210b91456c760",
1903
- "3b341c3a37414a10832b1dc9fad77298",
1904
- "2790e327bfd94b66b1ef411489f21d6f",
1905
- "88a5e37f2eb24ded9e3ec4a7835c34c4",
1906
- "b541e9cead9d43179b72d3f966d156bc",
1907
- "9d8b4e12f6fa478eba0624457d8b7804",
1908
- "21311ed9e97e42eab18d6eb58497f125",
1909
- "65c49dcd741b4101a349752fb9b2db49"
1910
- ]
1911
- },
1912
- "outputId": "4c2d8236-8af4-4d96-e4f0-c6a4da2d0da2"
1913
  },
1914
- "execution_count": 2,
1915
- "outputs": [
1916
- {
1917
- "output_type": "display_data",
1918
- "data": {
1919
- "text/plain": [
1920
- "tokenizer_config.json: 0%| | 0.00/905 [00:00<?, ?B/s]"
1921
- ],
1922
- "application/vnd.jupyter.widget-view+json": {
1923
- "version_major": 2,
1924
- "version_minor": 0,
1925
- "model_id": "7b073d9c166a45539c651ecd0b27285c"
1926
- }
1927
- },
1928
- "metadata": {}
1929
- },
1930
- {
1931
- "output_type": "display_data",
1932
- "data": {
1933
- "text/plain": [
1934
- "vocab.json: 0%| | 0.00/961k [00:00<?, ?B/s]"
1935
- ],
1936
- "application/vnd.jupyter.widget-view+json": {
1937
- "version_major": 2,
1938
- "version_minor": 0,
1939
- "model_id": "26cb34af09544ea1b17526891fd319e3"
1940
- }
1941
- },
1942
- "metadata": {}
1943
- },
1944
- {
1945
- "output_type": "display_data",
1946
- "data": {
1947
- "text/plain": [
1948
- "merges.txt: 0%| | 0.00/525k [00:00<?, ?B/s]"
1949
- ],
1950
- "application/vnd.jupyter.widget-view+json": {
1951
- "version_major": 2,
1952
- "version_minor": 0,
1953
- "model_id": "a7a66681148748ae94788ee276925bec"
1954
- }
1955
- },
1956
- "metadata": {}
1957
- },
1958
- {
1959
- "output_type": "display_data",
1960
- "data": {
1961
- "text/plain": [
1962
- "tokenizer.json: 0%| | 0.00/2.22M [00:00<?, ?B/s]"
1963
- ],
1964
- "application/vnd.jupyter.widget-view+json": {
1965
- "version_major": 2,
1966
- "version_minor": 0,
1967
- "model_id": "f580a1015d334b398d024357adf898cf"
1968
- }
1969
- },
1970
- "metadata": {}
1971
- },
1972
- {
1973
- "output_type": "display_data",
1974
- "data": {
1975
- "text/plain": [
1976
- "special_tokens_map.json: 0%| | 0.00/389 [00:00<?, ?B/s]"
1977
- ],
1978
- "application/vnd.jupyter.widget-view+json": {
1979
- "version_major": 2,
1980
- "version_minor": 0,
1981
- "model_id": "f5dbb938e5f542ef8f2a4a92c98b6ed7"
1982
- }
1983
- },
1984
- "metadata": {}
1985
- },
1986
- {
1987
- "output_type": "stream",
1988
- "name": "stdout",
1989
- "text": [
1990
- "[49406, 8922, 49407]\n"
1991
- ]
1992
- }
1993
- ]
1994
  },
1995
  {
1996
  "cell_type": "markdown",
1997
  "source": [
1998
- "OPTIONAL : Add/subtract + normalize above result with another token"
1999
  ],
2000
  "metadata": {
2001
  "id": "JKnz0aLFVGXc"
@@ -2008,7 +151,11 @@
2008
  "mix_method = \"None\" # @param [\"None\" , \"Average\", \"Subtract\"] {allow-input: true}\n",
2009
  "w = 0.5 # @param {type:\"slider\", min:0, max:1, step:0.01}\n",
2010
  "\n",
2011
- "\n",
 
 
 
 
2012
  "\n",
2013
  "tokenizer_output = tokenizer(text = mix_with)\n",
2014
  "input_ids = tokenizer_output['input_ids']\n",
@@ -2016,22 +163,35 @@
2016
  "C = token[id_C]\n",
2017
  "_C = LA.vector_norm(C, ord=2)\n",
2018
  "\n",
 
 
 
 
 
 
 
 
 
 
 
2019
  "if (mix_method == \"Average\"):\n",
2020
  " A = w*A + (1-w)*C\n",
2021
  " _A = LA.vector_norm(A, ord=2)\n",
 
2022
  "\n",
2023
  "if (mix_method == \"Subtract\"):\n",
2024
  " tmp = w*A - (1-w)*C\n",
2025
  " _tmp = LA.vector_norm(tmp, ord=2)\n",
2026
  " A = tmp*((w*_A + (1-w)*_C)/_tmp)\n",
2027
  " _A = LA.vector_norm(A, ord=2)\n",
 
2028
  "\n",
2029
  "\n"
2030
  ],
2031
  "metadata": {
2032
  "id": "oXbNSRSKPgRr"
2033
  },
2034
- "execution_count": 5,
2035
  "outputs": []
2036
  },
2037
  {
@@ -2056,32 +216,29 @@
2056
  " result = absolute_value(result.item())\n",
2057
  " dots[index] = result\n",
2058
  "\n",
 
 
 
 
 
 
 
 
 
2059
  "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
2060
  "#----#\n",
2061
  "if (mix_method == \"Average\"):\n",
2062
- " print(f'Calculated all cosine-similarities between the average of token {vocab[id_A]} and {vocab[id_C]} with ID = {id_A} and mixed ID = {id_C} as a 1x{sorted.shape[0]} tensor')\n",
2063
  "if (mix_method == \"Subtract\"):\n",
2064
- " print(f'Calculated all cosine-similarities between the subtract of token {vocab[id_A]} and {vocab[id_C]} with ID = {id_A} and mixed ID = {id_C} as a 1x{sorted.shape[0]} tensor')\n",
2065
  "if (mix_method == \"None\"):\n",
2066
- " print(f'Calculated all cosine-similarities between the token {vocab[id_A]} with ID = {id_A} the rest of the {NUM_TOKENS} tokens as a 1x{sorted.shape[0]} tensor')"
2067
  ],
2068
  "metadata": {
2069
- "id": "juxsvco9B0iV",
2070
- "colab": {
2071
- "base_uri": "https://localhost:8080/"
2072
- },
2073
- "outputId": "ce920e92-5ff5-412c-98d2-5f2c284ca7b4"
2074
  },
2075
- "execution_count": 6,
2076
- "outputs": [
2077
- {
2078
- "output_type": "stream",
2079
- "name": "stdout",
2080
- "text": [
2081
- "Calculated all cosine-similarities between the token banana</w> with ID = 8922 the rest of the 49407 tokens as a 1x49407 tensor\n"
2082
- ]
2083
- }
2084
- ]
2085
  },
2086
  {
2087
  "cell_type": "markdown",
@@ -2115,321 +272,10 @@
2115
  ],
2116
  "metadata": {
2117
  "id": "YIEmLAzbHeuo",
2118
- "colab": {
2119
- "base_uri": "https://localhost:8080/"
2120
- },
2121
- "collapsed": true,
2122
- "outputId": "a03b49b8-a6ec-457b-f78e-be3cb3029eae"
2123
  },
2124
- "execution_count": 7,
2125
- "outputs": [
2126
- {
2127
- "output_type": "stream",
2128
- "name": "stdout",
2129
- "text": [
2130
- "banana</w>\n",
2131
- "similiarity = 100.0 %\n",
2132
- "--------\n",
2133
- "bananas</w>\n",
2134
- "similiarity = 38.93 %\n",
2135
- "--------\n",
2136
- "banan\n",
2137
- "similiarity = 30.8 %\n",
2138
- "--------\n",
2139
- "ðŁįĮ</w>\n",
2140
- "similiarity = 27.12 %\n",
2141
- "--------\n",
2142
- "pineapple</w>\n",
2143
- "similiarity = 19.7 %\n",
2144
- "--------\n",
2145
- "chicken</w>\n",
2146
- "similiarity = 19.24 %\n",
2147
- "--------\n",
2148
- "potassium</w>\n",
2149
- "similiarity = 19.21 %\n",
2150
- "--------\n",
2151
- "sausage</w>\n",
2152
- "similiarity = 19.07 %\n",
2153
- "--------\n",
2154
- "lemon</w>\n",
2155
- "similiarity = 18.82 %\n",
2156
- "--------\n",
2157
- "orange</w>\n",
2158
- "similiarity = 18.42 %\n",
2159
- "--------\n",
2160
- "peanut</w>\n",
2161
- "similiarity = 17.84 %\n",
2162
- "--------\n",
2163
- "parachute</w>\n",
2164
- "similiarity = 17.19 %\n",
2165
- "--------\n",
2166
- "duck\n",
2167
- "similiarity = 16.8 %\n",
2168
- "--------\n",
2169
- "yellow</w>\n",
2170
- "similiarity = 16.21 %\n",
2171
- "--------\n",
2172
- "grape</w>\n",
2173
- "similiarity = 16.19 %\n",
2174
- "--------\n",
2175
- "kangaroo</w>\n",
2176
- "similiarity = 16.13 %\n",
2177
- "--------\n",
2178
- "apple</w>\n",
2179
- "similiarity = 16.13 %\n",
2180
- "--------\n",
2181
- "tangerine</w>\n",
2182
- "similiarity = 16.08 %\n",
2183
- "--------\n",
2184
- "giraffe</w>\n",
2185
- "similiarity = 16.04 %\n",
2186
- "--------\n",
2187
- "mango</w>\n",
2188
- "similiarity = 16.03 %\n",
2189
- "--------\n",
2190
- "rubber</w>\n",
2191
- "similiarity = 15.95 %\n",
2192
- "--------\n",
2193
- "bamboo</w>\n",
2194
- "similiarity = 15.88 %\n",
2195
- "--------\n",
2196
- "umbrella</w>\n",
2197
- "similiarity = 15.82 %\n",
2198
- "--------\n",
2199
- "nutella</w>\n",
2200
- "similiarity = 15.69 %\n",
2201
- "--------\n",
2202
- "ferrari</w>\n",
2203
- "similiarity = 15.69 %\n",
2204
- "--------\n",
2205
- "oranges</w>\n",
2206
- "similiarity = 15.65 %\n",
2207
- "--------\n",
2208
- "peanuts</w>\n",
2209
- "similiarity = 15.62 %\n",
2210
- "--------\n",
2211
- "ali</w>\n",
2212
- "similiarity = 15.49 %\n",
2213
- "--------\n",
2214
- "dae</w>\n",
2215
- "similiarity = 15.45 %\n",
2216
- "--------\n",
2217
- "apd</w>\n",
2218
- "similiarity = 15.43 %\n",
2219
- "--------\n",
2220
- "cucumber</w>\n",
2221
- "similiarity = 15.32 %\n",
2222
- "--------\n",
2223
- "potato</w>\n",
2224
- "similiarity = 15.22 %\n",
2225
- "--------\n",
2226
- "monkey</w>\n",
2227
- "similiarity = 15.2 %\n",
2228
- "--------\n",
2229
- "croissant</w>\n",
2230
- "similiarity = 15.18 %\n",
2231
- "--------\n",
2232
- "papaya</w>\n",
2233
- "similiarity = 15.17 %\n",
2234
- "--------\n",
2235
- "christmas</w>\n",
2236
- "similiarity = 15.12 %\n",
2237
- "--------\n",
2238
- "sandwich</w>\n",
2239
- "similiarity = 15.0 %\n",
2240
- "--------\n",
2241
- "rainbow</w>\n",
2242
- "similiarity = 14.98 %\n",
2243
- "--------\n",
2244
- "tomato</w>\n",
2245
- "similiarity = 14.96 %\n",
2246
- "--------\n",
2247
- "martini</w>\n",
2248
- "similiarity = 14.93 %\n",
2249
- "--------\n",
2250
- "cabaret</w>\n",
2251
- "similiarity = 14.83 %\n",
2252
- "--------\n",
2253
- "ginger</w>\n",
2254
- "similiarity = 14.82 %\n",
2255
- "--------\n",
2256
- "µ\n",
2257
- "similiarity = 14.8 %\n",
2258
- "--------\n",
2259
- "animal</w>\n",
2260
- "similiarity = 14.76 %\n",
2261
- "--------\n",
2262
- "vanilla</w>\n",
2263
- "similiarity = 14.73 %\n",
2264
- "--------\n",
2265
- "ells</w>\n",
2266
- "similiarity = 14.66 %\n",
2267
- "--------\n",
2268
- "mustache</w>\n",
2269
- "similiarity = 14.64 %\n",
2270
- "--------\n",
2271
- "lime</w>\n",
2272
- "similiarity = 14.62 %\n",
2273
- "--------\n",
2274
- "sickle</w>\n",
2275
- "similiarity = 14.6 %\n",
2276
- "--------\n",
2277
- "vista</w>\n",
2278
- "similiarity = 14.53 %\n",
2279
- "--------\n",
2280
- "coconut</w>\n",
2281
- "similiarity = 14.52 %\n",
2282
- "--------\n",
2283
- "kara</w>\n",
2284
- "similiarity = 14.46 %\n",
2285
- "--------\n",
2286
- "alligator</w>\n",
2287
- "similiarity = 14.39 %\n",
2288
- "--------\n",
2289
- "blueberry</w>\n",
2290
- "similiarity = 14.34 %\n",
2291
- "--------\n",
2292
- "ðŁĻĤ\n",
2293
- "similiarity = 14.3 %\n",
2294
- "--------\n",
2295
- "squirrel</w>\n",
2296
- "similiarity = 14.29 %\n",
2297
- "--------\n",
2298
- "atore</w>\n",
2299
- "similiarity = 14.19 %\n",
2300
- "--------\n",
2301
- "watermelon</w>\n",
2302
- "similiarity = 14.13 %\n",
2303
- "--------\n",
2304
- "nana</w>\n",
2305
- "similiarity = 14.09 %\n",
2306
- "--------\n",
2307
- "latex</w>\n",
2308
- "similiarity = 14.08 %\n",
2309
- "--------\n",
2310
- "reas\n",
2311
- "similiarity = 14.07 %\n",
2312
- "--------\n",
2313
- "agricultural</w>\n",
2314
- "similiarity = 14.02 %\n",
2315
- "--------\n",
2316
- "davis\n",
2317
- "similiarity = 14.0 %\n",
2318
- "--------\n",
2319
- "zucchini</w>\n",
2320
- "similiarity = 14.0 %\n",
2321
- "--------\n",
2322
- "saxophone</w>\n",
2323
- "similiarity = 13.93 %\n",
2324
- "--------\n",
2325
- "rough\n",
2326
- "similiarity = 13.92 %\n",
2327
- "--------\n",
2328
- "mozzarella</w>\n",
2329
- "similiarity = 13.91 %\n",
2330
- "--------\n",
2331
- "eggplant</w>\n",
2332
- "similiarity = 13.9 %\n",
2333
- "--------\n",
2334
- "pickle</w>\n",
2335
- "similiarity = 13.89 %\n",
2336
- "--------\n",
2337
- "tortilla</w>\n",
2338
- "similiarity = 13.88 %\n",
2339
- "--------\n",
2340
- "maniac</w>\n",
2341
- "similiarity = 13.84 %\n",
2342
- "--------\n",
2343
- "milk</w>\n",
2344
- "similiarity = 13.83 %\n",
2345
- "--------\n",
2346
- "ylde</w>\n",
2347
- "similiarity = 13.82 %\n",
2348
- "--------\n",
2349
- "cellphone</w>\n",
2350
- "similiarity = 13.78 %\n",
2351
- "--------\n",
2352
- "duck</w>\n",
2353
- "similiarity = 13.73 %\n",
2354
- "--------\n",
2355
- "finely</w>\n",
2356
- "similiarity = 13.72 %\n",
2357
- "--------\n",
2358
- "umbrel\n",
2359
- "similiarity = 13.71 %\n",
2360
- "--------\n",
2361
- "fanny</w>\n",
2362
- "similiarity = 13.69 %\n",
2363
- "--------\n",
2364
- "navis</w>\n",
2365
- "similiarity = 13.68 %\n",
2366
- "--------\n",
2367
- "twister</w>\n",
2368
- "similiarity = 13.67 %\n",
2369
- "--------\n",
2370
- "moustache</w>\n",
2371
- "similiarity = 13.66 %\n",
2372
- "--------\n",
2373
- "manafort</w>\n",
2374
- "similiarity = 13.66 %\n",
2375
- "--------\n",
2376
- "nod</w>\n",
2377
- "similiarity = 13.62 %\n",
2378
- "--------\n",
2379
- "dros</w>\n",
2380
- "similiarity = 13.6 %\n",
2381
- "--------\n",
2382
- "grapefruit</w>\n",
2383
- "similiarity = 13.6 %\n",
2384
- "--------\n",
2385
- "broom</w>\n",
2386
- "similiarity = 13.59 %\n",
2387
- "--------\n",
2388
- "scorpion</w>\n",
2389
- "similiarity = 13.59 %\n",
2390
- "--------\n",
2391
- "د\n",
2392
- "similiarity = 13.58 %\n",
2393
- "--------\n",
2394
- "fruit\n",
2395
- "similiarity = 13.57 %\n",
2396
- "--------\n",
2397
- "dv\n",
2398
- "similiarity = 13.55 %\n",
2399
- "--------\n",
2400
- "agan\n",
2401
- "similiarity = 13.53 %\n",
2402
- "--------\n",
2403
- "sunflower</w>\n",
2404
- "similiarity = 13.49 %\n",
2405
- "--------\n",
2406
- "banc\n",
2407
- "similiarity = 13.46 %\n",
2408
- "--------\n",
2409
- "literature</w>\n",
2410
- "similiarity = 13.45 %\n",
2411
- "--------\n",
2412
- "pelican</w>\n",
2413
- "similiarity = 13.43 %\n",
2414
- "--------\n",
2415
- "breakfast</w>\n",
2416
- "similiarity = 13.42 %\n",
2417
- "--------\n",
2418
- "pear</w>\n",
2419
- "similiarity = 13.42 %\n",
2420
- "--------\n",
2421
- "gri\n",
2422
- "similiarity = 13.42 %\n",
2423
- "--------\n",
2424
- "orange\n",
2425
- "similiarity = 13.4 %\n",
2426
- "--------\n",
2427
- "monet</w>\n",
2428
- "similiarity = 13.4 %\n",
2429
- "--------\n"
2430
- ]
2431
- }
2432
- ]
2433
  },
2434
  {
2435
  "cell_type": "markdown",
 
11
  },
12
  "language_info": {
13
  "name": "python"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  }
15
  },
16
  "cells": [
 
73
  "mix_method = \"None\""
74
  ],
75
  "metadata": {
76
+ "id": "Ch9puvwKH1s3"
 
 
 
 
77
  },
78
+ "execution_count": null,
79
+ "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  },
81
  {
82
  "cell_type": "code",
83
  "source": [
84
+ "#print(vocab[8922]) #the vocab item for ID 8922\n",
85
+ "#print(token[8922].shape) #dimension of the token"
86
  ],
87
  "metadata": {
88
  "id": "S_Yh9gH_OUA1"
 
95
  "source": [
96
  "Get the IDs from a prompt text.\n",
97
  "\n",
98
+ "The prompt will be enclosed with the <|start-of-text|> and <|end-of-text|> tokens. Leave the field empty to get a random value tensor"
99
  ],
100
  "metadata": {
101
  "id": "f1-jS7YJApiO"
 
106
  "source": [
107
  "from transformers import AutoTokenizer\n",
108
  "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
109
+ "\n",
110
  "prompt= \"banana\" # @param {type:'string'}\n",
111
+ "\n",
112
  "tokenizer_output = tokenizer(text = prompt)\n",
113
  "input_ids = tokenizer_output['input_ids']\n",
114
  "print(input_ids)\n",
115
  "id_A = input_ids[1]\n",
116
  "A = token[id_A]\n",
117
+ "_A = LA.vector_norm(A, ord=2)\n",
118
+ "\n",
119
+ "#if no imput exists we just randomize the entire thing\n",
120
+ "if (prompt == \"\"):\n",
121
+ " id_A = -1\n",
122
+ " print(\"Tokenized prompt tensor A is a random valued tensor with no ID\")\n",
123
+ " R = torch.rand(768)\n",
124
+ " _R = LA.vector_norm(R, ord=2)\n",
125
+ " A = R*(_A/_R)\n",
126
+ "\n",
127
+ "#Save a copy of the tensor A\n",
128
+ "id_P = input_ids[1]\n",
129
+ "P = token[id_A]\n",
130
+ "_P = LA.vector_norm(A, ord=2)"
131
  ],
132
  "metadata": {
133
+ "id": "RPdkYzT2_X85"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  },
135
+ "execution_count": null,
136
+ "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  },
138
  {
139
  "cell_type": "markdown",
140
  "source": [
141
+ "OPTIONAL : Add/subtract + normalize above result with another token. Leave field empty to get a random value tensor"
142
  ],
143
  "metadata": {
144
  "id": "JKnz0aLFVGXc"
 
151
  "mix_method = \"None\" # @param [\"None\" , \"Average\", \"Subtract\"] {allow-input: true}\n",
152
  "w = 0.5 # @param {type:\"slider\", min:0, max:1, step:0.01}\n",
153
  "\n",
154
+ "#prevent re-iterating A by reading from stored copy\n",
155
+ "id_A = id_P\n",
156
+ "A = P\n",
157
+ "_A = _P\n",
158
+ "#----#\n",
159
  "\n",
160
  "tokenizer_output = tokenizer(text = mix_with)\n",
161
  "input_ids = tokenizer_output['input_ids']\n",
 
163
  "C = token[id_C]\n",
164
  "_C = LA.vector_norm(C, ord=2)\n",
165
  "\n",
166
+ "#if no imput exists we just randomize the entire thing\n",
167
+ "if (mix_with == \"\"):\n",
168
+ " id_C = -1\n",
169
+ " print(\"Tokenized prompt 'mix_with' tensor C is a random valued tensor with no ID\")\n",
170
+ " R = torch.rand(768)\n",
171
+ " _R = LA.vector_norm(R, ord=2)\n",
172
+ " C = R*(_C/_R)\n",
173
+ "\n",
174
+ "if (mix_method == \"None\"):\n",
175
+ " print(\"No operation\")\n",
176
+ "\n",
177
  "if (mix_method == \"Average\"):\n",
178
  " A = w*A + (1-w)*C\n",
179
  " _A = LA.vector_norm(A, ord=2)\n",
180
+ " print(\"Tokenized prompt tensor A has been recalculated as A = w*A + (1-w)*C , where C is the tokenized prompt 'mix_with' tensor C\")\n",
181
  "\n",
182
  "if (mix_method == \"Subtract\"):\n",
183
  " tmp = w*A - (1-w)*C\n",
184
  " _tmp = LA.vector_norm(tmp, ord=2)\n",
185
  " A = tmp*((w*_A + (1-w)*_C)/_tmp)\n",
186
  " _A = LA.vector_norm(A, ord=2)\n",
187
+ " print(\"Tokenized prompt tensor A has been recalculated as A = (w*_A + (1-w)*_C) * norm(w*A - (1-w)*C) , where C is the tokenized prompt 'mix_with' tensor C\")\n",
188
  "\n",
189
  "\n"
190
  ],
191
  "metadata": {
192
  "id": "oXbNSRSKPgRr"
193
  },
194
+ "execution_count": null,
195
  "outputs": []
196
  },
197
  {
 
216
  " result = absolute_value(result.item())\n",
217
  " dots[index] = result\n",
218
  "\n",
219
+ "name_A = \"A of random type\"\n",
220
+ "if (id_A>-1):\n",
221
+ " name_A = vocab[id_A]\n",
222
+ "\n",
223
+ "name_C = \"token C of random type\"\n",
224
+ "if (id_C>-1):\n",
225
+ " name_C = vocab[id_C]\n",
226
+ "\n",
227
+ "\n",
228
  "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
229
  "#----#\n",
230
  "if (mix_method == \"Average\"):\n",
231
+ " print(f'Calculated all cosine-similarities between the average of token {name_A} and {name_C} with Id_A = {id_A} and mixed Id_C = {id_C} as a 1x{sorted.shape[0]} tensor')\n",
232
  "if (mix_method == \"Subtract\"):\n",
233
+ " print(f'Calculated all cosine-similarities between the subtract of token {name_A} and {name_C} with Id_A = {id_A} and mixed Id_C = {id_C} as a 1x{sorted.shape[0]} tensor')\n",
234
  "if (mix_method == \"None\"):\n",
235
+ " print(f'Calculated all cosine-similarities between the token {name_A} with Id_A = {id_A} with the the rest of the {NUM_TOKENS} tokens as a 1x{sorted.shape[0]} tensor')"
236
  ],
237
  "metadata": {
238
+ "id": "juxsvco9B0iV"
 
 
 
 
239
  },
240
+ "execution_count": null,
241
+ "outputs": []
 
 
 
 
 
 
 
 
242
  },
243
  {
244
  "cell_type": "markdown",
 
272
  ],
273
  "metadata": {
274
  "id": "YIEmLAzbHeuo",
275
+ "collapsed": true
 
 
 
 
276
  },
277
+ "execution_count": null,
278
+ "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  },
280
  {
281
  "cell_type": "markdown",