Hugging Face's logo

qq574599829
/
codes

TensorBoard
Model card Files
xet
Metrics Community
qq574599829 commited on
Commit
daa9440
·
1 Parent(s): 482f48c

stdtokenizer

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. OmniTokenizer/OmniTokenizer/data.py +2 -2
  2. OmniTokenizer/OmniTokenizer/modules/codebook.py +2 -2
  3. OmniTokenizer/OmniTokenizer/omnitokenizer.py +2 -2
  4. OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py +2 -2
  5. OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh +2 -2
  6. OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101_video.sh +3 -0
  7. OmniTokenizer/videoAR/data/dataset_image_video.py +2 -2
  8. OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py +1 -1
  9. OmniTokenizer/videoAR/validation_prompts_openvid.txt +2 -2
  10. STDTokenizer/.vscode/settings.json +3 -0
  11. STDTokenizer/STDtokenizer/__pycache__/train_utils.cpython-310.pyc +3 -0
  12. STDTokenizer/STDtokenizer/__pycache__/utils.cpython-310.pyc +3 -0
  13. STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml +2 -2
  14. STDTokenizer/STDtokenizer/configs/stage2/stdtokenizer_l32.yaml +3 -0
  15. STDTokenizer/STDtokenizer/data/__pycache__/bucket_sampler.cpython-310.pyc +3 -0
  16. STDTokenizer/STDtokenizer/data/__pycache__/dataset_image.cpython-310.pyc +3 -0
  17. STDTokenizer/STDtokenizer/data/__pycache__/dataset_image_video.cpython-310.pyc +3 -0
  18. STDTokenizer/STDtokenizer/data/bucket_sampler.py +3 -0
  19. STDTokenizer/STDtokenizer/data/dataset_image.py +3 -0
  20. STDTokenizer/STDtokenizer/data/dataset_image_video.py +3 -0
  21. STDTokenizer/STDtokenizer/data/dataset_video.py +3 -0
  22. STDTokenizer/STDtokenizer/modeling/__pycache__/base.cpython-310.pyc +3 -0
  23. STDTokenizer/STDtokenizer/modeling/__pycache__/stdtokenizer.cpython-310.pyc +3 -0
  24. STDTokenizer/STDtokenizer/modeling/base.py +3 -0
  25. STDTokenizer/STDtokenizer/modeling/modules/__init__.py +3 -0
  26. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc +3 -0
  27. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/attention.cpython-310.pyc +3 -0
  28. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/callbacks.cpython-310.pyc +3 -0
  29. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/codebook.cpython-310.pyc +3 -0
  30. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/diffaug.cpython-310.pyc +3 -0
  31. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/discriminator.cpython-310.pyc +3 -0
  32. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/encoders.cpython-310.pyc +3 -0
  33. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/gpt.cpython-310.pyc +3 -0
  34. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/lpips.cpython-310.pyc +3 -0
  35. STDTokenizer/STDtokenizer/modeling/modules/__pycache__/vae.cpython-310.pyc +3 -0
  36. STDTokenizer/STDtokenizer/modeling/modules/attention.py +3 -0
  37. STDTokenizer/STDtokenizer/modeling/modules/cache/vgg.pth +3 -0
  38. STDTokenizer/STDtokenizer/modeling/modules/callbacks.py +3 -0
  39. STDTokenizer/STDtokenizer/modeling/modules/codebook.py +3 -0
  40. STDTokenizer/STDtokenizer/modeling/modules/diffaug.py +3 -0
  41. STDTokenizer/STDtokenizer/modeling/modules/discriminator.py +3 -0
  42. STDTokenizer/STDtokenizer/modeling/modules/encoders.py +3 -0
  43. STDTokenizer/STDtokenizer/modeling/modules/gpt.py +3 -0
  44. STDTokenizer/STDtokenizer/modeling/modules/hiera.py +3 -0
  45. STDTokenizer/STDtokenizer/modeling/modules/lpips.py +3 -0
  46. STDTokenizer/STDtokenizer/modeling/modules/vae.py +3 -0
  47. STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc +1 -1
  48. STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py +2 -2
  49. STDTokenizer/STDtokenizer/modeling/stdtokenizer.py +3 -0
  50. STDTokenizer/STDtokenizer/train_utils.py +3 -0
OmniTokenizer/OmniTokenizer/data.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:93b6e068cf35f00a3c8a945653db14ad9d5b20976dd478bc0ce2f7a645a72d57
3
- size 40488
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c117215edf3dd6bf93e5b64065d5f695473626cd30932d0aca58e2ca28f10f7
3
+ size 40501
OmniTokenizer/OmniTokenizer/modules/codebook.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c7f9af0cc4339e3f92d98c527a41d1db9333ed6b8a63c551afe41cc897cca5bc
3
- size 6020
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c35802273a700c89e40453d917122f44eb35b1796ac992d83c954da917a2bb6
3
+ size 6033
OmniTokenizer/OmniTokenizer/omnitokenizer.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:75b83467ded2074fe5eb9df0f1fe6242ef5380c4dcc10d7413f31a4bde63e186
3
- size 52029
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e036814f6edf2564cbde4e47794a4a1fed8759b1c564803ba0527807982a27a3
3
+ size 52041
OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2f37cfff9c88be84c6c3a8289a9c313aba79e1a5dca6cb88d4fe5691dcc61b01
3
- size 74520
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eddc9dfaafe7744fda823f4ef3d4afcee8f7b01c199c186cb1faf75bde5e2368
3
+ size 74513
OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cba2163c71ced01c1e437a99f98b1364aebf29e8dcb46828466aa0f1de04ce1
3
- size 2466
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3557b4d7d15c35b283a192d60f257056fc11e03f4c4c9d4f11e4bd12cedb8ac7
3
+ size 2574
OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101_video.sh ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a413c65bb78896d2d26b503be520aa34bd9b1efc6122743377d86a60a20d8e07
3
+ size 2544
OmniTokenizer/videoAR/data/dataset_image_video.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0b9120f0b7aade06c94d18d06cba5a1488499e7b45fbdb3d9e74e6408118fb5b
3
- size 14503
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbd8929f3cbbe0c341c7eb1da50ea1b9442b26d4e84f717d0abc0c6dcff91fa7
3
+ size 14511
OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dcc382480552e529ac6fcbb9ef3943e166c552328595434329dd48dc07b7d415
3
  size 49240
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fad8a5b7532604d76674eb4a96907c25571c0ce75148b9a33dfdbdaa6dbb65b
3
  size 49240
OmniTokenizer/videoAR/validation_prompts_openvid.txt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:df12bba385fd2c8d61456944d6782421de536a570e48aab18b13ad596d318533
3
- size 394
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:476a5d08321b7297d9cf3556f275f4bb9c318de86a268ed9a36d5d20b64c17e2
3
+ size 358
STDTokenizer/.vscode/settings.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8eb95bcbc154530931e15fc418c8b1fe991095671409552099ea1aa596999ede
3
+ size 3
STDTokenizer/STDtokenizer/__pycache__/train_utils.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:442647935d41632c8dc7fe74a8ab2c56a77553b6ed384966216c30f64c5c3bbc
3
+ size 5424
STDTokenizer/STDtokenizer/__pycache__/utils.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa9fbacccaf0543cada21dd2479b58b7b1c556a99d01dc50491ce1c1d4c7c2eb
3
+ size 7614
STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:550e468f1b110fdf1522d658b643fa5f2f9d81bae50ef46c3cfc91c2e24c3112
3
- size 1768
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f0b52f4fe45df01bb11bede2396e89f970b61a075625578669d949de24f263f
3
+ size 3081
STDTokenizer/STDtokenizer/configs/stage2/stdtokenizer_l32.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc25fbc794e23d6b0bc5686c8c2863e326f8b258446e8f54ca52701e19fee7f3
3
+ size 3268
STDTokenizer/STDtokenizer/data/__pycache__/bucket_sampler.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26498796b35a208ad9dcd71f35fdb0f56acd657be0ff6dcec587a6bf8bfe1d9b
3
+ size 11560
STDTokenizer/STDtokenizer/data/__pycache__/dataset_image.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e6c8c3dacff640433390d752a3067c0efee62272915cb577ecdd117e8e2e7d7
3
+ size 2328
STDTokenizer/STDtokenizer/data/__pycache__/dataset_image_video.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e559b0c6ca19c23dd1bbc3e2d26927e119b4b8416f72541dd343b3a15bb6307
3
+ size 8772
STDTokenizer/STDtokenizer/data/bucket_sampler.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32e2cc751e9a1f415ea27e80d620f52a0281794e918b66563387203bc2ebd8ee
3
+ size 17117
STDTokenizer/STDtokenizer/data/dataset_image.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:534df24f57f4127f93dc95527ed627aa3001cca3aecfef65554e93dc4e86dfed
3
+ size 2340
STDTokenizer/STDtokenizer/data/dataset_image_video.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9de1b044e2e78bda70e6aa7cb6240af50d64f8a9c0ab545d1a2826ee761014c2
3
+ size 14480
STDTokenizer/STDtokenizer/data/dataset_video.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1394ab90f5846c07a517d04c38881d42afbf5548915100fa64fb5e52eed56b02
3
+ size 9831
STDTokenizer/STDtokenizer/modeling/__pycache__/base.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f66f6b7dd25786da2e93596e004f8e70b337f59cf65c749d57b4a262d2434f65
3
+ size 18164
STDTokenizer/STDtokenizer/modeling/__pycache__/stdtokenizer.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b8fae5875df71e642a41d9d356ff05f3ed2f5df35b881f3c533bab66607fac7
3
+ size 21738
STDTokenizer/STDtokenizer/modeling/base.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05408c70d3a26646e70a1eb64498fa1ca2131deb421b9c8fdf71782c2fafe408
3
+ size 25127
STDTokenizer/STDtokenizer/modeling/modules/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b340207c72632de14af91550cf80c42a4ab8b456695e2a4fd4a80f049e636a5c
3
+ size 113
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b47c7143c42d0f628782b0f4cb9c3d50b3bb408fcd66bc049ca6072a0376e37d
3
+ size 344
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/attention.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0cd61fffae941c41ac09fe9e59a6760159aa1a6e63db5958dad74a4c641ba1a
3
+ size 19851
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/callbacks.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2a67ccb22b67e9cabb8051c42141b5e5f3fd19ffc45163eca87087a8245d24d
3
+ size 5026
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/codebook.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:446520efad6bfdcbcf1d5ea95e799c94c4483a83b202a18a8658fd3e5a43a0f4
3
+ size 4151
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/diffaug.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:326b75dbbb68ceeacd8b26a1eba34e3fa4ea1550cf803940aefba3fc4e0b13dd
3
+ size 3715
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/discriminator.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:257f8b1f0bfa4c68e6a90ee978008e591005a2565cf896f984cd267ea193ee9f
3
+ size 16029
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/encoders.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3f5d605643ea70ff2a92194d7ba54380a07bbb01675cfa3ffd8c87d323c25b4
3
+ size 2201
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/gpt.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29b8bfb3ff0558f19e9b6de66447d4f1f77112be3e43d08b71c5bdd0cbd0b4f3
3
+ size 14060
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/lpips.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22af845628695180979f0b1f0733de6afd81c97f5940179ebab031f54deaed92
3
+ size 6837
STDTokenizer/STDtokenizer/modeling/modules/__pycache__/vae.cpython-310.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ead8346719f2ccff5e4a0ceb8f6ff515589445c8853b13c8036b3d6ab6903aca
3
+ size 2836
STDTokenizer/STDtokenizer/modeling/modules/attention.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5882172a08f47f842ad71dffbed482172e1e8a629997f1cbde2160562392ca8f
3
+ size 24426
STDTokenizer/STDtokenizer/modeling/modules/cache/vgg.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868
3
+ size 7289
STDTokenizer/STDtokenizer/modeling/modules/callbacks.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf0c36e46463f0f62601fe31ba5ace9d0ac12f875665094486e700bf330e6a9b
3
+ size 6303
STDTokenizer/STDtokenizer/modeling/modules/codebook.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a5c8ecfc33938f59b2c5dcf1ade9ba048e2a99f959e0e135aea6c746101c220
3
+ size 6021
STDTokenizer/STDtokenizer/modeling/modules/diffaug.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:895f074a12a0d7242fd1894a99859e1fe2f0ca72ca2b8be4fe9e5eaec0cec306
3
+ size 5490
STDTokenizer/STDtokenizer/modeling/modules/discriminator.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81ae11b195e7dd4b22ed72cdd85fcd1584b1dd842e1d35dd9851df5d06d60ef8
3
+ size 27308
STDTokenizer/STDtokenizer/modeling/modules/encoders.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1fab33e707640453c588ee041b2204b81d4414a78b59fd21600cd1a2345b0df
3
+ size 1473
STDTokenizer/STDtokenizer/modeling/modules/gpt.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d952f8c624f68489ab535841c04911b20cc1ca9bf12b48e84946fe9d14edb81
3
+ size 22070
STDTokenizer/STDtokenizer/modeling/modules/hiera.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f7d8881087294280f8f4948b5e30b0c396f24efce0bc7c6726bf38087900bc0
3
+ size 4874
STDTokenizer/STDtokenizer/modeling/modules/lpips.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cb27a27dcd7d7cfc50aafa323c06e94f92a8ee7cd731b296839df70e0775a5b
3
+ size 6434
STDTokenizer/STDtokenizer/modeling/modules/vae.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd8de96a05d004c34a7f629ad5999469e539564d863ec56fdee3dc7a24c449c9
3
+ size 2621
STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:44717cbaa35d07c9feae68560440b43f4ba7852451f5b4edc0c1b705aa18ef93
3
  size 11119
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1906be1d60152c95bbfc96eedc5f0b6d06b28fbac5203782e9a2706c9924f40
3
  size 11119
STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0937009e8b744ddfbd7495dd6bf1640863990a0a02a7c04a824db077d64b0c89
3
- size 15073
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59801f0ad43a61e92a503417c2ec109ce667282fd301734434fee991a32adf1b
3
+ size 15083
STDTokenizer/STDtokenizer/modeling/stdtokenizer.py CHANGED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15501cd5f1308e190b0ab51b03f90f1b946722e637f759840c9bc5ec544a459b
3
+ size 37732
STDTokenizer/STDtokenizer/train_utils.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d476fb1db97b5a9d29972679b20a7b4c7c1763e192f695b670337d65c398a13
3
+ size 11969