diff --git a/OmniTokenizer/OmniTokenizer/data.py b/OmniTokenizer/OmniTokenizer/data.py index 427d508ff447de0144886531483dbf78302792a2..2ddc6e4475a4bb2ca19e49b5562715a33d46da9e 100644 --- a/OmniTokenizer/OmniTokenizer/data.py +++ b/OmniTokenizer/OmniTokenizer/data.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:93b6e068cf35f00a3c8a945653db14ad9d5b20976dd478bc0ce2f7a645a72d57 -size 40488 +oid sha256:1c117215edf3dd6bf93e5b64065d5f695473626cd30932d0aca58e2ca28f10f7 +size 40501 diff --git a/OmniTokenizer/OmniTokenizer/modules/codebook.py b/OmniTokenizer/OmniTokenizer/modules/codebook.py index 6f4d2ca6e77b1b43078c884d29ea968775cd449f..16722ab48d66f19e27ffa98cdb40b30f747273ab 100644 --- a/OmniTokenizer/OmniTokenizer/modules/codebook.py +++ b/OmniTokenizer/OmniTokenizer/modules/codebook.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c7f9af0cc4339e3f92d98c527a41d1db9333ed6b8a63c551afe41cc897cca5bc -size 6020 +oid sha256:6c35802273a700c89e40453d917122f44eb35b1796ac992d83c954da917a2bb6 +size 6033 diff --git a/OmniTokenizer/OmniTokenizer/omnitokenizer.py b/OmniTokenizer/OmniTokenizer/omnitokenizer.py index 1d73ab3eb9808d8af103a70f419a4a4bf434230c..2de468736b04ec9c1545577817b81f35b0ef0b06 100644 --- a/OmniTokenizer/OmniTokenizer/omnitokenizer.py +++ b/OmniTokenizer/OmniTokenizer/omnitokenizer.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:75b83467ded2074fe5eb9df0f1fe6242ef5380c4dcc10d7413f31a4bde63e186 -size 52029 +oid sha256:e036814f6edf2564cbde4e47794a4a1fed8759b1c564803ba0527807982a27a3 +size 52041 diff --git a/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py index c17bb882b4681ed4e001a73d2b98680b01e03a9e..b73d9b2abdaebe1a6a2dbd7e131d5fbd576d1334 100644 --- a/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py +++ b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f37cfff9c88be84c6c3a8289a9c313aba79e1a5dca6cb88d4fe5691dcc61b01 -size 74520 +oid sha256:eddc9dfaafe7744fda823f4ef3d4afcee8f7b01c199c186cb1faf75bde5e2368 +size 74513 diff --git a/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh index 4a6b2caabe6487e42603a0777ace3f3fe07a0599..ad89968346d4b3001abf760f8bb244b07155b2e3 100644 --- a/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh +++ b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101.sh @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0cba2163c71ced01c1e437a99f98b1364aebf29e8dcb46828466aa0f1de04ce1 -size 2466 +oid sha256:3557b4d7d15c35b283a192d60f257056fc11e03f4c4c9d4f11e4bd12cedb8ac7 +size 2574 diff --git a/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101_video.sh b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101_video.sh new file mode 100644 index 0000000000000000000000000000000000000000..694646ac28a5fe1b06b701f6acdbea69cb39dfce --- /dev/null +++ b/OmniTokenizer/train_t2v_ns_disc_alltext_maskgit_ucf101_video.sh @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a413c65bb78896d2d26b503be520aa34bd9b1efc6122743377d86a60a20d8e07 +size 2544 diff --git a/OmniTokenizer/videoAR/data/dataset_image_video.py b/OmniTokenizer/videoAR/data/dataset_image_video.py index 27f59fdca8721073c57040b54e770b307878d7d0..4e66cf02339133fd885e70679f0b41260576f723 100644 --- a/OmniTokenizer/videoAR/data/dataset_image_video.py +++ b/OmniTokenizer/videoAR/data/dataset_image_video.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b9120f0b7aade06c94d18d06cba5a1488499e7b45fbdb3d9e74e6408118fb5b -size 14503 +oid sha256:bbd8929f3cbbe0c341c7eb1da50ea1b9442b26d4e84f717d0abc0c6dcff91fa7 +size 14511 diff --git a/OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py b/OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py index 09d7b47d45ca6045f19b9c2bb53c662bce2bbaa2..73e372c45d0ffde48bf99c3045246bac6c7279a2 100644 --- a/OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py +++ b/OmniTokenizer/videoAR/models/LlamaDiscNSAR_alltext_class_maskgit.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dcc382480552e529ac6fcbb9ef3943e166c552328595434329dd48dc07b7d415 +oid sha256:2fad8a5b7532604d76674eb4a96907c25571c0ce75148b9a33dfdbdaa6dbb65b size 49240 diff --git a/OmniTokenizer/videoAR/validation_prompts_openvid.txt b/OmniTokenizer/videoAR/validation_prompts_openvid.txt index 1423a30fe73c54aeca44464eeb0dae91ff22b9f1..80da335c55acc981b50e2620d4053b690464211f 100644 --- a/OmniTokenizer/videoAR/validation_prompts_openvid.txt +++ b/OmniTokenizer/videoAR/validation_prompts_openvid.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:df12bba385fd2c8d61456944d6782421de536a570e48aab18b13ad596d318533 -size 394 +oid sha256:476a5d08321b7297d9cf3556f275f4bb9c318de86a268ed9a36d5d20b64c17e2 +size 358 diff --git a/STDTokenizer/.vscode/settings.json b/STDTokenizer/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..9bbb674a254146ca5970591c845c76bc5eae4196 --- /dev/null +++ b/STDTokenizer/.vscode/settings.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb95bcbc154530931e15fc418c8b1fe991095671409552099ea1aa596999ede +size 3 diff --git a/STDTokenizer/STDtokenizer/__pycache__/train_utils.cpython-310.pyc b/STDTokenizer/STDtokenizer/__pycache__/train_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9c8bd84db35d5075a246ae840d1da4fa4055997 --- /dev/null +++ b/STDTokenizer/STDtokenizer/__pycache__/train_utils.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:442647935d41632c8dc7fe74a8ab2c56a77553b6ed384966216c30f64c5c3bbc +size 5424 diff --git a/STDTokenizer/STDtokenizer/__pycache__/utils.cpython-310.pyc b/STDTokenizer/STDtokenizer/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb58cf3cfdb354bceff46936f7f4d31e36cb6419 --- /dev/null +++ b/STDTokenizer/STDtokenizer/__pycache__/utils.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9fbacccaf0543cada21dd2479b58b7b1c556a99d01dc50491ce1c1d4c7c2eb +size 7614 diff --git a/STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml b/STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml index b374f5b958e29dd7fe2839d7ec6c4819ceb5fa99..e43f8cd5eb138c0b448baba10a617f45fd445401 100644 --- a/STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml +++ b/STDTokenizer/STDtokenizer/configs/stage1/stdtokenizer_l32.yaml @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:550e468f1b110fdf1522d658b643fa5f2f9d81bae50ef46c3cfc91c2e24c3112 -size 1768 +oid sha256:4f0b52f4fe45df01bb11bede2396e89f970b61a075625578669d949de24f263f +size 3081 diff --git a/STDTokenizer/STDtokenizer/configs/stage2/stdtokenizer_l32.yaml b/STDTokenizer/STDtokenizer/configs/stage2/stdtokenizer_l32.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a06f8ec2e0225ac5c86a8303f2e75d56b728239 --- /dev/null +++ b/STDTokenizer/STDtokenizer/configs/stage2/stdtokenizer_l32.yaml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc25fbc794e23d6b0bc5686c8c2863e326f8b258446e8f54ca52701e19fee7f3 +size 3268 diff --git a/STDTokenizer/STDtokenizer/data/__pycache__/bucket_sampler.cpython-310.pyc b/STDTokenizer/STDtokenizer/data/__pycache__/bucket_sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95281ab3e63d7b636074ca83690d2b35b81f9bb6 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/__pycache__/bucket_sampler.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26498796b35a208ad9dcd71f35fdb0f56acd657be0ff6dcec587a6bf8bfe1d9b +size 11560 diff --git a/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image.cpython-310.pyc b/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09c5aecabec99785cd761b45831db4f658b2d1f8 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e6c8c3dacff640433390d752a3067c0efee62272915cb577ecdd117e8e2e7d7 +size 2328 diff --git a/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image_video.cpython-310.pyc b/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image_video.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bba459e787d2d186ba29278b00ad5037a895e014 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/__pycache__/dataset_image_video.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e559b0c6ca19c23dd1bbc3e2d26927e119b4b8416f72541dd343b3a15bb6307 +size 8772 diff --git a/STDTokenizer/STDtokenizer/data/bucket_sampler.py b/STDTokenizer/STDtokenizer/data/bucket_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..0345ed734e2b4e441b842f9175f0e6623f7e0fd0 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/bucket_sampler.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32e2cc751e9a1f415ea27e80d620f52a0281794e918b66563387203bc2ebd8ee +size 17117 diff --git a/STDTokenizer/STDtokenizer/data/dataset_image.py b/STDTokenizer/STDtokenizer/data/dataset_image.py new file mode 100644 index 0000000000000000000000000000000000000000..c31442cef851e490e58577a44a94b89e9d7cd522 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/dataset_image.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:534df24f57f4127f93dc95527ed627aa3001cca3aecfef65554e93dc4e86dfed +size 2340 diff --git a/STDTokenizer/STDtokenizer/data/dataset_image_video.py b/STDTokenizer/STDtokenizer/data/dataset_image_video.py new file mode 100644 index 0000000000000000000000000000000000000000..7af46821e1506fad1b7c99e2a37db629f51452ff --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/dataset_image_video.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9de1b044e2e78bda70e6aa7cb6240af50d64f8a9c0ab545d1a2826ee761014c2 +size 14480 diff --git a/STDTokenizer/STDtokenizer/data/dataset_video.py b/STDTokenizer/STDtokenizer/data/dataset_video.py new file mode 100644 index 0000000000000000000000000000000000000000..acc71393f118021c50292cfa70d64360ba643cf6 --- /dev/null +++ b/STDTokenizer/STDtokenizer/data/dataset_video.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1394ab90f5846c07a517d04c38881d42afbf5548915100fa64fb5e52eed56b02 +size 9831 diff --git a/STDTokenizer/STDtokenizer/modeling/__pycache__/base.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/__pycache__/base.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..055ff953a3b076c832b4acf56b573d7747f88b9c --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/__pycache__/base.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f66f6b7dd25786da2e93596e004f8e70b337f59cf65c749d57b4a262d2434f65 +size 18164 diff --git a/STDTokenizer/STDtokenizer/modeling/__pycache__/stdtokenizer.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/__pycache__/stdtokenizer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2806af9223a659f4c528e859191b2583655c3581 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/__pycache__/stdtokenizer.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8fae5875df71e642a41d9d356ff05f3ed2f5df35b881f3c533bab66607fac7 +size 21738 diff --git a/STDTokenizer/STDtokenizer/modeling/base.py b/STDTokenizer/STDtokenizer/modeling/base.py new file mode 100644 index 0000000000000000000000000000000000000000..f623b45a6b3114257643d2670f4a2810300dd1b7 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/base.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05408c70d3a26646e70a1eb64498fa1ca2131deb421b9c8fdf71782c2fafe408 +size 25127 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__init__.py b/STDTokenizer/STDtokenizer/modeling/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ad5d24b197006ff28ad54012f5495f8f95adac4 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__init__.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b340207c72632de14af91550cf80c42a4ab8b456695e2a4fd4a80f049e636a5c +size 113 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5ea73f34c7f68ae3ca3309ce529d0d5abe9f8cd --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b47c7143c42d0f628782b0f4cb9c3d50b3bb408fcd66bc049ca6072a0376e37d +size 344 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/attention.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/attention.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bdca5f1b47bb8c8e1519fa797014a017fda176d --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/attention.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0cd61fffae941c41ac09fe9e59a6760159aa1a6e63db5958dad74a4c641ba1a +size 19851 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/callbacks.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/callbacks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..972844b44efc33d77e71b2fcf6455193e35f82af --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/callbacks.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2a67ccb22b67e9cabb8051c42141b5e5f3fd19ffc45163eca87087a8245d24d +size 5026 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/codebook.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/codebook.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d8fcde2162cb081eddf58fbdb97cf153cf30900 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/codebook.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:446520efad6bfdcbcf1d5ea95e799c94c4483a83b202a18a8658fd3e5a43a0f4 +size 4151 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/diffaug.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/diffaug.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36c79add6cab8f4cba3b6286f3d182b3b9042363 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/diffaug.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326b75dbbb68ceeacd8b26a1eba34e3fa4ea1550cf803940aefba3fc4e0b13dd +size 3715 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/discriminator.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/discriminator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e7b705c2da8c00761d74b83daaf52019958eb11 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/discriminator.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257f8b1f0bfa4c68e6a90ee978008e591005a2565cf896f984cd267ea193ee9f +size 16029 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/encoders.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/encoders.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d767185f804079ac302a0923b7739e2187c40f8 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/encoders.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f5d605643ea70ff2a92194d7ba54380a07bbb01675cfa3ffd8c87d323c25b4 +size 2201 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/gpt.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/gpt.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52907f82438b77d10c10652e12202396267f6c11 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/gpt.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29b8bfb3ff0558f19e9b6de66447d4f1f77112be3e43d08b71c5bdd0cbd0b4f3 +size 14060 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/lpips.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/lpips.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7035685bab04ef2b1f7f3bb242309fc97a363dda --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/lpips.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22af845628695180979f0b1f0733de6afd81c97f5940179ebab031f54deaed92 +size 6837 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/vae.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/vae.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f1aa574e7f62c8b532466559e348ff097b233ff --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/__pycache__/vae.cpython-310.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ead8346719f2ccff5e4a0ceb8f6ff515589445c8853b13c8036b3d6ab6903aca +size 2836 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/attention.py b/STDTokenizer/STDtokenizer/modeling/modules/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..6c577cc8cb5c2efa190c7d4f21ab4fbb2f13886d --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/attention.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5882172a08f47f842ad71dffbed482172e1e8a629997f1cbde2160562392ca8f +size 24426 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/cache/vgg.pth b/STDTokenizer/STDtokenizer/modeling/modules/cache/vgg.pth new file mode 100644 index 0000000000000000000000000000000000000000..f57dcf5cc764d61c8a460365847fb2137ff0a62d --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/cache/vgg.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868 +size 7289 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/callbacks.py b/STDTokenizer/STDtokenizer/modeling/modules/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..154aff8bb1e1c5e83bd66323157dc46a3f88e1cb --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/callbacks.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf0c36e46463f0f62601fe31ba5ace9d0ac12f875665094486e700bf330e6a9b +size 6303 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/codebook.py b/STDTokenizer/STDtokenizer/modeling/modules/codebook.py new file mode 100644 index 0000000000000000000000000000000000000000..4f5d330823fe08eb08407994f1335ed99ef68c20 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/codebook.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5c8ecfc33938f59b2c5dcf1ade9ba048e2a99f959e0e135aea6c746101c220 +size 6021 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/diffaug.py b/STDTokenizer/STDtokenizer/modeling/modules/diffaug.py new file mode 100644 index 0000000000000000000000000000000000000000..e489016a2dcf1ac60a8923b63bc1a683e5e76016 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/diffaug.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895f074a12a0d7242fd1894a99859e1fe2f0ca72ca2b8be4fe9e5eaec0cec306 +size 5490 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/discriminator.py b/STDTokenizer/STDtokenizer/modeling/modules/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..81f0212a2db7b060f1d8850491f3ec70ecffefa3 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/discriminator.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ae11b195e7dd4b22ed72cdd85fcd1584b1dd842e1d35dd9851df5d06d60ef8 +size 27308 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/encoders.py b/STDTokenizer/STDtokenizer/modeling/modules/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..351bb245fb3f4b2f58ff06fe50bd63d0abef2c97 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/encoders.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1fab33e707640453c588ee041b2204b81d4414a78b59fd21600cd1a2345b0df +size 1473 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/gpt.py b/STDTokenizer/STDtokenizer/modeling/modules/gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..8fdf7d82437f16ed62ab11b011ae85f18c462015 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/gpt.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d952f8c624f68489ab535841c04911b20cc1ca9bf12b48e84946fe9d14edb81 +size 22070 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/hiera.py b/STDTokenizer/STDtokenizer/modeling/modules/hiera.py new file mode 100644 index 0000000000000000000000000000000000000000..aef1831b3d3b915a8258be169ffdd8744a03e5ff --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/hiera.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7d8881087294280f8f4948b5e30b0c396f24efce0bc7c6726bf38087900bc0 +size 4874 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/lpips.py b/STDTokenizer/STDtokenizer/modeling/modules/lpips.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c69470eec3752cdf16a51d89512aca0455c316 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/lpips.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb27a27dcd7d7cfc50aafa323c06e94f92a8ee7cd731b296839df70e0775a5b +size 6434 diff --git a/STDTokenizer/STDtokenizer/modeling/modules/vae.py b/STDTokenizer/STDtokenizer/modeling/modules/vae.py new file mode 100644 index 0000000000000000000000000000000000000000..e7175450f81c0b58f7826f9e34af82cead809cb6 --- /dev/null +++ b/STDTokenizer/STDtokenizer/modeling/modules/vae.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd8de96a05d004c34a7f629ad5999469e539564d863ec56fdee3dc7a24c449c9 +size 2621 diff --git a/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc b/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc index a22484f6cc3ebc8f8a08440162d2c609212866d9..d38514b5cb367df2505b0cd13519790e2db649ee 100644 --- a/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc +++ b/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/__pycache__/pretrain_showo_magvit.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44717cbaa35d07c9feae68560440b43f4ba7852451f5b4edc0c1b705aa18ef93 +oid sha256:f1906be1d60152c95bbfc96eedc5f0b6d06b28fbac5203782e9a2706c9924f40 size 11119 diff --git a/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py b/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py index bd4d22316f6d088102c3fc5a840ea85d8473152d..2c003a6f883dc34a9df843bdda953880eaa55179 100644 --- a/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py +++ b/STDTokenizer/STDtokenizer/modeling/showo_magvitv2/pretrain_showo_magvit.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0937009e8b744ddfbd7495dd6bf1640863990a0a02a7c04a824db077d64b0c89 -size 15073 +oid sha256:59801f0ad43a61e92a503417c2ec109ce667282fd301734434fee991a32adf1b +size 15083 diff --git a/STDTokenizer/STDtokenizer/modeling/stdtokenizer.py b/STDTokenizer/STDtokenizer/modeling/stdtokenizer.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..3cef2098ff2ac2e668265dd532ea40ca06605bac 100644 --- a/STDTokenizer/STDtokenizer/modeling/stdtokenizer.py +++ b/STDTokenizer/STDtokenizer/modeling/stdtokenizer.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15501cd5f1308e190b0ab51b03f90f1b946722e637f759840c9bc5ec544a459b +size 37732 diff --git a/STDTokenizer/STDtokenizer/train_utils.py b/STDTokenizer/STDtokenizer/train_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..032549d1acd93022f61fb46e29d47219a9e3c26e --- /dev/null +++ b/STDTokenizer/STDtokenizer/train_utils.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d476fb1db97b5a9d29972679b20a7b4c7c1763e192f695b670337d65c398a13 +size 11969 diff --git a/STDTokenizer/STDtokenizer/utils.py b/STDTokenizer/STDtokenizer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..edf6f0d9b247b5d7ab35299654f326ca98f25922 --- /dev/null +++ b/STDTokenizer/STDtokenizer/utils.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5cd2bfb2eba732bbc5ba2310e460c5baec4f316611400086dd60c2b50e4bd61 +size 10541 diff --git a/STDTokenizer/dataset_process_codes b/STDTokenizer/dataset_process_codes new file mode 120000 index 0000000000000000000000000000000000000000..69fdd930e672d1e9fe997fb4c57b9cf720991e11 --- /dev/null +++ b/STDTokenizer/dataset_process_codes @@ -0,0 +1 @@ +../dataset_process_codes \ No newline at end of file diff --git a/STDTokenizer/modeling/modules/__init__.py b/STDTokenizer/modeling/modules/__init__.py index 2af88eacb41604eb0480b254d979ef054312ef82..95daa9b479483f118524f3900e45ac43b313285f 100644 --- a/STDTokenizer/modeling/modules/__init__.py +++ b/STDTokenizer/modeling/modules/__init__.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3cf2a0bae46ea19fcde065fdd5b3fec306bbe2e9dbedccb23f5f90838d6f15a5 -size 319 +oid sha256:d4e7edabaa605c243b81955e2a820297d5ac385f6a3069428a646c5f4de83e80 +size 379 diff --git a/STDTokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc b/STDTokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc index 73ebb9a642fab2d7694a5141cf5c0e390486eaec..f4dfd63a93e31949f9b7e672cbbd061af46745b3 100644 --- a/STDTokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc +++ b/STDTokenizer/modeling/modules/__pycache__/__init__.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fedf5427b616a3e5b22a94d70988b49ad65cbaec2d2399c018be61502d7fe0ad -size 566 +oid sha256:3abfaee63c81fa9c595117df999ef804a0763ee1d63d4e929e61183ddac27c52 +size 644 diff --git a/STDTokenizer/modeling/modules/__pycache__/blocks.cpython-310.pyc b/STDTokenizer/modeling/modules/__pycache__/blocks.cpython-310.pyc index 27148a158eded4ba35ad182a31ff6d973b8ffadb..56b40934b841df46991e7feabf1b9ddb96ada50b 100644 --- a/STDTokenizer/modeling/modules/__pycache__/blocks.cpython-310.pyc +++ b/STDTokenizer/modeling/modules/__pycache__/blocks.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f65a608a862bfcf86e6ce459021f2853874ac760ef9eb108b8807fbe0fae42e3 +oid sha256:fdd7039b04f82a694fa589fa69fb3bb71dc3511577fa89e83beefbeca226af92 size 11350 diff --git a/STDTokenizer/modeling/modules/__pycache__/losses.cpython-310.pyc b/STDTokenizer/modeling/modules/__pycache__/losses.cpython-310.pyc index 68f66865fee2547eca9f9ed3062c1d8a64354f33..601167de30f5d3572404d710f6f3e04e31ec47b9 100644 --- a/STDTokenizer/modeling/modules/__pycache__/losses.cpython-310.pyc +++ b/STDTokenizer/modeling/modules/__pycache__/losses.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cbc299476b31ed9f6e61840102544036921d9583f2a4e0f6ac5f8f578fc15863 -size 8587 +oid sha256:c1d9533adb2cbbfc414937e03914f57e6df7c7587171576a33f5fd85c087492c +size 11741 diff --git a/STDTokenizer/modeling/modules/blocks.py b/STDTokenizer/modeling/modules/blocks.py index cb1c39591cd17789a88ce1337d7d9c98489f86a9..9c103defbdf215022a90c1a61c8570e2dad94d39 100644 --- a/STDTokenizer/modeling/modules/blocks.py +++ b/STDTokenizer/modeling/modules/blocks.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:79b4361a7b1e6d5e2554eea95e0a0f60022ea7b1112b09c1d4d37297cf6ac878 -size 14109 +oid sha256:0dfb7e810fd96ffb0614f2d2b05b66c2a5653dcc390882701a08945455548f28 +size 14130 diff --git a/STDTokenizer/modeling/modules/losses.py b/STDTokenizer/modeling/modules/losses.py index 12664852da6013e71a610de73c7f94e1ced22072..3f164fa4387f979cc36ff2c3423bedf9989e79c8 100644 --- a/STDTokenizer/modeling/modules/losses.py +++ b/STDTokenizer/modeling/modules/losses.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9fb09cd6086b3c25d83a6f36044abc8ce42fd89afc60c95492391095112e7d8f -size 12186 +oid sha256:c9068fdc39e599533d88bb9a1e3bf603f3c76a7f795c91918cacb6e5b277e288 +size 21942 diff --git a/STDTokenizer/modeling/quantizer/__pycache__/quantizer.cpython-310.pyc b/STDTokenizer/modeling/quantizer/__pycache__/quantizer.cpython-310.pyc index 9ad8923b75692e387932f9b0d45afe0347966de9..08814f79bdca2dfcfa95e07c08686583e976a2c9 100644 --- a/STDTokenizer/modeling/quantizer/__pycache__/quantizer.cpython-310.pyc +++ b/STDTokenizer/modeling/quantizer/__pycache__/quantizer.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c75eb783f6c6e000517e3d24d68d6e6d340e4bf5a3bdcc759a6ed861d15346b2 +oid sha256:3e6e329933705a6155591fd207c1306873eea77752762346f4491300d5b7dfdf size 3223 diff --git a/STDTokenizer/modeling/quantizer/quantizer.py b/STDTokenizer/modeling/quantizer/quantizer.py index fe4c2e821a11ab0267ae7e8ffc1f707c40ece7f4..94c56e0fa255803b0aeb871d9ca3ad7901d67381 100644 --- a/STDTokenizer/modeling/quantizer/quantizer.py +++ b/STDTokenizer/modeling/quantizer/quantizer.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f129b140f43afe7462fb384cca88ec9562feab813af3844cbd38dab7dc121c47 -size 3652 +oid sha256:5201a68491e9f3c3689d960f74e293782a329550d42b8dc82d1a55921fa94af9 +size 3679 diff --git a/STDTokenizer/titok_b64_stage1_run1/config.yaml b/STDTokenizer/titok_b64_stage1_run1/config.yaml index 785c89efe77eeae390f3c37c1fe7360c34a7a0d0..0739cce2307b7595d500a3f2d82de497553959c3 100644 --- a/STDTokenizer/titok_b64_stage1_run1/config.yaml +++ b/STDTokenizer/titok_b64_stage1_run1/config.yaml @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d5359306cd4b9723d720c89421a55f4c7b0c121bd02cfdf476968a5eb644f99b -size 1610 +oid sha256:223b83867b8dc05bff3c7b566c022ac5807970ce31c76e41622726baea6cc60d +size 2280 diff --git a/STDTokenizer/titok_b64_stage1_run1/log0.txt b/STDTokenizer/titok_b64_stage1_run1/log0.txt index 8d3aa427311aafb7c13fae709ad166c0670fb36a..84dacc9fc5180ab68d24ff3c118357cb176ba15d 100644 --- a/STDTokenizer/titok_b64_stage1_run1/log0.txt +++ b/STDTokenizer/titok_b64_stage1_run1/log0.txt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ba5eb1d27d67c95cb5ddb73836842949cff42d82171926f2c6c3f4e79d80adc5 -size 3443 +oid sha256:7a9a914a978982ae360c93b754bd94320cf241cfa92a47ed59da2d9d16924aad +size 8385 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/debug-internal.log b/STDTokenizer/titok_b64_stage1_run1/wandb/debug-internal.log index 9b537abaa936c82a7f4eecfcb520ccecf5ea6084..8c8b3f87c2de01eb7304380791dc1459f486f988 120000 --- a/STDTokenizer/titok_b64_stage1_run1/wandb/debug-internal.log +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/debug-internal.log @@ -1 +1 @@ -run-20241004_214117-7gejd95h/logs/debug-internal.log \ No newline at end of file +run-20241005_025350-kjdtove4/logs/debug-internal.log \ No newline at end of file diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/debug.log b/STDTokenizer/titok_b64_stage1_run1/wandb/debug.log index fa2926932d740dfd1de2c931db67a9d6ceb66bd4..63d88817c611e92255a2a36b01fdcc9181df447c 120000 --- a/STDTokenizer/titok_b64_stage1_run1/wandb/debug.log +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/debug.log @@ -1 +1 @@ -run-20241004_214117-7gejd95h/logs/debug.log \ No newline at end of file +run-20241005_025350-kjdtove4/logs/debug.log \ No newline at end of file diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/latest-run b/STDTokenizer/titok_b64_stage1_run1/wandb/latest-run index 736a1605ca633db18c365cc5bcee22d908eb1722..2476255c888e9ba76ba04e58ac26b940c3d329a1 120000 --- a/STDTokenizer/titok_b64_stage1_run1/wandb/latest-run +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/latest-run @@ -1 +1 @@ -run-20241004_214117-7gejd95h \ No newline at end of file +run-20241005_025350-kjdtove4 \ No newline at end of file diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/conda-environment.yaml b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cabfc2f7e19ff79ac99e305929d6f6639bef0c7f --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/conda-environment.yaml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fee534d3a65ff862b97e6a66ac980a6789845a02c139421c506de456356212e +size 8628 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/config.yaml b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99a3fcfcfb938cc2e87fd71b6f7d3f9f4426736b --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/config.yaml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fddb7df159f5a65eef58d67b4a7946b229509d3c04d08bffbc2a4e90abde6fd +size 605 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/output.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4fb1077519fbd7186528fb4545793e62eff77214 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/output.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e86a8ddf21fb41836dfca08f0bcad94c99b4b738416a0a42e5916e2fe9bbf7b4 +size 2925 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/requirements.txt b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..91faf7cedc872de66d6509b2f793dc0e3fa0590c --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/requirements.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2dfa91656860c0a58ae8d4dcd695314ef72e4b7a7120e7a25037651724f7139 +size 4685 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-metadata.json b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e2f955c729a68cbb7563e8415966e1b49582ef20 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b905fd89a3565edafa5cec73be0683b5de73f935b03dd5b84c09a58d36f21f2d +size 25472 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-summary.json b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f7340b658f407004fa9afd6b769ecdbc9ecd90b6 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/files/wandb-summary.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a7a69ebd3c6549cefa65070edca7adf3334d67b9519a06c55e45f9f636161bc +size 27 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug-internal.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..991df6361359febc42355121839b38dbd5df45a9 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug-internal.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc598ffdf1950d334b54df46d15badd247f9262c7a27862471424858dde80d35 +size 39590 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ba4bfb53a2163b6ba1076941fd8490c4351d226f --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/logs/debug.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d42b5a034e703a97afee5e709119efac5dbd7c3105def5adaefc01dbcdd85d5a +size 3123 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/run-a1a3g3s1.wandb b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/run-a1a3g3s1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..260c12d7c18959091f9c40c2a2ad3bd4c4d31987 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025118-a1a3g3s1/run-a1a3g3s1.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99edb1c66dcdc0b182628fc181abce069b2e97a1fbb3e562d468963dd206b04e +size 22454 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/conda-environment.yaml b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0548d043f876aba8c4215b8ab5e5a3f68d9d6c9 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/conda-environment.yaml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cec301f10b2282edee3694b71297b03d71ebdbb1d234b9ecedd420598f6e101 +size 8629 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/config.yaml b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d16e64f423d65776f5336b0ad4ace6b366f2a9c0 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/config.yaml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45a7e2fcee20dc858ab71dee72322f3e7fb6d719bb8edb4d1d98bdd5ee0df4b6 +size 605 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/output.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..329f6635bc602d451dfe3e4daf61487ed32c0a8d --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/output.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e7f32f8d51a078a106a4c49e1a955fbb1e34ad6322db6204a00ed09427a9a59 +size 5632 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/requirements.txt b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..91faf7cedc872de66d6509b2f793dc0e3fa0590c --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/requirements.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2dfa91656860c0a58ae8d4dcd695314ef72e4b7a7120e7a25037651724f7139 +size 4685 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-metadata.json b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..df731cee0f7ee574aba3246829ee25d6cfb631db --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:932f8a1e866b1250c4532c43bfd904ac74215af80d40750b43224824a8e8190b +size 25470 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-summary.json b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c02f77d41428f5c2bb214d72c4e23bae41589a0c --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/files/wandb-summary.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa4bde922b1e35acf6d3eb8b99cb716434843b8e5aa98cd1a8d1c47f8ef3ef64 +size 28 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug-internal.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..330c3fd09fd18d8bd7431119bbb6b4196a7c1995 --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug-internal.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a2a2d0b78038a49ec0c003ae1e5ff82d9e76c87bd57224967fddbad0ea37a6 +size 51942 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug.log b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..657266d2a3df31ed58ceb3a7ab792cf3e71ae01f --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/logs/debug.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55954348f6218d0e6dfe4e267e215905e131eae0d1b8f4e78f131c96c530ecdd +size 3123 diff --git a/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/run-kjdtove4.wandb b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/run-kjdtove4.wandb new file mode 100644 index 0000000000000000000000000000000000000000..25a8a0b622d08efabc962ac1de3be5c63be0ff0e --- /dev/null +++ b/STDTokenizer/titok_b64_stage1_run1/wandb/run-20241005_025350-kjdtove4/run-kjdtove4.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a278b097e897981a0306873b958099d51c30486c81fe7c70900c24d9de6dc72 +size 98942 diff --git a/STDTokenizer/train_STDtokenizer.py b/STDTokenizer/train_STDtokenizer.py index f94af3b5bbb48c447103c12150ee6659c8e0c3ef..fd105f5478002554896fc83f265ac0c2375d16cd 100644 --- a/STDTokenizer/train_STDtokenizer.py +++ b/STDTokenizer/train_STDtokenizer.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e8be54175d66154975d381056ce77778cd3f53b9370a2f872f877c8e71754135 -size 6968 +oid sha256:a0b2b971cf8a8eeea7bf8f3e0c40056b899ae0fa6109c30e49c59fb8119e001c +size 7584 diff --git a/STDTokenizer/train_STDtokenizer_stage1.sh b/STDTokenizer/train_STDtokenizer_stage1.sh index b8b887fe28965cf77f77fb35b0e26f40da89afa4..33e2f3812c80db150bc935f2b9ea2638de173466 100644 --- a/STDTokenizer/train_STDtokenizer_stage1.sh +++ b/STDTokenizer/train_STDtokenizer_stage1.sh @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1e909980ba181421b2fb73b274aa6fb4c15b989fd587483d4d07eed0f4708475 -size 479 +oid sha256:e708292871185ec16f5e2475d9f4375c4a9998bded32fd1e9611acf4dd3a150f +size 337 diff --git a/STDTokenizer/train_STDtokenizer_stage2.sh b/STDTokenizer/train_STDtokenizer_stage2.sh new file mode 100644 index 0000000000000000000000000000000000000000..2742964e43a4f2e90afe1fcad8cea53380428e9c --- /dev/null +++ b/STDTokenizer/train_STDtokenizer_stage2.sh @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceeb2b12705e6eec2746b563aff407a19bc79d16ebd1e62d4a5600980b1ac98a +size 328 diff --git a/STDTokenizer/utils/__pycache__/train_utils.cpython-310.pyc b/STDTokenizer/utils/__pycache__/train_utils.cpython-310.pyc index 4873bbd8c5d2c145279cf28fed66dcf61844a03d..4085da3b27a8107ae6f3b4235de410d3f057919b 100644 --- a/STDTokenizer/utils/__pycache__/train_utils.cpython-310.pyc +++ b/STDTokenizer/utils/__pycache__/train_utils.cpython-310.pyc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:94008ddfb458115d8417b46f3616792c82811696e96debe2f76617f4e50d9465 -size 20425 +oid sha256:21925a832edbe1937db22997216f651ae6211476ca5658238e3b28b37c9d8989 +size 25304 diff --git a/STDTokenizer/utils/train_utils.py b/STDTokenizer/utils/train_utils.py index 8ca43764b9aa4155676dfba58c9deedd72404664..0467ab301c4a9b1f9849abeb945fa4ff2f5bc153 100644 --- a/STDTokenizer/utils/train_utils.py +++ b/STDTokenizer/utils/train_utils.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:80202f1a9f57b0e17d3159bcf515dc11bd3c1111f79baf4adc17bfc602fb5ef8 -size 35828 +oid sha256:133b512e74a235fa1ecd587f986762d6655f73bdf62d13b32d52e0d1573c2acd +size 42348 diff --git a/dataset_process_codes/imagenet-1k-processed-test.py b/dataset_process_codes/imagenet-1k-processed-test.py new file mode 100644 index 0000000000000000000000000000000000000000..521637289e27b85f823c289ea011b2a6ea88707c --- /dev/null +++ b/dataset_process_codes/imagenet-1k-processed-test.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb35423cc0dba7cbd393e3c4df50c06142c60aa264bca3e00514efed4d150bb +size 3080