Upload code from /mnt/43.oT_eV

Meissonic/.github/FUNDING.yml

@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github: viiika
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: # Replace with a single Buy Me a Coffee username
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

Meissonic/.gitignore

@@ -0,0 +1,166 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Mac OS related
+.DS_Store
+*.DS_Store
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Meissonic/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Meissonic/README.md

@@ -0,0 +1,222 @@
+# Meissonic: Revitalizing Masked Generative Transformers for Efficient High-Resolution Text-to-Image Synthesis
+
+<div align="center">
+<img width="1421" alt="Meissonic Banner" src="https://github.com/user-attachments/assets/703f6882-163a-42d0-8da8-3680231ca75e">
+
+[![arXiv](https://img.shields.io/badge/arXiv-2410.08261-b31b1b.svg)](https://arxiv.org/abs/2410.08261)
+[![Hugging Face](https://img.shields.io/badge/🤗%20Huggingface-Model_Meissonic-yellow)](https://huggingface.co/MeissonFlow/Meissonic)
+[![GitHub](https://img.shields.io/badge/GitHub-Repo-181717?logo=github)](https://github.com/viiika/Meissonic)
+[![YouTube](https://img.shields.io/badge/YouTube-Tutorial_EN-FF0000?logo=youtube)](https://www.youtube.com/watch?v=PlmifElhr6M)
+[![YouTube](https://img.shields.io/badge/YouTube-Tutorial_JA-FF0000?logo=youtube)](https://www.youtube.com/watch?v=rJDrf49wF64)
+[![Demo](https://img.shields.io/badge/Live-Demo_Meissonic-blue?logo=huggingface)](https://huggingface.co/spaces/MeissonFlow/meissonic)
+[![Replicate](https://replicate.com/chenxwh/meissonic/badge)](https://replicate.com/chenxwh/meissonic)
+
+[![Hugging Face](https://img.shields.io/badge/🤗%20Huggingface-Model_Monetico-yellow)](https://huggingface.co/Collov-Labs/Monetico)
+[![Demo](https://img.shields.io/badge/Live-Demo_Monetico-blue?logo=huggingface)](https://huggingface.co/spaces/Collov-Labs/Monetico)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2411.10781-b31b1b.svg)](https://arxiv.org/abs/2411.10781)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2503.15457-b31b1b.svg)](https://arxiv.org/abs/2503.15457)
+[![Hugging Face](https://img.shields.io/badge/🤗%20Huggingface-Model_DiMO-yellow)](https://huggingface.co/Yuanzhi/DiMO)
+[![GitHub](https://img.shields.io/badge/GitHub-Repo-181717?logo=github)](https://github.com/yuanzhi-zhu/DiMO)
+
+
+[![arXiv](https://img.shields.io/badge/arXiv-2505.23606-b31b1b.svg)](https://arxiv.org/abs/2505.23606)
+[![Hugging Face](https://img.shields.io/badge/🤗%20Huggingface-Model_Muddit-yellow)](https://huggingface.co/MeissonFlow/Muddit)
+[![GitHub](https://img.shields.io/badge/GitHub-Repo-181717?logo=github)](https://github.com/M-E-AGI-Lab/Muddit)
+[![Demo](https://img.shields.io/badge/Live-Demo_Muddit-blue?logo=huggingface)](https://huggingface.co/spaces/MeissonFlow/muddit)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2507.04947-b31b1b.svg)](https://arxiv.org/abs/2507.04947)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2508.10684-b31b1b.svg)](https://arxiv.org/abs/2508.10684)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2509.19244-b31b1b.svg)](https://arxiv.org/abs/2509.19244)
+[![arXiv](https://img.shields.io/badge/arXiv-2509.23919-b31b1b.svg)](https://arxiv.org/abs/2509.23919)
+[![arXiv](https://img.shields.io/badge/arXiv-2509.25171-b31b1b.svg)](https://arxiv.org/abs/2509.25171)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2510.06308-b31b1b.svg)](https://arxiv.org/abs/2510.06308)
+
+[![arXiv](https://img.shields.io/badge/arXiv-2510.20668-b31b1b.svg)](https://arxiv.org/abs/2510.20668) [![GitHub](https://img.shields.io/badge/GitHub-Repo-181717?logo=github)](https://github.com/M-E-AGI-Lab/Awesome-World-Models)
+
+</div>
+
+## 📝 Meissonic Updates and Family Papers
+
+- [MaskGIT: Masked Generative Image Transformer](https://arxiv.org/abs/2202.04200) [CVPR 2022]
+- [Muse: Text-To-Image Generation via Masked Generative Transformers](https://arxiv.org/abs/2301.00704) [ICML 2023]
+- [🌟][Meissonic: Revitalizing Masked Generative Transformers for Efficient High-Resolution Text-to-Image Synthesis](https://arxiv.org/abs/2410.08261) [ICLR 2025]
+- [Bag of Design Choices for Inference of High-Resolution Masked Generative Transformer](https://arxiv.org/abs/2411.10781)
+- [Di[𝙼]O: Distilling Masked Diffusion Models into One-step Generator](https://arxiv.org/abs/2503.15457) [ICCV 2025]
+- [🌟][Muddit: Liberating Generation Beyond Text-to-Image with a Unified Discrete Diffusion Model](https://arxiv.org/abs/2505.23606)
+- [DC-AR: Efficient Masked Autoregressive Image Generation with Deep Compression Hybrid Tokenizer](https://arxiv.org/pdf/2507.04947) [ICCV 2025]
+- [MDNS: Masked Diffusion Neural Sampler via Stochastic Optimal Control](https://arxiv.org/abs/2508.10684)
+- [Lavida-O: Elastic Large Masked Diffusion Models for Unified Multimodal Understanding and Generation](https://arxiv.org/abs/2509.19244)
+- [🌟][Lumina-DiMOO: An Omni Diffusion Large Language Model for Multi-Modal Generation and Understanding](https://arxiv.org/abs/2510.06308)
+- [Token Painter: Training-Free Text-Guided Image Inpainting via Mask Autoregressive Models](https://arxiv.org/abs/2509.23919)
+- [TR2-D2: Tree Search Guided Trajectory-Aware Fine-Tuning for Discrete Diffusion](https://arxiv.org/abs/2509.25171)
+- [OneFlow: Concurrent Mixed-Modal and Interleaved Generation with Edit Flows](https://arxiv.org/abs/2510.03506)
+- [Diffuse Everything: Multimodal Diffusion Models on Arbitrary State Spaces](https://arxiv.org/abs/2506.07903) [ICML 2025]
+- [Towards Better & Faster Autoregressive Image Generation: From the Perspective of Entropy](https://arxiv.org/abs/2510.09012) [NeurIPS 2025]
+- [🌟][From Masks to Worlds: A Hitchhiker's Guide to World Models](https://arxiv.org/abs/2510.20668)
+- [Soft-Di[M]O: Improving One-Step Discrete Image Generation with Soft Embeddings](https://arxiv.org/abs/2509.22925)
+
+- More papers are coming soon!
+See [MeissonFlow Research](https://huggingface.co/MeissonFlow) (Organization Card) for more about our vision.
+
+
+![Meissonic Demos](./assets/demos.png)
+
+## 🚀 Introduction
+
+Meissonic is a non-autoregressive mask image modeling text-to-image synthesis model that can generate high-resolution images. It is designed to run on consumer graphics cards.
+
+![Architecture](./assets/architecture.png)
+
+**Key Features:**
+- 🖼️ High-resolution image generation (up to 1024x1024)
+- 💻 Designed to run on consumer GPUs
+- 🎨 Versatile applications: text-to-image, image-to-image
+
+## 🛠️ Prerequisites
+
+### Step 1: Clone the repository
+```bash
+git clone https://github.com/viiika/Meissonic/
+cd Meissonic
+```
+
+### Step 2: Create virtual environment
+```bash
+conda create --name meissonic python
+conda activate meissonic
+pip install -r requirements.txt
+```
+
+### Step 3: Install diffusers
+```bash
+git clone https://github.com/huggingface/diffusers.git
+cd diffusers
+pip install -e .
+```
+
+## 💡 Inference Usage
+
+### Gradio Web UI
+
+```bash
+python app.py
+```
+
+### Command-line Interface
+
+#### Text-to-Image Generation
+
+```bash
+python inference.py --prompt "Your creative prompt here"
+```
+
+#### Inpainting and Outpainting
+
+```bash
+python inpaint.py --mode inpaint --input_image path/to/image.jpg
+python inpaint.py --mode outpaint --input_image path/to/image.jpg
+```
+
+### Advanced: FP8 Quantization
+
+Optimize performance with FP8 quantization:
+
+Requirements:
+- CUDA 12.4
+- PyTorch 2.4.1
+- TorchAO
+
+Note: Windows users install TorchAO using
+```shell
+pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cpu
+```
+
+Command-line inference
+```shell
+python inference_fp8.py --quantization fp8
+```
+
+Gradio for FP8 (Select Quantization Method in Advanced settings)
+```shell
+python app_fp8.py
+```
+
+#### Performance Benchmarks
+
+| Precision (Steps=64, Resolution=1024x1024) | Batch Size=1 (Avg. Time) | Memory Usage |
+|-------------------------------------------|--------------------------|--------------|
+| FP32                                      | 13.32s                   | 12GB         |
+| FP16                                      | 12.35s                   | 9.5GB        |
+| FP8                                       | 12.93s                   | 8.7GB        |
+
+## 🎨 Showcase
+
+<div align="center">
+  <img src="https://github.com/user-attachments/assets/b30a7912-5453-48ba-aff4-bfb547bbe626" width="320" alt="A pillow with a picture of a Husky on it.">
+  <p><i>"A pillow with a picture of a Husky on it."</i></p>
+</div>
+
+<div align="center">
+  <img src="https://github.com/user-attachments/assets/b23a1603-399d-40d6-8e16-c077d3d12a08" width="320" alt="A white coffee mug, a solid black background">
+  <p><i>"A white coffee mug, a solid black background"</i></p>
+</div>
+
+## 🎓 Training
+
+To train Meissonic, follow these steps:
+
+1. Install dependencies:
+   ```bash
+   cd train
+   pip install -r requirements.txt
+   ```
+
+2. Download the [Meissonic](https://huggingface.co/MeissonFlow/Meissonic) base model from Hugging Face.
+
+3. Prepare your dataset:
+   - Use the sample dataset: [MeissonFlow/splash](https://huggingface.co/datasets/MeissonFlow/lemon/resolve/main/0000.parquet)
+   - Or prepare your own dataset and dataset class following the format in line 100 in [dataset_utils.py](./train/dataset_utils.py) and line 656-680 in [train_meissonic.py](./train/train_meissonic.py)
+   - Modify [train.sh](./train/train.sh) with your dataset path
+
+4. Start training:
+   ```bash
+   bash train/train.sh
+   ```
+
+Note: For custom datasets, you'll likely need to implement your own dataset class.
+
+
+## 📚 Citation
+
+If you find this work helpful, please consider citing:
+
+```bibtex
+@article{bai2024meissonic,
+  title={Meissonic: Revitalizing Masked Generative Transformers for Efficient High-Resolution Text-to-Image Synthesis},
+  author={Bai, Jinbin and Ye, Tian and Chow, Wei and Song, Enxin and Chen, Qing-Guo and Li, Xiangtai and Dong, Zhen and Zhu, Lei and Yan, Shuicheng},
+  journal={arXiv preprint arXiv:2410.08261},
+  year={2024}
+}
+```
+
+## 🙏 Acknowledgements
+
+We thank the community and contributors for their invaluable support in developing Meissonic. We thank apolinario@multimodal.art for making Meissonic [Demo](https://huggingface.co/spaces/MeissonFlow/meissonic). We thank @NewGenAI and @飛鷹しずか@自称文系プログラマの勉強 for making YouTube tutorials. We thank @pprp for making fp8 and int4 quantization. We thank @camenduru for making [jupyter tutorial](https://github.com/camenduru/Meissonic-jupyter). We thank @chenxwh for making Replicate demo and api. We thank Collov Labs for reproducing [Monetico](https://huggingface.co/Collov-Labs/Monetico). We thank [Shitong et al.](https://arxiv.org/abs/2411.10781) for identifying effective design choices for enhancing visual quality.
+
+
+---
+
+<p align="center">
+  <a href="https://star-history.com/#viiika/Meissonic&Date">
+    <img src="https://api.star-history.com/svg?repos=viiika/Meissonic&type=Date" alt="Star History Chart">
+  </a>
+</p>
+
+<p align="center">
+  Made with ❤️ by the MeissonFlow Research
+</p>

Meissonic/app.py

@@ -0,0 +1,149 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from torchvision import transforms
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+import gradio as gr
+
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+model_path = "MeissonFlow/Meissonic"
+model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer")
+vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae")
+# text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path, subfolder="text_encoder")
+text_encoder = CLIPTextModelWithProjection.from_pretrained(  #more stable sampling for some cases
+            "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+        )
+tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer")
+scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler")
+pipe = Pipeline(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler)
+pipe.to(device)
+
+MAX_SEED = 2**32 - 1
+MAX_IMAGE_SIZE = 1024
+
+
+def generate_image(prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps, progress=gr.Progress(track_tqdm=True)):
+    if randomize_seed or seed == 0:
+        seed = torch.randint(0, MAX_SEED, (1,)).item()
+    torch.manual_seed(seed)
+    
+    image = pipe(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        height=height,
+        width=width,
+        guidance_scale=guidance_scale,
+        num_inference_steps=num_inference_steps
+    ).images[0]
+    
+    return image, seed
+
+# Default negative prompt
+default_negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+css = """
+#col-container {
+    margin: 0 auto;
+    max-width: 640px;
+}
+"""
+
+examples = [
+    "Modern Architecture render with pleasing aesthetics.",
+    "An image of a Pikachu wearing a birthday hat and playing guitar.",
+    "A statue of a lion stands in front of a building.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "A metal sculpture of a deer with antlers.",
+    "A bronze statue of an owl with its wings spread.",
+    "A white table with a vase of flowers and a cup of coffee on top of it.",
+    "A woman stands on a dock in the fog.",
+    "A lion's head is shown in a grayscale image.",
+    "A sculpture of a Greek woman head with a headband and a head of hair."
+]
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown("# Meissonic Text-to-Image Generator")
+        with gr.Row():
+            prompt = gr.Text(
+                label="Prompt",
+                show_label=False,
+                max_lines=1,
+                placeholder="Enter your prompt",
+                container=False,
+            )
+            run_button = gr.Button("Run", scale=0, variant="primary")
+        result = gr.Image(label="Result", show_label=False)
+        with gr.Accordion("Advanced Settings", open=False):
+            negative_prompt = gr.Text(
+                label="Negative prompt",
+                max_lines=1,
+                placeholder="Enter a negative prompt",
+                value=default_negative_prompt,
+            )
+            seed = gr.Slider(
+                label="Seed",
+                minimum=0,
+                maximum=MAX_SEED,
+                step=1,
+                value=0,
+            )
+            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+            with gr.Row():
+                width = gr.Slider(
+                    label="Width",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=1024,
+                )
+                height = gr.Slider(
+                    label="Height",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=1024,
+                )
+            with gr.Row():
+                guidance_scale = gr.Slider(
+                    label="Guidance scale",
+                    minimum=0.0,
+                    maximum=20.0,
+                    step=0.1,
+                    value=9.0,
+                )
+                num_inference_steps = gr.Slider(
+                    label="Number of inference steps",
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    value=64,
+                )
+        gr.Examples(examples=examples, inputs=[prompt])
+    gr.on(
+        triggers=[run_button.click, prompt.submit],
+        fn=generate_image,
+        inputs=[
+            prompt,
+            negative_prompt,
+            seed,
+            randomize_seed,
+            width,
+            height,
+            guidance_scale,
+            num_inference_steps,
+        ],
+        outputs=[result, seed],
+    )
+
+demo.launch()

Meissonic/app_Monetico.py

@@ -0,0 +1,151 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from torchvision import transforms
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+import gradio as gr
+import spaces 
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+dtype = torch.bfloat16
+
+model_path = "Collov-Labs/Monetico"
+
+model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", torch_dtype=dtype)
+vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", torch_dtype=dtype)
+text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path, subfolder="text_encoder", torch_dtype=dtype) # better for Monetico
+# text_encoder = CLIPTextModelWithProjection.from_pretrained(  #more stable sampling for some cases
+#             "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", torch_dtype=dtype
+#         )
+tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer", torch_dtype=dtype)
+scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler", torch_dtype=dtype)
+pipe = Pipeline(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler)
+pipe.to(device)
+
+MAX_SEED = 2**32 - 1
+MAX_IMAGE_SIZE = 512
+
+@spaces.GPU
+def generate_image(prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps, progress=gr.Progress(track_tqdm=True)):
+    if randomize_seed or seed == 0:
+        seed = torch.randint(0, MAX_SEED, (1,)).item()
+    torch.manual_seed(seed)
+    
+    image = pipe(
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        height=height,
+        width=width,
+        guidance_scale=guidance_scale,
+        num_inference_steps=num_inference_steps
+    ).images[0]
+    
+    return image, seed
+
+# Default negative prompt
+default_negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+css = """
+#col-container {
+    margin: 0 auto;
+    max-width: 640px;
+}
+"""
+
+examples = [
+    "Modern Architecture render with pleasing aesthetics.",
+    "An image of a Pikachu wearing a birthday hat and playing guitar.",
+    "A statue of a lion stands in front of a building.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "A metal sculpture of a deer with antlers.",
+    "A bronze statue of an owl with its wings spread.",
+    "A white table with a vase of flowers and a cup of coffee on top of it.",
+    "A woman stands on a dock in the fog.",
+    "A lion's head is shown in a grayscale image.",
+    "A sculpture of a Greek woman head with a headband and a head of hair."
+]
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown("# Monetico Text-to-Image Generator")
+        with gr.Row():
+            prompt = gr.Text(
+                label="Prompt",
+                show_label=False,
+                max_lines=1,
+                placeholder="Enter your prompt",
+                container=False,
+            )
+            run_button = gr.Button("Run", scale=0, variant="primary")
+        result = gr.Image(label="Result", show_label=False)
+        with gr.Accordion("Advanced Settings", open=False):
+            negative_prompt = gr.Text(
+                label="Negative prompt",
+                max_lines=1,
+                placeholder="Enter a negative prompt",
+                value=default_negative_prompt,
+            )
+            seed = gr.Slider(
+                label="Seed",
+                minimum=0,
+                maximum=MAX_SEED,
+                step=1,
+                value=0,
+            )
+            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+            with gr.Row():
+                width = gr.Slider(
+                    label="Width",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=512,
+                )
+                height = gr.Slider(
+                    label="Height",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=512,
+                )
+            with gr.Row():
+                guidance_scale = gr.Slider(
+                    label="Guidance scale",
+                    minimum=0.0,
+                    maximum=20.0,
+                    step=0.1,
+                    value=9.0,
+                )
+                num_inference_steps = gr.Slider(
+                    label="Number of inference steps",
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    value=48,
+                )
+        gr.Examples(examples=examples, inputs=[prompt])
+    gr.on(
+        triggers=[run_button.click, prompt.submit],
+        fn=generate_image,
+        inputs=[
+            prompt,
+            negative_prompt,
+            seed,
+            randomize_seed,
+            width,
+            height,
+            guidance_scale,
+            num_inference_steps,
+        ],
+        outputs=[result, seed],
+    )
+
+demo.launch()

Meissonic/app_fp8.py

@@ -0,0 +1,223 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+import gradio as gr
+import time
+from torchao.quantization.quant_api import (
+    quantize_,
+    float8_weight_only,
+)
+
+device = 'cuda'
+
+def get_quantization_method(method):
+    quantization_methods = {
+        'fp8': lambda: float8_weight_only(),
+        'none': None
+    }
+    return quantization_methods.get(method, None)
+
+def load_models(quantization_method='none'):
+    model_path = "MeissonFlow/Meissonic"
+    dtype = torch.float16
+    model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", torch_dtype=dtype)
+    vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", torch_dtype=dtype)
+    text_encoder = CLIPTextModelWithProjection.from_pretrained(
+        "laion/CLIP-ViT-H-14-laion2B-s32B-b79K",
+        torch_dtype=dtype
+    )
+    tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer")
+    scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler")
+    
+    if quantization_method != 'none':
+        quant_method = get_quantization_method(quantization_method)
+        if quant_method:
+            quantize_(model, quant_method())
+
+    pipe = Pipeline(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler)
+    return pipe.to(device)
+
+# Global variable to store the pipeline
+global_pipe = None
+current_quantization = 'none'
+
+def initialize_pipeline(quantization):
+    global global_pipe, current_quantization
+    if global_pipe is None or current_quantization != quantization:
+        global_pipe = load_models(quantization)
+        current_quantization = quantization
+    return global_pipe
+
+def generate_images(prompt, negative_prompt, seed, randomize_seed, width, height, 
+                   guidance_scale, num_inference_steps, quantization_method, batch_size=1, 
+                   progress=gr.Progress(track_tqdm=True)):
+    if randomize_seed or seed == 0:
+        seed = torch.randint(0, MAX_SEED, (1,)).item()
+    torch.manual_seed(seed)
+    
+    # Initialize or update pipeline if needed
+    pipe = initialize_pipeline(quantization_method)
+    
+    # Reset CUDA memory stats
+    torch.cuda.reset_peak_memory_stats()
+    start_time = time.time()
+    
+    # Handle batch generation
+    if isinstance(prompt, str):
+        prompts = [prompt] * batch_size
+    else:
+        prompts = prompt[:batch_size]
+    
+    images = pipe(
+        prompt=prompts,
+        negative_prompt=[negative_prompt] * batch_size,
+        height=height,
+        width=width,
+        guidance_scale=guidance_scale,
+        num_inference_steps=num_inference_steps
+    ).images
+    
+    # Calculate performance metrics
+    inference_time = time.time() - start_time
+    memory_used = torch.cuda.max_memory_reserved() / (1024 ** 3)  # Convert to GB
+    
+    performance_info = f"""
+    Inference Time: {inference_time:.2f} seconds
+    Memory Used: {memory_used:.2f} GB
+    Quantization: {quantization_method}
+    """
+    
+    return images[0] if batch_size == 1 else images, seed, performance_info
+
+MAX_SEED = 2**32 - 1
+MAX_IMAGE_SIZE = 1024
+default_negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+examples = [
+    "Two actors are posing for a pictur with one wearing a black and white face paint.",
+    "A large body of water with a rock in the middle and mountains in the background.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "The sun is setting over a city skyline with a river in the foreground.",
+    "A black and white cat with blue eyes.", 
+    "Three boats in the ocean with a rainbow in the sky.", 
+    "A robot playing the piano.",
+    "A cat wearing a hat.",
+    "A dog in a jungle."
+]
+
+css = """
+#col-container {
+    margin: 0 auto;
+    max-width: 640px;
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown("# Meissonic Text-to-Image Generator (with FP8 Support)")
+        
+        with gr.Row():
+            prompt = gr.Text(
+                label="Prompt",
+                show_label=False,
+                max_lines=1,
+                placeholder="Enter your prompt",
+                container=False,
+            )
+            run_button = gr.Button("Run", scale=0, variant="primary")
+        
+        result = gr.Image(label="Result", show_label=False)
+        performance_info = gr.Textbox(label="Performance Metrics", lines=4)
+        
+        with gr.Accordion("Advanced Settings", open=False):
+            quantization = gr.Radio(
+                choices=['none', 'fp8'],
+                value='none',
+                label="Quantization Method",
+            )
+            negative_prompt = gr.Text(
+                label="Negative prompt",
+                max_lines=1,
+                placeholder="Enter a negative prompt",
+                value=default_negative_prompt,
+            )
+            seed = gr.Slider(
+                label="Seed",
+                minimum=0,
+                maximum=MAX_SEED,
+                step=1,
+                value=0,
+            )
+            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+            
+            with gr.Row():
+                width = gr.Slider(
+                    label="Width",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=1024,
+                )
+                height = gr.Slider(
+                    label="Height",
+                    minimum=256,
+                    maximum=MAX_IMAGE_SIZE,
+                    step=32,
+                    value=1024,
+                )
+            
+            with gr.Row():
+                guidance_scale = gr.Slider(
+                    label="Guidance scale",
+                    minimum=0.0,
+                    maximum=20.0,
+                    step=0.1,
+                    value=9.0,
+                )
+                num_inference_steps = gr.Slider(
+                    label="Number of inference steps",
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    value=64,
+                )
+            
+            batch_size = gr.Slider(
+                label="Batch Size",
+                minimum=1,
+                maximum=8,
+                step=1,
+                value=1,
+            )
+        
+        gr.Examples(examples=examples, inputs=[prompt])
+    
+    gr.on(
+        triggers=[run_button.click, prompt.submit],
+        fn=generate_images,
+        inputs=[
+            prompt,
+            negative_prompt,
+            seed,
+            randomize_seed,
+            width,
+            height,
+            guidance_scale,
+            num_inference_steps,
+            quantization,
+            batch_size,
+        ],
+        outputs=[result, seed, performance_info],
+    )
+
+demo.launch()

Meissonic/assets/demos.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d14191e0b8e9fdf4cb3a7199cf36554e60e456cdeba11509d305a8201e6b131
+size 2476203

Meissonic/assets/inpaint/cases.json

@@ -0,0 +1,20 @@
+[
+  {
+    "input":"./assets/inpaint/_Rh_zxIUWXA.jpg",
+    "mask": "./assets/inpaint/_Rh_zxIUWXA.png",
+    "prompt": "A woman with short hair wears a silver gas mask.",
+    "negative_prompts": null
+  },
+  {
+    "input":"./assets/inpaint/0eKR4M2uuL8.jpg",
+    "mask": "./assets/inpaint/0eKR4M2uuL8.png",
+    "prompt": "A stylish dog wearing sunglasses.",
+    "negative_prompts": null
+  },
+  {
+    "input":"./assets/inpaint/__Owak0IgJk.jpg",
+    "mask": "./assets/inpaint/__Owak0IgJk.png",
+    "prompt": "A woman wearing a white suspender skirt is sitting",
+    "negative_prompts": null
+  }
+]

Meissonic/assets/outpaint/cases.json

@@ -0,0 +1,20 @@
+[
+  {
+    "input":"./assets/outpaint/__G2yFuW7jQ.jpg",
+    "mask": "./assets/outpaint/__G2yFuW7jQ.png",
+    "prompt": "fall mountains",
+    "negative_prompts": "The artwork avoids the pitfalls of bad art, such as ugly and deformed eyes and faces, poorly drawn, blurry, and disfigured bodies with extra limbs and close-ups that look weird. It also avoids other common issues such as watermarking, text errors, missing fingers or digits, cropping, poor quality, and JPEG artifacts. The artwork is free of signature or watermark and avoids framing issues.The hands are not deformed, the eyes are not disfigured, and there areno extra bodies or limbs. The artwork is not blurry, out of focus, or poorly drawn, and the proportions are not bad or deformed. There are no mutations, missing limbs, or floating or disconnected limbs. The hands and neck are not malformed, and there are no extra heads or out-of-frame elements. The artwork is not low-res or disgusting and is a well-drawn, highly detailed, and beautiful rendering."
+  },
+  {
+    "input":"./assets/outpaint/__G2yFuW7jQ.jpg",
+    "mask": "./assets/outpaint/__G2yFuW7jQ.png",
+    "prompt": "Rocket launch site",
+    "negative_prompts": "The artwork avoids the pitfalls of bad art, such as ugly and deformed eyes and faces, poorly drawn, blurry, and disfigured bodies with extra limbs and close-ups that look weird. It also avoids other common issues such as watermarking, text errors, missing fingers or digits, cropping, poor quality, and JPEG artifacts. The artwork is free of signature or watermark and avoids framing issues.The hands are not deformed, the eyes are not disfigured, and there areno extra bodies or limbs. The artwork is not blurry, out of focus, or poorly drawn, and the proportions are not bad or deformed. There are no mutations, missing limbs, or floating or disconnected limbs. The hands and neck are not malformed, and there are no extra heads or out-of-frame elements. The artwork is not low-res or disgusting and is a well-drawn, highly detailed, and beautiful rendering."
+  },
+  {
+    "input":"./assets/outpaint/__G2yFuW7jQ.jpg",
+    "mask": "./assets/outpaint/__G2yFuW7jQ.png",
+    "prompt": "Volcano",
+    "negative_prompts": "The artwork avoids the pitfalls of bad art, such as ugly and deformed eyes and faces, poorly drawn, blurry, and disfigured bodies with extra limbs and close-ups that look weird. It also avoids other common issues such as watermarking, text errors, missing fingers or digits, cropping, poor quality, and JPEG artifacts. The artwork is free of signature or watermark and avoids framing issues.The hands are not deformed, the eyes are not disfigured, and there areno extra bodies or limbs. The artwork is not blurry, out of focus, or poorly drawn, and the proportions are not bad or deformed. There are no mutations, missing limbs, or floating or disconnected limbs. The hands and neck are not malformed, and there are no extra heads or out-of-frame elements. The artwork is not low-res or disgusting and is a well-drawn, highly detailed, and beautiful rendering."
+  }
+]

Meissonic/cog.yaml

@@ -0,0 +1,29 @@
+# Configuration for Cog ⚙️
+# Reference: https://cog.run/yaml
+
+build:
+  # set to true if your model requires a GPU
+  gpu: true
+
+  # a list of ubuntu apt packages to install
+  system_packages:
+    - "libgl1-mesa-glx"
+    - "libglib2.0-0"
+
+  # python version in the form '3.11' or '3.11.4'
+  python_version: "3.11"
+
+  # a list of packages in the format <package-name>==<version>
+  python_packages:
+    - torch
+    - torchvision
+    - git+https://github.com/huggingface/diffusers.git
+    - accelerate
+    - transformers
+
+  # commands run after the environment is setup
+  run:
+    - curl -o /usr/local/bin/pget -L "https://github.com/replicate/pget/releases/download/v0.8.2/pget_linux_x86_64" && chmod +x /usr/local/bin/pget
+
+# predict.py defines how predictions are run on your model
+predict: "predict.py:Predictor"

Meissonic/cosmos_test_output/metrics_video_0.txt

@@ -0,0 +1,12 @@
+Video Index: 0
+Video Path: 000/000/000/0.mp4
+Caption: In the video, a man is seen in a living room setting, standing in front of a window with blinds. He is wearing a black sweater and appears to be in the middle of a conversation. The room is dimly lit, with a lamp providing a soft glow in the background. The man's expression is serious, suggesting that the conversation is of importance. The overall style of the video is realistic and naturalistic, capturing a candid moment in the man's life.
+
+=== Metrics ===
+Average PSNR: 27.54 dB
+Average MSE: 0.001764
+Average SSIM: 0.9779
+
+Per-frame PSNR: [26.747089385986328, 27.265975952148438, 27.32347297668457, 27.352922439575195, 27.334339141845703, 27.782726287841797, 27.661243438720703, 27.803525924682617, 27.705425262451172, 27.679603576660156, 27.297304153442383, 27.51146125793457, 27.4649658203125, 27.89719581604004, 27.753822326660156, 27.86109161376953, 27.80060577392578]
+Per-frame MSE: [0.002114907605573535, 0.0018767336150631309, 0.0018520501907914877, 0.001839534263126552, 0.001847422681748867, 0.001666200696490705, 0.0017134671797975898, 0.0016582406824454665, 0.0016961240908131003, 0.0017062382539734244, 0.0018632437568157911, 0.001773593365214765, 0.001792682334780693, 0.0016228572931140661, 0.0016773275565356016, 0.0016364054754376411, 0.0016593559412285686]
+Per-frame SSIM: [0.9738484025001526, 0.9765720963478088, 0.9768512845039368, 0.9769563674926758, 0.9766926765441895, 0.9789631962776184, 0.9783727526664734, 0.9790931344032288, 0.9786423444747925, 0.9784184694290161, 0.9765286445617676, 0.9777430295944214, 0.9775411486625671, 0.9799171090126038, 0.9794126749038696, 0.9798620343208313, 0.9795750379562378]

Meissonic/cosmos_test_output/metrics_video_1.txt

@@ -0,0 +1,12 @@
+Video Index: 1
+Video Path: 000/000/001/1.mp4
+Caption: The video shows a man standing next to a purple van with a floral design on the side. The man is wearing a black t-shirt and jeans, and he is smiling and waving his hands in the air. The van has pink rims and a black roof rack. The van is parked in front of a building with a glass door. The man appears to be happy and excited about the van. The video is likely a short clip of a man showing off his van.
+
+=== Metrics ===
+Average PSNR: 25.14 dB
+Average MSE: 0.003232
+Average SSIM: 0.9700
+
+Per-frame PSNR: [29.570905685424805, 25.845619201660156, 24.151002883911133, 24.53882598876953, 26.607555389404297, 23.609159469604492, 23.5848445892334, 24.532224655151367, 26.290340423583984, 23.606443405151367, 23.633737564086914, 24.562894821166992, 26.255611419677734, 24.259323120117188, 25.643463134765625, 25.491649627685547]
+Per-frame MSE: [0.0011038482189178467, 0.002602784661576152, 0.003845029277727008, 0.0035165559966117144, 0.0021839593537151814, 0.004355963785201311, 0.004380417056381702, 0.003521904582157731, 0.0023494488559663296, 0.004358689300715923, 0.004331381060183048, 0.0034971192944794893, 0.0023683111649006605, 0.0037503137718886137, 0.0027268033009022474, 0.002823807764798403]
+Per-frame SSIM: [0.9893906712532043, 0.974087119102478, 0.9622812867164612, 0.9658014178276062, 0.9791845083236694, 0.957465648651123, 0.957618772983551, 0.9664595127105713, 0.977811872959137, 0.9590921998023987, 0.960818350315094, 0.9692971110343933, 0.9799112677574158, 0.9681466817855835, 0.9765862822532654, 0.9759609699249268]

Meissonic/cosmos_test_output/metrics_video_2.txt

@@ -0,0 +1,12 @@
+Video Index: 2
+Video Path: 000/000/002/2.mp4
+Caption: The video is a news segment featuring a man in a red baseball cap and a blue vest, standing in front of a statue of a soldier and two children. The man appears to be a veteran, as indicated by the cap and the context of the event. The event is an honorary ceremony for lost submarines and submarine veterans, taking place near the World Peace Bell in Newport. The news segment is titled "Connected to the Community" and is scheduled to air at 11:10 PM on ABC 9. The style of the video is informative and respectful, focusing on the man and the event, with a clear and concise presentation of the details.
+
+=== Metrics ===
+Average PSNR: 22.09 dB
+Average MSE: 0.006399
+Average SSIM: 0.9607
+
+Per-frame PSNR: [24.496965408325195, 22.367679595947266, 22.21709442138672, 22.679195404052734, 23.883594512939453, 22.220516204833984, 22.20623207092285, 21.4675350189209, 22.316797256469727, 19.425098419189453, 21.102333068847656, 21.321147918701172, 23.025981903076172, 21.053565979003906, 21.95743179321289, 21.684494018554688]
+Per-frame MSE: [0.0035506151616573334, 0.005797383841127157, 0.006001925095915794, 0.005396105814725161, 0.0040892185643315315, 0.00599720049649477, 0.006016954779624939, 0.007132581900805235, 0.005865707993507385, 0.011415375396609306, 0.007758304942399263, 0.007377093657851219, 0.00498197739943862, 0.007845907472074032, 0.006371723022311926, 0.0067850141786038876]
+Per-frame SSIM: [0.9770643711090088, 0.9623730778694153, 0.9605475068092346, 0.9618802070617676, 0.9713600277900696, 0.9565339088439941, 0.9568989872932434, 0.9560506939888, 0.9673117399215698, 0.9364117383956909, 0.9567262530326843, 0.9589394927024841, 0.9706904888153076, 0.9546973705291748, 0.9623250365257263, 0.9610125422477722]

Meissonic/cosmos_test_output/metrics_video_3.txt

@@ -0,0 +1,12 @@
+Video Index: 3
+Video Path: 000/000/003/3.mp4
+Caption: The video features a man in a pink shirt and a black bucket hat, wearing glasses and a necklace. He is holding a spoon and making a playful face, as if he is about to eat something. The background shows a lush garden with trees and a wooden structure. The man's expression and the spoon suggest that he is about to taste something, possibly food. The overall style of the video is casual and fun, with a focus on the man's reaction to the food.
+
+=== Metrics ===
+Average PSNR: 26.22 dB
+Average MSE: 0.002459
+Average SSIM: 0.9856
+
+Per-frame PSNR: [27.509328842163086, 26.409242630004883, 25.4619140625, 25.407241821289062, 26.446935653686523, 23.73136329650879, 25.60137176513672, 26.993793487548828, 28.306987762451172, 25.729787826538086, 25.27326774597168, 26.266807556152344, 27.462078094482422, 25.950550079345703, 26.63888168334961, 26.327953338623047]
+Per-frame MSE: [0.001774463220499456, 0.0022859980817884207, 0.002843207446858287, 0.0028792270459234715, 0.0022662426345050335, 0.004235099535435438, 0.002753359731286764, 0.0019981153309345245, 0.0014767315005883574, 0.002673137467354536, 0.00296943006105721, 0.0023622140288352966, 0.0017938758246600628, 0.0025406500790268183, 0.0021682626102119684, 0.002329188399016857]
+Per-frame SSIM: [0.9894547462463379, 0.9864593744277954, 0.983065664768219, 0.9832437634468079, 0.9867878556251526, 0.9748696088790894, 0.9840085506439209, 0.9884393215179443, 0.991378664970398, 0.9843630194664001, 0.9830930829048157, 0.986225962638855, 0.989523708820343, 0.9850807189941406, 0.9871661067008972, 0.9861734509468079]

Meissonic/inference.py

@@ -0,0 +1,65 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from torchvision import transforms
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+
+device = 'cuda'
+
+model_path = "MeissonFlow/Meissonic"
+model = Transformer2DModel.from_pretrained(model_path,subfolder="transformer",)
+vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", )
+# text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path,subfolder="text_encoder",)
+text_encoder = CLIPTextModelWithProjection.from_pretrained(   #using original text enc for stable sampling
+            "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+        )
+tokenizer = CLIPTokenizer.from_pretrained(model_path,subfolder="tokenizer",)
+scheduler = Scheduler.from_pretrained(model_path,subfolder="scheduler",)
+pipe=Pipeline(vq_model, tokenizer=tokenizer,text_encoder=text_encoder,transformer=model,scheduler=scheduler)
+
+pipe = pipe.to(device)
+
+steps = 64
+CFG = 9
+resolution = 1024 
+negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+prompts = [
+    "Two actors are posing for a pictur with one wearing a black and white face paint.",
+    "A large body of water with a rock in the middle and mountains in the background.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "A statue of a man with a crown on his head.",
+    "A man in a yellow wet suit is holding a big black dog in the water.",
+    "A white table with a vase of flowers and a cup of coffee on top of it.",
+    "A woman stands on a dock in the fog.",
+    "A woman is standing next to a picture of another woman."
+]
+
+batched_generation = False
+num_images = len(prompts) if batched_generation else 1
+
+images = pipe(
+    prompt=prompts[:num_images], 
+    negative_prompt=[negative_prompt] * num_images,
+    height=resolution,
+    width=resolution,
+    guidance_scale=CFG,
+    num_inference_steps=steps
+    ).images
+
+output_dir = "./output"
+os.makedirs(output_dir, exist_ok=True)
+for i, prompt in enumerate(prompts[:num_images]):
+    sanitized_prompt = prompt.replace(" ", "_")
+    file_path = os.path.join(output_dir, f"{sanitized_prompt}_{resolution}_{steps}_{CFG}.png")
+    images[i].save(file_path)
+    print(f"The {i+1}/{num_images} image is saved to {file_path}")

Meissonic/inference_fp16.py

@@ -0,0 +1,64 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from torchvision import transforms
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+
+device = 'cuda'
+dtype = torch.bfloat16
+model_path = "MeissonFlow/Meissonic"
+model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", torch_dtype=dtype)
+vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", torch_dtype=dtype)
+# text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path,subfolder="text_encoder", torch_dtype=dtype)
+text_encoder = CLIPTextModelWithProjection.from_pretrained(   #using original text enc for stable sampling
+            "laion/CLIP-ViT-H-14-laion2B-s32B-b79K",torch_dtype=dtype)
+tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer", torch_dtype=dtype)
+scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler")
+pipe=Pipeline(vq_model, tokenizer=tokenizer,text_encoder=text_encoder,transformer=model,scheduler=scheduler)
+
+pipe = pipe.to(device)
+
+steps = 64
+CFG = 9
+resolution = 1024 
+negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+prompts = [
+    "Two actors are posing for a pictur with one wearing a black and white face paint.",
+    "A large body of water with a rock in the middle and mountains in the background.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "A statue of a man with a crown on his head.",
+    "A man in a yellow wet suit is holding a big black dog in the water.",
+    "A white table with a vase of flowers and a cup of coffee on top of it.",
+    "A woman stands on a dock in the fog.",
+    "A woman is standing next to a picture of another woman."
+]
+
+batched_generation = False
+num_images = len(prompts) if batched_generation else 1
+
+images = pipe(
+    prompt=prompts[:num_images], 
+    negative_prompt=[negative_prompt] * num_images,
+    height=resolution,
+    width=resolution,
+    guidance_scale=CFG,
+    num_inference_steps=steps
+    ).images
+
+output_dir = "./output"
+os.makedirs(output_dir, exist_ok=True)
+for i, prompt in enumerate(prompts[:num_images]):
+    sanitized_prompt = prompt.replace(" ", "_")
+    file_path = os.path.join(output_dir, f"{sanitized_prompt}_{resolution}_{steps}_{CFG}.png")
+    images[i].save(file_path)
+    print(f"The {i+1}/{num_images} image is saved to {file_path}")

Meissonic/inference_fp16_Monetico.py

@@ -0,0 +1,64 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from torchvision import transforms
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+
+device = 'cuda'
+dtype = torch.bfloat16
+model_path = "Collov-Labs/Monetico"
+model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", torch_dtype=dtype)
+vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", torch_dtype=dtype)
+text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path, subfolder="text_encoder", torch_dtype=dtype) # better for Monetico
+# text_encoder = CLIPTextModelWithProjection.from_pretrained(  #more stable sampling for some cases
+#             "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", torch_dtype=dtype
+#         )
+tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer", torch_dtype=dtype)
+scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler", torch_dtype=dtype)
+pipe = Pipeline(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler)
+pipe.to(device)
+
+steps = 48
+CFG = 9
+resolution = 512 
+negative_prompt = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+prompts = [
+    "Two actors are posing for a pictur with one wearing a black and white face paint.",
+    "A large body of water with a rock in the middle and mountains in the background.",
+    "A white and blue coffee mug with a picture of a man on it.",
+    "A statue of a man with a crown on his head.",
+    "A man in a yellow wet suit is holding a big black dog in the water.",
+    "A white table with a vase of flowers and a cup of coffee on top of it.",
+    "A woman stands on a dock in the fog.",
+    "A woman is standing next to a picture of another woman."
+]
+
+batched_generation = False
+num_images = len(prompts) if batched_generation else 1
+
+images = pipe(
+    prompt=prompts[:num_images], 
+    negative_prompt=[negative_prompt] * num_images,
+    height=resolution,
+    width=resolution,
+    guidance_scale=CFG,
+    num_inference_steps=steps
+    ).images
+
+output_dir = "./output"
+os.makedirs(output_dir, exist_ok=True)
+for i, prompt in enumerate(prompts[:num_images]):
+    sanitized_prompt = prompt.replace(" ", "_")
+    file_path = os.path.join(output_dir, f"{sanitized_prompt}_{resolution}_{steps}_{CFG}.png")
+    images[i].save(file_path)
+    print(f"The {i+1}/{num_images} image is saved to {file_path}")

Meissonic/inference_fp8.py

@@ -0,0 +1,103 @@
+import os
+import sys
+sys.path.append("./")
+
+import torch
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+import time
+import argparse
+
+from torchao.quantization.quant_api import (
+    quantize_,
+    float8_weight_only, # A8W8 FP8
+)
+
+device = 'cuda'
+
+def get_quantization_method(method):
+    quantization_methods = {
+        'fp8': lambda: float8_weight_only(),
+    }
+    return quantization_methods.get(method, None)
+
+def load_models(quantization_method=None):
+    model_path = "MeissonFlow/Meissonic"
+    dtype = torch.float16
+    model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", torch_dtype=dtype)
+    vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", torch_dtype=dtype)
+    text_encoder = CLIPTextModelWithProjection.from_pretrained(
+        "laion/CLIP-ViT-H-14-laion2B-s32B-b79K",
+        torch_dtype=dtype
+    )
+    tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer")
+    scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler")
+    
+    if quantization_method:
+        quant_method = get_quantization_method(quantization_method)
+        if quant_method:
+            quantize_(model, quant_method())
+        else:
+            print(f"Unsupported quantization method: {quantization_method}")
+
+    
+    pipe = Pipeline(vq_model, tokenizer=tokenizer, text_encoder=text_encoder, transformer=model, scheduler=scheduler)
+    return pipe.to(device)
+
+def run_inference(pipe, prompt, negative_prompt, resolution, cfg, steps):
+    return pipe(prompt=prompt, negative_prompt=negative_prompt, height=resolution, width=resolution, guidance_scale=cfg, num_inference_steps=steps).images[0]
+
+def main(quantization_method):
+    steps = 64
+    CFG = 9
+    resolution = 1024 
+    negative_prompts = "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+    prompts = [
+        "Two actors are posing for a pictur with one wearing a black and white face paint.",
+        "A large body of water with a rock in the middle and mountains in the background.",
+        "A white and blue coffee mug with a picture of a man on it.",
+        "The sun is setting over a city skyline with a river in the foreground.",
+        "A black and white cat with blue eyes.", 
+        "Three boats in the ocean with a rainbow in the sky.", 
+        "A robot playing the piano.",
+        "A cat wearing a hat.",
+        "A dog in a jungle.",
+    ]
+
+    output_dir = "./output"
+    os.makedirs(output_dir, exist_ok=True)
+
+    pipe = load_models(quantization_method)
+    start_time = time.time()
+    total_memory_used = 0
+    for i, prompt in enumerate(prompts):
+        torch.cuda.reset_peak_memory_stats()
+        image_start_time = time.time()
+        image = run_inference(pipe, prompt, negative_prompts, resolution, CFG, steps)
+        image_end_time = time.time()
+        image.save(os.path.join(output_dir, f"{prompt[:10]}_{resolution}_{steps}_{CFG}_{quantization_method}.png"))
+        
+        memory_used = torch.cuda.max_memory_reserved() / (1024 ** 3)  # Convert to GB
+        total_memory_used += memory_used
+        
+        print(f"Image {i+1} time: {image_end_time - image_start_time:.2f} seconds")
+        print(f"Image {i+1} max memory used: {memory_used:.2f} GB")
+    
+    total_time = time.time() - start_time
+    avg_memory_used = total_memory_used / len(prompts)
+    print(f"Total inference time ({quantization_method}): {total_time:.2f} seconds")
+    print(f"Average memory used per image: {avg_memory_used:.2f} GB")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run inference with specified quantization method.")
+    parser.add_argument("--quantization", type=str, choices=['fp8'], 
+                        help="Quantization method to use")
+    args = parser.parse_args()
+    main(args.quantization)

Meissonic/inpaint.py

@@ -0,0 +1,55 @@
+import os
+import sys
+sys.path.append("./")
+
+import argparse
+import json
+from PIL import Image
+from src.transformer import Transformer2DModel
+from src.pipeline_inpaint import InpaintPipeline
+from src.scheduler import Scheduler
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+
+def get_parse_args():
+    parser = argparse.ArgumentParser(description="Meissonic Inpaint and Outpaint")
+    parser.add_argument("--mode", type=str,default="inpaint", choices=["inpaint", "outpaint"], help="Inpaint or Outpaint")
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    args = get_parse_args()
+    device = 'cuda'
+
+    model_path = "MeissonFlow/Meissonic"
+    model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer", )
+    vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae", )
+    # text_encoder = CLIPTextModelWithProjection.from_pretrained(model_path,subfolder="text_encoder",)
+    text_encoder = CLIPTextModelWithProjection.from_pretrained(  # using original text enc for stable sampling
+        "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+    )
+    tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer", )
+    scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler", )
+    
+    pipe=InpaintPipeline(vq_model, tokenizer=tokenizer,text_encoder=text_encoder,transformer=model,scheduler=scheduler)
+    pipe = pipe.to(device)
+
+    with open(f"./assets/{args.mode}/cases.json", 'r', encoding='utf-8') as file:
+        cases = json.load(file)
+    item = cases[0]
+
+    steps = 64
+    CFG = 9
+    resolution = 1024
+    negative_prompts = item["negative_prompts"] if "negative_prompts" in item.keys() else "worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark"
+
+    image = Image.open(item["input"]).resize((resolution, resolution)).convert("RGB")
+    mask = Image.open(item["mask"]).resize((resolution, resolution)).convert("RGB")
+
+    image = pipe(prompt=item["prompt"],negative_prompt=negative_prompts,image =image, mask_image =mask, guidance_scale=CFG, num_inference_steps=steps).images[0]
+
+    output_dir = "./output"
+    os.makedirs(output_dir, exist_ok=True)
+    image.save(os.path.join(output_dir, f"{item['prompt'][:10]}_{resolution}_{steps}_{CFG}.png"))

Meissonic/predict.py

@@ -0,0 +1,105 @@
+# Prediction interface for Cog ⚙️
+# https://cog.run/python
+
+import os
+import subprocess
+import time
+import torch
+from transformers import (
+    CLIPTextModelWithProjection,
+    CLIPTokenizer,
+)
+from diffusers import VQModel
+from cog import BasePredictor, Input, Path
+
+from src.transformer import Transformer2DModel
+from src.pipeline import Pipeline
+from src.scheduler import Scheduler
+
+
+MODEL_CACHE = "model_cache"
+MODEL_URL = (
+    f"https://weights.replicate.delivery/default/viiika/Meissonic/{MODEL_CACHE}.tar"
+)
+
+os.environ.update(
+    {
+        "HF_DATASETS_OFFLINE": "1",
+        "TRANSFORMERS_OFFLINE": "1",
+        "HF_HOME": MODEL_CACHE,
+        "TORCH_HOME": MODEL_CACHE,
+        "HF_DATASETS_CACHE": MODEL_CACHE,
+        "TRANSFORMERS_CACHE": MODEL_CACHE,
+        "HUGGINGFACE_HUB_CACHE": MODEL_CACHE,
+    }
+)
+
+
+def download_weights(url, dest):
+    start = time.time()
+    print("downloading url: ", url)
+    print("downloading to: ", dest)
+    subprocess.check_call(["pget", "-x", url, dest], close_fds=False)
+    print("downloading took: ", time.time() - start)
+
+
+class Predictor(BasePredictor):
+    def setup(self) -> None:
+        """Load the model into memory to make running multiple predictions efficient"""
+
+        if not os.path.exists(MODEL_CACHE):
+            download_weights(MODEL_URL, MODEL_CACHE)
+
+        model_path = f"{MODEL_CACHE}/MeissonFlow/Meissonic"
+        model = Transformer2DModel.from_pretrained(model_path, subfolder="transformer")
+        vq_model = VQModel.from_pretrained(model_path, subfolder="vqvae")
+        text_encoder = CLIPTextModelWithProjection.from_pretrained(  # more stable sampling for some cases
+            f"{MODEL_CACHE}/laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
+        )
+        tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder="tokenizer")
+        scheduler = Scheduler.from_pretrained(model_path, subfolder="scheduler")
+        self.pipe = Pipeline(
+            vq_model,
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            transformer=model,
+            scheduler=scheduler,
+        ).to("cuda")
+
+    def predict(
+        self,
+        prompt: str = Input(
+            description="Input prompt",
+            default="a photo of an astronaut riding a horse on mars",
+        ),
+        negative_prompt: str = Input(
+            description="Specify things to not see in the output",
+            default="worst quality, low quality, low res, blurry, distortion, watermark, logo, signature, text, jpeg artifacts, signature, sketch, duplicate, ugly, identifying mark",
+        ),
+        num_inference_steps: int = Input(
+            description="Number of denoising steps", ge=1, le=100, default=64
+        ),
+        guidance_scale: float = Input(
+            description="Scale for classifier-free guidance", ge=0, le=20, default=9
+        ),
+        seed: int = Input(
+            description="Random seed. Leave blank to randomize the seed", default=None
+        ),
+    ) -> Path:
+        """Run a single prediction on the model"""
+        if seed is None:
+            seed = int.from_bytes(os.urandom(2), "big")
+        print(f"Using seed: {seed}")
+        torch.manual_seed(seed)
+
+        image = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            height=1024,
+            width=1024,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+        ).images[0]
+        output_path = f"/tmp/out.png"
+        image.save(output_path)
+        return Path(output_path)

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+autoencoder.jit filter=lfs diff=lfs merge=lfs -text
+decoder.jit filter=lfs diff=lfs merge=lfs -text
+encoder.jit filter=lfs diff=lfs merge=lfs -text

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/README.md

@@ -0,0 +1,326 @@
+---
+license: other
+license_name: nvidia-open-model-license
+license_link: >-
+  https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf
+library_name: nemo
+---
+# **Cosmos Tokenizer**: A suite of image and video tokenizers
+
+[**Website**](https://research.nvidia.com/labs/dir/cosmos-tokenizer) | [**Code**](https://github.com/NVIDIA/Cosmos-Tokenizer) | [**Video**](https://youtu.be/Soy_myOfWIU)
+
+
+# Model Overview
+
+## Description:
+**Cosmos Tokenizer** is a suite of visual tokenizers for images and videos that delivers various compression rates while maintaining high reconstruction quality. Cosmos Tokenizer can serve as an effective and efficient building block in both diffusion-based and autoregressive models for image and video generation.
+
+
+Our tokenizers come in two types: **Continuous** (C) and **Discrete** (D), each with **Image** (I) and **Video** (V) variants:
+* Continuous tokenizers encode visual data into continuous latent embeddings, as shown in latent diffusion models like [Stable Diffusion](https://github.com/CompVis/stable-diffusion). These embeddings are suitable for models that generate data by sampling from continuous distributions. 
+* Discrete tokenizers encode visual data into discrete latent codes, mapping them into quantized indices, as seen in autoregressive transformers such as [VideoPoet](https://sites.research.google/videopoet/). This discretization is required for models that generate data by optimizing the cross-entropy loss, such as the GPT models.
+
+
+|                   | Continuous ( C )    | Discrete ( D )      |
+| ------------------|---------------------|---------------------|
+| **Images ( I )**        | Cosmos-Tokenizer-CI            | Cosmos-Tokenizer-DI            |
+| **Videos ( V )**        | Cosmos-Tokenizer-CV            | Cosmos-Tokenizer-DV            |
+
+
+Given an image or a video, Cosmos Tokenizer outputs either continuous latents or discrete tokens. Cosmos Tokenizer achieves spatial compression rates of 8x8 or 16x16 and temporal compression factors of 4x or 8x, resulting in a total compression factor of up to 2048x (=8x16x16). Cosmos Tokenizer delivers 8x more total compression than state-of-the-art (SOTA) methods while simultaneously maintaining higher image quality and running up to 12x faster than the best available SOTA tokenizers.
+
+**Model Developer**: NVIDIA
+
+## Model Versions
+
+The initial release (v1.0) of Cosmos Tokenizer includes the following tokenizers:
+* **Continuous Tokenizers**
+    * Continuous Image (CI) Tokenizer
+        * [Cosmos-Tokenizer-CI8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CI8x8) (8x8 spatial compression)
+        * [Cosmos-Tokenizer-CI16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-CI16x16) (16x16 spatial compression)
+    * Continuous Video (CV) Tokenizer
+        * [Cosmos-Tokenizer-CV4x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV4x8x8) (4x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-CV8x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV8x8x8) (8x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-CV8x16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV8x16x16) (8x temporal compression, 16x16 spatial compression)
+* **Discrete Tokenizers**
+    * Discrete Image (DI) Tokenizer
+        * [Cosmos-Tokenizer-DI8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DI8x8) (8x8 spatial compression)
+        * [Cosmos-Tokenizer-DI16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-DI16x16) (16x16 spatial compression)
+    * Discrete Video (DV) Tokenizer
+        * [Cosmos-Tokenizer-DV4x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV4x8x8) (4x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-DV8x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV8x8x8) (8x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-DV8x16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV8x16x16) (8x temporal compression, 16x16 spatial compression)
+
+
+### License/Terms of Use: 
+[NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+
+Under the NVIDIA Open Model License, NVIDIA confirms:
+
+* Models are commercially usable.
+* You are free to create and distribute Derivative Models.
+* NVIDIA does not claim ownership to any outputs generated using the Models or Derivative Models.
+
+## Model Architecture: 
+
+We designed Cosmos Tokenizer using a lightweight and computationally efficient architecture, featuring a temporally causal design. Specifically, we employ causal temporal convolution and causal temporal attention layers to preserve the natural temporal order of video frames, ensuring seamless tokenization of images and videos using a single unified network architecture. The encoder and decoder form a symmetrical pair, which are mirrors of each other. The encoder starts with a 2-level [Haar wavelet](https://link.springer.com/book/10.1007/978-3-319-04295-4) transform layer, which down-samples inputs by a factor of 4 in both spatial and temporal dimensions. Likewise, the decoder ends with an inverse wavelet transform. We employ the vanilla autoencoder (AE) formulation to model the latent space for continuous tokenizers. For discrete tokenizers, we adopt the [Finite-Scalar-Quantization](https://openreview.net/forum?id=8ishA3LxN8) (FSQ) as the latent space quantizer.
+
+![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/638fb8cf2380ffd99caf8c2a/gQH5n9iCEtqZc7uutUwdL.jpeg)
+
+
+
+## Input/Output Specifications
+
+### Encoder
+* **Input**
+    * **Types:** Images or Videos
+    * **Format:** RGB (Red, Green, Blue)
+    * **Resolution:** 
+        * Minimum: 256px (shorter side)
+        * Maximum: Up to 4K
+    * **Video Length:** Up to 8 seconds for 1080p videos (bounded by A100 80G GPU memory; higher resolutions will have shorter supported durations)
+
+* **Output**
+    * **Types:** Tokens
+        * Continuous Image/Video Tokenizers: Continuous value feature vectors
+        * Discrete Image/Video Tokenizers: Integer indices
+
+### Decoder
+* **Input**
+    * **Types:** Tokens from encoder
+
+* **Output**
+    * **Types:** Images or Videos (matching input type)
+    * **Format:** RGB (Red, Green, Blue)
+    * **Resolution:** Same as input resolution
+    * **Video Length:** Same as input video length
+
+## Software Integration (Required For NVIDIA Models Only):
+**Runtime Engine(s):** 
+* [Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer)
+* [NeMo](https://github.com/NVIDIA/NeMo) (please install the latest version from the GitHub main branch)
+
+**Supported Hardware Microarchitecture Compatibility:**
+* NVIDIA Ampere (e.g., A100)
+* NVIDIA Hopper (e.g., H100)
+
+Note: We have only tested Cosmos Tokenizer with BF16 precision on Ampere and Hopper GPUs. If you are using older versions of NVIDIA GPUs (e.g., NVIDIA Volta GPUs), you may need to switch to FP32 precision.
+
+
+**Operating System(s):**
+* Linux (We have not tested on other operating systems.)
+
+# Usage
+Inference Engines: 
+* [Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer) (PyTorch)
+* [NeMo](https://github.com/NVIDIA/NeMo)
+
+## Inference with `Cosmos-Tokenizer` (PyTorch)
+### Step-1: Installation of `Cosmos-Tokenizer` 
+Note: Currently, the `Cosmos-Tokenizer` code is only supported on Linux.
+
+- Please clone the `Cosmos-Tokenizer` from GitHub repo [github.com/NVIDIA/Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer).
+
+    ```bash
+    git clone https://github.com/NVIDIA/Cosmos-Tokenizer.git
+    cd Cosmos-Tokenizer
+    ```
+- Install dependencies
+
+    ```bash
+    pip3 install -r requirements.txt
+    apt-get install -y ffmpeg
+    ```
+
+- Preferably, you could build a docker image using our provided Dockerfile.
+    ```bash
+    docker build -t cosmos-docker -f Dockerfile.    
+    # You can run the container as:
+    docker run --gpus all -it --rm -v /home/${USER}:/home/${USER} \
+        --workdir ${PWD} cosmos-docker /bin/bash
+    ```
+
+### Step-2: Download Pre-trained Checkpoints
+- Create a local directory for the pre-trained checkpoints and download the 
+pre-trained checkpoints from HuggingFace.
+
+    ```python
+    from huggingface_hub import login, snapshot_download
+    import os
+    # You could get your Hugging Face token from https://huggingface.co/settings/tokens
+    login(token=<YOUT-HF-TOKEN>, add_to_git_credential=True)
+    # You could specify the tokenizers you want to download.
+    model_names = [
+            "Cosmos-Tokenizer-CI8x8",
+            "Cosmos-Tokenizer-CI16x16",
+            "Cosmos-Tokenizer-CV4x8x8",
+            "Cosmos-Tokenizer-CV8x8x8",
+            "Cosmos-Tokenizer-CV8x16x16",
+            "Cosmos-Tokenizer-DI8x8",
+            "Cosmos-Tokenizer-DI16x16",
+            "Cosmos-Tokenizer-DV4x8x8",
+            "Cosmos-Tokenizer-DV8x8x8",
+            "Cosmos-Tokenizer-DV8x16x16",
+    ]
+    for model_name in model_names:
+        hf_repo = "nvidia/" + model_name
+        local_dir = "pretrained_ckpts/" + model_name
+        os.makedirs(local_dir, exist_ok=True)
+        print(f"downloading {model_name} to {local_dir}...")
+        snapshot_download(repo_id=hf_repo, local_dir=local_dir)
+    ```
+
+- Under the ech checkpoint directory `pretrained_ckpts/<model-name>`, we provide the encoder, 
+decoder and the full autoencoder JIT models.
+
+    ```bash 
+    ├── pretrained_ckpts/   
+    │   ├── Cosmos-Tokenizer-DV8x8x8/
+    │   │   ├── encoder.jit
+    │   │   ├── decoder.jit
+    │   │   ├── autoencoder.jit
+    │   ...
+    ```
+
+### Step-3: Run Inference
+You can use the following example commands to encode and decode images or videos. For each, the same command works for both continuous and discrete tokenization. Simply provide the proper JIT-compiled ckpt to `checkpoint_enc`, `checkpoint_dec`, or the full autoencoder ckpt to `checkpoint`.
+
+```python
+import torch
+from cosmos_tokenizer.video_lib import CausalVideoTokenizer
+model_name = "Cosmos-Tokenizer-DV4x8x8"
+input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16)
+encoder = CausalVideoTokenizer(checkpoint_enc=f'pretrained_ckpts/{model_name}/encoder.jit')
+(indices, codes) = encoder.encode(input_tensor)
+torch.testing.assert_close(indices.shape, (1, 3, 64, 64))
+torch.testing.assert_close(codes.shape, (1, 6, 3, 64, 64))
+
+# The input tensor can be reconstructed by the decoder as:
+decoder = CausalVideoTokenizer(checkpoint_dec=f'pretrained_ckpts/{model_name}/decoder.jit')
+reconstructed_tensor = decoder.decode(indices)
+torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape)
+```
+
+The `indices` will have the shape `(1, 3, 64, 64)` and contain integral values in the range `[1..64K]`, where the first of the three integral maps represents the first frame. 
+The `codes` will contain the pre-quantization continuous latent with shape `(1, 6, 3, 64, 64)`, where C=6 represents the number of FSQ levels.
+
+**Note**: More inference usage commands, including both TorchScript (JIT) and PyTorch Inference APIs on real images and videos, can be found on our GitHub repository [github.com/NVIDIA/Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer).
+
+
+## Inference with NeMo
+
+### Step-1: Install NeMo
+Please install NeMo from the GitHub `main` branch following the instructions [here](https://github.com/NVIDIA/NeMo?tab=readme-ov-file#pip-from-a-source-branch).
+
+### Step-2: Run Inference
+Run the following code to tokenize the video:
+
+```python
+import torch
+from nemo.collections.common.video_tokenizers.cosmos_vision_tokenizer import CausalVideoTokenizer
+model_name = "Cosmos-Tokenizer-DV4x8x8"
+model = CausalVideoTokenizer.from_pretrained(model_name)
+input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16)
+(indices, codes) = model.encode(input_tensor)
+```
+Please see the [Cosmos Tokenizer README within the NeMo repository](https://github.com/NVIDIA/NeMo/tree/main/nemo/collections/common/video_tokenizers) for additional examples to create training datasets with the Cosmos Tokenizer.
+
+
+# Evaluation
+
+## TokenizationPerformance Comparison
+We have extensively evaluated the **Cosmos Tokenizer** suite on various image and video benchmark datasets. In addition to commonly used datasets such as [MS-COCO](https://cocodataset.org/#home) and [DAVIS](https://davischallenge.org/), in order to cover a wide variety of visual data and standardize the evaluation, we created a benchmark called [TokenBench](https://github.com/NVlabs/Token-Bench), which is a mixed sampling of video data from diverse domains.
+
+| Tokenizer | Compression Ratio | Quantization | PSNR (DAVIS) | SSIM (DAVIS) | rFVD (DAVIS) | PSNR (TokenBench) | SSIM (TokenBench) | rFVD (TokenBench) |
+|-----------|------------------|--------------|--------------|--------------|--------------|------------------|------------------|------------------|
+| VideoGPT | 4×4×4 | VQ | 32.23 | **0.850** | 72.33 | 35.11 | **0.914** | **13.85** |
+| Omnitokenizer | 4×8×8 | VQ | 28.44 | 0.712 | 188.60 | 30.15 | 0.827 | 53.55 |
+| Cosmos-Tokenizer-DV | 4×8×8 | FSQ | **32.98** | 0.818 | **37.36** | **35.13** | 0.887 | 19.67 |
+| Cosmos-Tokenizer-DV | 8×8×8 | FSQ | 32.11 | 0.775 | 100.15 | 34.74 | 0.872 | 43.86 |
+| Cosmos-Tokenizer-DV | 8×16×16 | FSQ | 31.42 | 0.716 | 241.52 | 33.71 | 0.828 | 113.48 |
+
+* We compare with the state-of-the-art discrete video tokenizer, [OmniTokenizer](https://github.com/FoundationVision/OmniTokenizer).
+* Evaluation metrics:
+    * Peak Signal-to-Noise Ratio (PSNR)
+    * Structural Similarity (SSIM)
+    * Reconstruction Fréchet Video Distance (rFVD)
+
+## Runtime Comparison
+
+The following table shows the number of parameters and the averaged encoding and decoding times per image or video frame, measured on a single A100 80GB GPU. For comparison, we also list the parameters and average speeds of prior state-of-the-art tokenizer(s) with the same compression ratio.
+
+| Tokenizer      | Resolution | Compression Ratio | Parameters | Time (ms) |
+|----------------|------------|-------------------|------------|-----------|
+| OmniTokenizer  | 720x1280   | 4×8×8            | 54M        | 53.2      |
+| Cosmos-DV      | 720x1280   | 4×8×8            | 105M       | 51.5      |
+
+Note: We benchmarked the runtime for images under the 8x8 compression and videos under the 4×8×8 compression. Tokenizers with different compression ratios are not included in this comparison.
+
+## Ethical Considerations
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications.  When downloaded or used in accordance with our terms of service, developers should work with their internal model team to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse.  
+
+For more detailed information on ethical considerations for this model, please see the subcards of Explainability, Bias, Safety & Security, and Privacy below. Please report security vulnerabilities or NVIDIA AI Concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability/).
+
+### Bias
+
+Field                                                                                               |  Response
+:---------------------------------------------------------------------------------------------------|:---------------
+Participation considerations from adversely impacted groups [protected classes](https://www.senate.ca.gov/content/protected-classes) in model design and testing:  |  None
+Measures taken to mitigate against unwanted bias:                                                   |  None
+ 
+
+### Explainability
+
+Field                                                                                                  |  Response
+:------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------
+Intended Application & Domain:                                                                   |  Tokenization of images and videos
+Model Type:                                                                                            |  Auto-Encoder
+Intended Users:                                                                                        |  Generative AI developers for image and video generation models
+Output:                                                                                                |  Images/Videos and Latent Tokens 
+Describe how the model works:                                                                          |  Compresses and decompresses visual input (image/video).
+Technical Limitations:                                                                                 |  Due to tokenizer compression limitations, some visual information (such as small text and other structured fine details) may not be reconstructed accurately.
+Verified to have met prescribed NVIDIA quality standards:  |  Yes
+Performance Metrics:                                                                                   |  Peak Signal-to-Noise Ratio (PSNR), Structural Similarity (SSIM), Reconstruction Fréchet Video Distance (rFVD), Reconstruction Fréchet Inception Distance (rFID), Latency
+Potential Known Risks:                                                                                 |  Tokenizer's output can parse all forms of input, including what may be considered toxic, offensive, or indecent. 
+Licensing:                                                                                             |  [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+
+
+### Privacy
+Field                                                                                                                              |  Response
+:----------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------
+Generatable or reverse engineerable personal information?                                                     |  No
+Protected class data used to create this model?                                                                                       |  None Known
+Was consent obtained for any personal data used?                                                                                             |  None Known
+How often is dataset reviewed?                                                                                                     |  Before Release
+Is a mechanism in place to honor data subject right of access or deletion of personal data?                                        |  Not Applicable
+If personal collected for the development of the model, was it collected directly by NVIDIA?                                            |  Not Applicable
+If personal collected for the development of the model by NVIDIA, do you maintain or have access to disclosures made to data subjects?  |  Not Applicable
+If personal collected for the development of this AI model, was it minimized to only what was required?                                 |  Not Applicable
+Is there provenance for all datasets used in training?                                                                                |  Yes
+Does data labeling (annotation, metadata) comply with privacy laws?                                                                |  Yes
+Is data compliant with data subject requests for data correction or removal, if such a request was made?                           |  Not Applicable
+
+### Safety
+
+Field                                               |  Response
+:---------------------------------------------------|:----------------------------------
+Model Application(s):                               |  Tokenization of images and videos 
+Describe the life critical impact (if present).   |  None Known
+Use Case Restrictions:                              |  See [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+Model and dataset restrictions:            |  The Principle of least privilege (PoLP) is applied limiting access for dataset generation and model development.  Restrictions enforce dataset access during training, and dataset license constraints adhered to. Model checkpoints are made available on Hugging Face, and may become available on cloud providers' model catalog.
+
+
+### Plus Plus (++) Promise
+
+We value you, the datasets, the diversity they represent, and what we have been entrusted with. This model and its associated data have been:
+* Verified to comply with current applicable disclosure laws, regulations, and industry standards.
+* Verified to comply with applicable privacy labeling requirements.
+* Annotated to describe the collector/source (NVIDIA or a third-party).
+* Characterized for technical limitations.
+* Reviewed to ensure proper disclosure is accessible to, maintained for, and in compliance with NVIDIA data subjects and their requests.
+* Reviewed before release.
+* Tagged for known restrictions and potential safety implications.
+
+
+# Core Contributors
+Fitsum Reda, Jinwei Gu, Xian Liu, Songwei Ge, Ting-Chun Wang, Haoxiang Wang, Ming-Yu Liu

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/autoencoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea104b84fee21d170fb20f99027c076ffd97e37b8d43a6a8f6135a2a61cfaf1
+size 211093069

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/config.json

@@ -0,0 +1,6 @@
+{
+  "architectures": [
+    "CosmosTokenizer"
+  ],
+}
+  

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/decoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b82dd6f4d489bbeb728e54c828d5a676f17e6eba9b9dfe2dc7839928bee73f
+size 125210440

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/encoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a0e8459ab5e0ecfd0c00f215571de43e368f090c16adeb1a69fa835177bdea6
+size 86641076

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV4x8x8/model_config.yaml

@@ -0,0 +1 @@
+nemo_version: https://github.com/NVIDIA/NeMo/commit/6a5d4b5d19e05262a4182a83613753d424153a8f

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/.gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+autoencoder.jit filter=lfs diff=lfs merge=lfs -text
+decoder.jit filter=lfs diff=lfs merge=lfs -text
+encoder.jit filter=lfs diff=lfs merge=lfs -text

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/README.md

@@ -0,0 +1,325 @@
+---
+license: other
+license_name: nvidia-open-model-license
+license_link: >-
+  https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf
+library_name: nemo
+---
+# **Cosmos Tokenizer**: A suite of image and video tokenizers
+
+[**Website**](https://research.nvidia.com/labs/dir/cosmos-tokenizer) | [**Code**](https://github.com/NVIDIA/Cosmos-Tokenizer) | **Video**
+
+
+# Model Overview
+
+## Description:
+**Cosmos Tokenizer** is a suite of visual tokenizers for images and videos that delivers various compression rates while maintaining high reconstruction quality. Cosmos Tokenizer can serve as an effective and efficient building block in both diffusion-based and autoregressive models for image and video generation.
+
+
+Our tokenizers come in two types: **Continuous** (C) and **Discrete** (D), each with **Image** (I) and **Video** (V) variants:
+* Continuous tokenizers encode visual data into continuous latent embeddings, as shown in latent diffusion models like [Stable Diffusion](https://github.com/CompVis/stable-diffusion). These embeddings are suitable for models that generate data by sampling from continuous distributions. 
+* Discrete tokenizers encode visual data into discrete latent codes, mapping them into quantized indices, as seen in autoregressive transformers such as [VideoPoet](https://sites.research.google/videopoet/). This discretization is required for models that generate data by optimizing the cross-entropy loss, such as the GPT models.
+
+
+|                   | Continuous ( C )    | Discrete ( D )      |
+| ------------------|---------------------|---------------------|
+| **Images ( I )**        | Cosmos-Tokenizer-CI            | Cosmos-Tokenizer-DI            |
+| **Videos ( V )**        | Cosmos-Tokenizer-CV            | Cosmos-Tokenizer-DV            |
+
+
+Given an image or a video, Cosmos Tokenizer outputs either continuous latents or discrete tokens. Cosmos Tokenizer achieves spatial compression rates of 8x8 or 16x16 and temporal compression factors of 4x or 8x, resulting in a total compression factor of up to 2048x (=8x16x16). Cosmos Tokenizer delivers 8x more total compression than state-of-the-art (SOTA) methods while simultaneously maintaining higher image quality and running up to 12x faster than the best available SOTA tokenizers.
+
+**Model Developer**: NVIDIA
+
+## Model Versions
+
+The initial release (v1.0) of Cosmos Tokenizer includes the following tokenizers:
+* **Continuous Tokenizers**
+    * Continuous Image (CI) Tokenizer
+        * [Cosmos-Tokenizer-CI8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CI8x8) (8x8 spatial compression)
+        * [Cosmos-Tokenizer-CI16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-CI16x16) (16x16 spatial compression)
+    * Continuous Video (CV) Tokenizer
+        * [Cosmos-Tokenizer-CV4x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV4x8x8) (4x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-CV8x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV8x8x8) (8x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-CV8x16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-CV8x16x16) (8x temporal compression, 16x16 spatial compression)
+* **Discrete Tokenizers**
+    * Discrete Image (DI) Tokenizer
+        * [Cosmos-Tokenizer-DI8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DI8x8) (8x8 spatial compression)
+        * [Cosmos-Tokenizer-DI16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-DI16x16) (16x16 spatial compression)
+    * Discrete Video (DV) Tokenizer
+        * [Cosmos-Tokenizer-DV4x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV4x8x8) (4x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-DV8x8x8](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV8x8x8) (8x temporal compression, 8x8 spatial compression)
+        * [Cosmos-Tokenizer-DV8x16x16](https://huggingface.co/nvidia/Cosmos-Tokenizer-DV8x16x16) (8x temporal compression, 16x16 spatial compression)
+
+
+### License/Terms of Use: 
+[NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+
+Under the NVIDIA Open Model License, NVIDIA confirms:
+
+* Models are commercially usable.
+* You are free to create and distribute Derivative Models.
+* NVIDIA does not claim ownership to any outputs generated using the Models or Derivative Models.
+
+## Model Architecture: 
+
+We designed Cosmos Tokenizer using a lightweight and computationally efficient architecture, featuring a temporally causal design. Specifically, we employ causal temporal convolution and causal temporal attention layers to preserve the natural temporal order of video frames, ensuring seamless tokenization of images and videos using a single unified network architecture. The encoder and decoder form a symmetrical pair, which are mirrors of each other. The encoder starts with a 2-level [Haar wavelet](https://link.springer.com/book/10.1007/978-3-319-04295-4) transform layer, which down-samples inputs by a factor of 4 in both spatial and temporal dimensions. Likewise, the decoder ends with an inverse wavelet transform. We employ the vanilla autoencoder (AE) formulation to model the latent space for continuous tokenizers. For discrete tokenizers, we adopt the [Finite-Scalar-Quantization](https://openreview.net/forum?id=8ishA3LxN8) (FSQ) as the latent space quantizer.
+
+![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/638fb8cf2380ffd99caf8c2a/gQH5n9iCEtqZc7uutUwdL.jpeg)
+
+
+
+## Input/Output Specifications
+
+### Encoder
+* **Input**
+    * **Types:** Images or Videos
+    * **Format:** RGB (Red, Green, Blue)
+    * **Resolution:** 
+        * Minimum: 256px (shorter side)
+        * Maximum: Up to 4K
+    * **Video Length:** Up to 8 seconds for 1080p videos (bounded by A100 80G GPU memory; higher resolutions will have shorter supported durations)
+
+* **Output**
+    * **Types:** Tokens
+        * Continuous Image/Video Tokenizers: Continuous value feature vectors
+        * Discrete Image/Video Tokenizers: Integer indices
+
+### Decoder
+* **Input**
+    * **Types:** Tokens from encoder
+
+* **Output**
+    * **Types:** Images or Videos (matching input type)
+    * **Format:** RGB (Red, Green, Blue)
+    * **Resolution:** Same as input resolution
+    * **Video Length:** Same as input video length
+
+## Software Integration (Required For NVIDIA Models Only):
+**Runtime Engine(s):** 
+* [Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer)
+* [NeMo](https://github.com/NVIDIA/NeMo) (please install the latest version from the GitHub main branch)
+
+**Supported Hardware Microarchitecture Compatibility:**
+* NVIDIA Ampere (e.g., A100)
+* NVIDIA Hopper (e.g., H100)
+
+Note: We have only tested Cosmos Tokenizer with BF16 precision on Ampere and Hopper GPUs. If you are using older versions of NVIDIA GPUs (e.g., NVIDIA Volta GPUs), you may need to switch to FP32 precision.
+
+
+**Operating System(s):**
+* Linux (We have not tested on other operating systems.)
+
+# Usage
+Inference Engines: 
+* [Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer) (PyTorch)
+* [NeMo](https://github.com/NVIDIA/NeMo)
+
+## Inference with `Cosmos-Tokenizer` (PyTorch)
+### Step-1: Installation of `Cosmos-Tokenizer` 
+Note: Currently, the `Cosmos-Tokenizer` code is only supported on Linux.
+
+- Please clone the `Cosmos-Tokenizer` from GitHub repo [github.com/NVIDIA/Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer).
+
+    ```bash
+    git clone https://github.com/NVIDIA/Cosmos-Tokenizer.git
+    cd Cosmos-Tokenizer
+    ```
+- Install dependencies
+
+    ```bash
+    pip3 install -r requirements.txt
+    apt-get install -y ffmpeg
+    ```
+
+- Preferably, you could build a docker image using our provided Dockerfile.
+    ```bash
+    docker build -t cosmos-docker -f Dockerfile.    
+    # You can run the container as:
+    docker run --gpus all -it --rm -v /home/${USER}:/home/${USER} \
+        --workdir ${PWD} cosmos-docker /bin/bash
+    ```
+
+### Step-2: Download Pre-trained Checkpoints
+- Create a local directory for the pre-trained checkpoints and download the 
+pre-trained checkpoints from HuggingFace.
+
+    ```python
+    from huggingface_hub import login, snapshot_download
+    import os
+    # You could get your Hugging Face token from https://huggingface.co/settings/tokens
+    login(token=<YOUT-HF-TOKEN>, add_to_git_credential=True)
+    # You could specify the tokenizers you want to download.
+    model_names = [
+            "Cosmos-Tokenizer-CI8x8",
+            "Cosmos-Tokenizer-CI16x16",
+            "Cosmos-Tokenizer-CV4x8x8",
+            "Cosmos-Tokenizer-CV8x8x8",
+            "Cosmos-Tokenizer-CV8x16x16",
+            "Cosmos-Tokenizer-DI8x8",
+            "Cosmos-Tokenizer-DI16x16",
+            "Cosmos-Tokenizer-DV4x8x8",
+            "Cosmos-Tokenizer-DV8x8x8",
+            "Cosmos-Tokenizer-DV8x16x16",
+    ]
+    for model_name in model_names:
+        hf_repo = "nvidia/" + model_name
+        local_dir = "pretrained_ckpts/" + model_name
+        os.makedirs(local_dir, exist_ok=True)
+        print(f"downloading {model_name} to {local_dir}...")
+        snapshot_download(repo_id=hf_repo, local_dir=local_dir)
+    ```
+
+- Under the ech checkpoint directory `pretrained_ckpts/<model-name>`, we provide the encoder, 
+decoder and the full autoencoder JIT models.
+
+    ```bash 
+    ├── pretrained_ckpts/   
+    │   ├── Cosmos-Tokenizer-DV8x8x8/
+    │   │   ├── encoder.jit
+    │   │   ├── decoder.jit
+    │   │   ├── autoencoder.jit
+    │   ...
+    ```
+
+### Step-3: Run Inference
+You can use the following example commands to encode and decode images or videos. For each, the same command works for both continuous and discrete tokenization. Simply provide the proper JIT-compiled ckpt to `checkpoint_enc`, `checkpoint_dec`, or the full autoencoder ckpt to `checkpoint`.
+
+```python
+import torch
+from cosmos_tokenizer.video_lib import CausalVideoTokenizer
+model_name = "Cosmos-Tokenizer-DV4x8x8"
+input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16)
+encoder = CausalVideoTokenizer(checkpoint_enc=f'pretrained_ckpts/{model_name}/encoder.jit')
+(indices, codes) = encoder.encode(input_tensor)
+torch.testing.assert_close(indices.shape, (1, 3, 64, 64))
+torch.testing.assert_close(codes.shape, (1, 6, 3, 64, 64))
+
+# The input tensor can be reconstructed by the decoder as:
+decoder = CausalVideoTokenizer(checkpoint_dec=f'pretrained_ckpts/{model_name}/decoder.jit')
+reconstructed_tensor = decoder.decode(indices)
+torch.testing.assert_close(reconstructed_tensor.shape, input_tensor.shape)
+```
+
+The `indices` will have the shape `(1, 3, 64, 64)` and contain integral values in the range `[1..64K]`, where the first of the three integral maps represents the first frame. 
+The `codes` will contain the pre-quantization continuous latent with shape `(1, 6, 3, 64, 64)`, where C=6 represents the number of FSQ levels.
+
+**Note**: More inference usage commands, including both TorchScript (JIT) and PyTorch Inference APIs on real images and videos, can be found on our GitHub repository [github.com/NVIDIA/Cosmos-Tokenizer](https://github.com/NVIDIA/Cosmos-Tokenizer).
+
+
+## Inference with NeMo
+
+### Step-1: Install NeMo
+Please install NeMo from the GitHub `main` branch following the instructions [here](https://github.com/NVIDIA/NeMo?tab=readme-ov-file#pip-from-a-source-branch).
+
+### Step-2: Run Inference
+Run the following code to tokenize the video:
+
+```python
+import torch
+from nemo.collections.common.video_tokenizers.cosmos_vision_tokenizer import CausalVideoTokenizer
+model_name = "Cosmos-Tokenizer-DV4x8x8"
+model = CausalVideoTokenizer.from_pretrained(model_name)
+input_tensor = torch.randn(1, 3, 9, 512, 512).to('cuda').to(torch.bfloat16)
+(indices, codes) = model.encode(input_tensor)
+```
+Please see the [Cosmos Tokenizer README within the NeMo repository](https://github.com/NVIDIA/NeMo/tree/main/nemo/collections/common/video_tokenizers) for additional examples to create training datasets with the Cosmos Tokenizer.
+
+# Evaluation
+
+## TokenizationPerformance Comparison
+We have extensively evaluated the **Cosmos Tokenizer** suite on various image and video benchmark datasets. In addition to commonly used datasets such as [MS-COCO](https://cocodataset.org/#home) and [DAVIS](https://davischallenge.org/), in order to cover a wide variety of visual data and standardize the evaluation, we created a benchmark called [TokenBench](https://github.com/NVlabs/Token-Bench), which is a mixed sampling of video data from diverse domains.
+
+| Tokenizer | Compression Ratio | Quantization | PSNR (DAVIS) | SSIM (DAVIS) | rFVD (DAVIS) | PSNR (TokenBench) | SSIM (TokenBench) | rFVD (TokenBench) |
+|-----------|------------------|--------------|--------------|--------------|--------------|------------------|------------------|------------------|
+| VideoGPT | 4×4×4 | VQ | 32.23 | **0.850** | 72.33 | 35.11 | **0.914** | **13.85** |
+| Omnitokenizer | 4×8×8 | VQ | 28.44 | 0.712 | 188.60 | 30.15 | 0.827 | 53.55 |
+| Cosmos-Tokenizer-DV | 4×8×8 | FSQ | **32.98** | 0.818 | **37.36** | **35.13** | 0.887 | 19.67 |
+| Cosmos-Tokenizer-DV | 8×8×8 | FSQ | 32.11 | 0.775 | 100.15 | 34.74 | 0.872 | 43.86 |
+| Cosmos-Tokenizer-DV | 8×16×16 | FSQ | 31.42 | 0.716 | 241.52 | 33.71 | 0.828 | 113.48 |
+
+* We compare with the state-of-the-art discrete video tokenizer, [OmniTokenizer](https://github.com/FoundationVision/OmniTokenizer).
+* Evaluation metrics:
+    * Peak Signal-to-Noise Ratio (PSNR)
+    * Structural Similarity (SSIM)
+    * Reconstruction Fréchet Video Distance (rFVD)
+
+## Runtime Comparison
+
+The following table shows the number of parameters and the averaged encoding and decoding times per image or video frame, measured on a single A100 80GB GPU. For comparison, we also list the parameters and average speeds of prior state-of-the-art tokenizer(s) with the same compression ratio.
+
+| Tokenizer      | Resolution | Compression Ratio | Parameters | Time (ms) |
+|----------------|------------|-------------------|------------|-----------|
+| OmniTokenizer  | 720x1280   | 4×8×8            | 54M        | 53.2      |
+| Cosmos-DV      | 720x1280   | 4×8×8            | 105M       | 51.5      |
+
+Note: We benchmarked the runtime for images under the 8x8 compression and videos under the 4×8×8 compression. Tokenizers with different compression ratios are not included in this comparison.
+
+## Ethical Considerations
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications.  When downloaded or used in accordance with our terms of service, developers should work with their internal model team to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse.  
+
+For more detailed information on ethical considerations for this model, please see the subcards of Explainability, Bias, Safety & Security, and Privacy below. Please report security vulnerabilities or NVIDIA AI Concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability/).
+
+### Bias
+
+Field                                                                                               |  Response
+:---------------------------------------------------------------------------------------------------|:---------------
+Participation considerations from adversely impacted groups [protected classes](https://www.senate.ca.gov/content/protected-classes) in model design and testing:  |  None
+Measures taken to mitigate against unwanted bias:                                                   |  None
+ 
+
+### Explainability
+
+Field                                                                                                  |  Response
+:------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------
+Intended Application & Domain:                                                                   |  Tokenization of images and videos
+Model Type:                                                                                            |  Auto-Encoder
+Intended Users:                                                                                        |  Generative AI developers for image and video generation models
+Output:                                                                                                |  Images/Videos and Latent Tokens 
+Describe how the model works:                                                                          |  Compresses and decompresses visual input (image/video).
+Technical Limitations:                                                                                 |  Due to tokenizer compression limitations, some visual information (such as small text and other structured fine details) may not be reconstructed accurately.
+Verified to have met prescribed NVIDIA quality standards:  |  Yes
+Performance Metrics:                                                                                   |  Peak Signal-to-Noise Ratio (PSNR), Structural Similarity (SSIM), Reconstruction Fréchet Video Distance (rFVD), Reconstruction Fréchet Inception Distance (rFID), Latency
+Potential Known Risks:                                                                                 |  Tokenizer's output can parse all forms of input, including what may be considered toxic, offensive, or indecent. 
+Licensing:                                                                                             |  [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+
+
+### Privacy
+Field                                                                                                                              |  Response
+:----------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------
+Generatable or reverse engineerable personal information?                                                     |  No
+Protected class data used to create this model?                                                                                       |  None Known
+Was consent obtained for any personal data used?                                                                                             |  None Known
+How often is dataset reviewed?                                                                                                     |  Before Release
+Is a mechanism in place to honor data subject right of access or deletion of personal data?                                        |  Not Applicable
+If personal collected for the development of the model, was it collected directly by NVIDIA?                                            |  Not Applicable
+If personal collected for the development of the model by NVIDIA, do you maintain or have access to disclosures made to data subjects?  |  Not Applicable
+If personal collected for the development of this AI model, was it minimized to only what was required?                                 |  Not Applicable
+Is there provenance for all datasets used in training?                                                                                |  Yes
+Does data labeling (annotation, metadata) comply with privacy laws?                                                                |  Yes
+Is data compliant with data subject requests for data correction or removal, if such a request was made?                           |  Not Applicable
+
+### Safety
+
+Field                                               |  Response
+:---------------------------------------------------|:----------------------------------
+Model Application(s):                               |  Tokenization of images and videos 
+Describe the life critical impact (if present).   |  None Known
+Use Case Restrictions:                              |  See [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
+Model and dataset restrictions:            |  The Principle of least privilege (PoLP) is applied limiting access for dataset generation and model development.  Restrictions enforce dataset access during training, and dataset license constraints adhered to. Model checkpoints are made available on Hugging Face, and may become available on cloud providers' model catalog.
+
+
+### Plus Plus (++) Promise
+
+We value you, the datasets, the diversity they represent, and what we have been entrusted with. This model and its associated data have been:
+* Verified to comply with current applicable disclosure laws, regulations, and industry standards.
+* Verified to comply with applicable privacy labeling requirements.
+* Annotated to describe the collector/source (NVIDIA or a third-party).
+* Characterized for technical limitations.
+* Reviewed to ensure proper disclosure is accessible to, maintained for, and in compliance with NVIDIA data subjects and their requests.
+* Reviewed before release.
+* Tagged for known restrictions and potential safety implications.
+
+
+# Core Contributors
+Fitsum Reda, Jinwei Gu, Xian Liu, Songwei Ge, Ting-Chun Wang, Haoxiang Wang, Ming-Yu Liu

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/autoencoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccf00856bc49c9e1c5ca6b01f47ef65d35bcdb37d58724641a3ea45751714724
+size 213071541

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/config.json

@@ -0,0 +1,6 @@
+{
+  "architectures": [
+    "CosmosTokenizer"
+  ],
+}
+  

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/decoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881f1f6317872fad3eeeaa1e595061aa3ee12590d14ce435ac9e9e5c883e797b
+size 126792092

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/encoder.jit

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a014af9f0bfae97a29a5bf071ca58ac29be40d4ffae12ef08a11004b92f0fb8d
+size 87042184

Meissonic/pretrained_ckpts/Cosmos-0.1-Tokenizer-DV8x8x8/model_config.yaml

@@ -0,0 +1 @@
+nemo_version: https://github.com/NVIDIA/NeMo/commit/6a5d4b5d19e05262a4182a83613753d424153a8f