>>' + prompt
+
+ start_time = time.time()
+ with torch.cuda.amp.autocast(enabled=True, dtype=torch.bfloat16, cache_enabled=True), torch.no_grad():
+ generated_image, _ = gen_one_example(
+ pipe.infinity,
+ pipe.vae,
+ pipe.text_tokenizer,
+ pipe.text_encoder,
+ prompt,
+ negative_prompt=negative_prompt,
+ g_seed=seed,
+ gt_leak=gt_leak,
+ gt_ls_Bl=gt_ls_Bl,
+ cfg_list=args.cfg,
+ tau_list=tau,
+ scale_schedule=scale_schedule,
+ cfg_insertion_layer=[0],
+ vae_type=args.vae_type,
+ sampling_per_bits=1,
+ enable_positive_prompt=0,
+ low_vram_mode=True,
+ args=args,
+ get_visual_rope_embeds=pipe.get_visual_rope_embeds,
+ context_info=context_info,
+ noise_list=None,
+ )
+ if len(generated_image.shape) == 3:
+ generated_image = generated_image.unsqueeze(0)
+ print(generated_image.shape)
+ generated_image_list.append(generated_image)
+
+ generated_image = torch.cat(generated_image_list, 2)
+ end_time = time.time()
+ elapsed_time = end_time - start_time
+
+ return {
+ "output": generated_image.cpu().numpy(),
+ "elapsed_time": elapsed_time,
+ }
+
+
+if __name__ == '__main__':
+ # For optimal performance, enabling the prompt rewriter is recommended.
+ # To utilize the GPT model, ensure the following environment variables are set:
+ # export OPEN_API_KEY="YOUR_API_KEY"
+ # export GLOBAL_AZURE_ENDPOINT="YOUR_ENDPOINT"
+ enable_rewriter=0
+ checkpoints_dir = './'
+
+
+ # infer args
+ args = Args()
+ args.pn='0.90M'
+ args.fps=16
+ args.video_frames=81
+ args.model_path=os.path.join(checkpoints_dir, 'infinitystar_8b_720p_weights')
+ args.checkpoint_type='torch_shard' # omnistore
+ args.vae_path=os.path.join(checkpoints_dir, 'infinitystar_videovae.pth')
+ args.text_encoder_ckpt=os.path.join(checkpoints_dir, 'text_encoder/flan-t5-xl-official/')
+ args.model_type='infinity_qwen8b'
+ args.text_channels=2048
+ args.dynamic_scale_schedule='infinity_elegant_clip20frames_v2'
+ args.bf16=1
+ args.use_apg=1
+ args.use_cfg=0
+ args.cfg=34
+ args.tau_image = 1
+ args.tau_video = 0.4
+ args.apg_norm_threshold=0.05
+ args.image_scale_repetition='[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]'
+ args.video_scale_repetition='[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 1, 1]'
+ args.append_duration2caption=1
+ args.use_two_stage_lfq=1
+ args.detail_scale_min_tokens=750
+ args.semantic_scales=12
+ args.max_repeat_times=10000
+ args.enable_rewriter=enable_rewriter
+
+ # load models
+ pipe = InferencePipe(args)
+
+
+ prompt = "A handsome smiling gardener inspecting plants, realistic cinematic lighting, detailed textures, ultra-realistic"
+ image_path = 'assets/reference_image.webp' # Remove this for Text-to-Video (T2V) generation
+ data = {
+ 'seed': 41,
+ 'image_path': image_path,
+ 'prompt': prompt,
+ }
+ if args.enable_rewriter:
+ # Step 1: Rewrite the prompt using GPT
+ # rewritten_prompt = prompt
+ rewritten_prompt = pipe.gpt_model(
+ prompt = ("Rewrite the following video descriptions, add more details of the subject and the camera movement to enhance the quality of the video. Do not use the word 'they' to refer to a single person or object. Concatenate all sentences together, not present them in paragraphs. Please rewrite with concise and clear language: "
+ + prompt),
+ system_prompt=pipe.system_prompt,
+ )
+ print(f"Rewritten prompt: {rewritten_prompt}")
+ prompt = rewritten_prompt
+ data['prompt'] = prompt
+
+ output_dict = perform_inference(pipe, data, args)
+ save_dir = 'output'
+ gen_video_path = osp.join(os.path.join(save_dir, 'gen_videos'), f'demo.mp4')
+ save_video(output_dict['output'], fps=args.fps, save_filepath=gen_video_path)
+
+ print(f"Video genernation done: {gen_video_path=}")
diff --git a/Meissonic/InfinityStar/tools/prompt_rewriter.py b/Meissonic/InfinityStar/tools/prompt_rewriter.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d10585c95aec5d7625ac742204b3e4aa649b473
--- /dev/null
+++ b/Meissonic/InfinityStar/tools/prompt_rewriter.py
@@ -0,0 +1,227 @@
+# Copyright (c) 2025 FoundationVision
+# SPDX-License-Identifier: MIT
+
+import os
+import random
+import time
+from threading import Lock
+from abc import abstractmethod
+from typing import Any, List, Union, Optional
+
+import numpy as np
+
+import openai
+
+# from auto_caption.models.base_model import Model
+OPEN_API_KEY = os.environ.get('OPEN_API_KEY')
+GLOBAL_AZURE_ENDPOINT = os.environ.get('GLOBAL_AZURE_ENDPOINT')
+
+class SingletonArgMeta(type):
+ """
+ This is a thread-safe implementation of Singleton.
+ """
+
+ _instances = {}
+
+ _lock: Lock = Lock()
+ """
+ We now have a lock object that will be used to synchronize threads during
+ first access to the Singleton.
+ """
+
+ def __call__(cls, *args, **kwargs):
+ """
+ changes to the value of the `__init__` argument do affect
+ the returned instance.
+ """
+ # Now, imagine that the program has just been launched. Since there's no
+ # Singleton instance yet, multiple threads can simultaneously pass the
+ # previous conditional and reach this point almost at the same time. The
+ # first of them will acquire lock and will proceed further, while the
+ # rest will wait here.
+ with cls._lock:
+ # The first thread to acquire the lock, reaches this conditional,
+ # goes inside and creates the Singleton instance. Once it leaves the
+ # lock block, a thread that might have been waiting for the lock
+ # release may then enter this section. But since the Singleton field with
+ # specific arguments is already initialized, the thread won't create a new object.
+ if cls.__name__+str(args)+str(kwargs) not in cls._instances:
+ instance = super().__call__(*args, **kwargs)
+ cls._instances[cls.__name__+str(args)+str(kwargs)] = instance
+ return cls._instances[cls.__name__+str(args)+str(kwargs)]
+
+
+class Model(metaclass=SingletonArgMeta):
+ """an abstrct model"""
+
+ def __init__(self, model_name: Union[str, List[str]], ak: Union[str, List[str]], token_stat_percent: Optional[float] = None) -> None:
+ self.clients = self._init_clients(model_name, ak)
+ if token_stat_percent is not None:
+ self._init_token_stat(token_stat_percent)
+
+ def _init_token_stat(self, token_stat_percent):
+ self.token_stat_percent = token_stat_percent
+ self.token_sort = []
+ self.token_stat = {'max_token': 0, 'mean_token': 0,
+ 'count': 0, f'p{token_stat_percent*100}_token_num': 0}
+ self.token_stat_percent = token_stat_percent
+
+ def _init_clients(self, model_name, ak):
+ if not isinstance(model_name, list):
+ model_name = [model_name]
+ if not isinstance(ak, list):
+ ak = [ak]
+ clients = []
+ if len(ak) > 1 and len(model_name) == 1:
+ model_name = model_name*len(ak)
+ elif len(ak) == 1 and len(model_name) > 1:
+ ak = ak*len(model_name)
+
+ assert len(ak) == len(
+ model_name), f"length of ak = {len(ak)} != length of model_name = {len(model_name)}"
+ for model, ak in zip(model_name, ak):
+ client = self._creat_client(model, ak)
+ clients.append(client)
+ print(f"init {len(clients)} clients!!!")
+ return clients
+
+ def _update(self, token_num):
+ self.token_sort.append(token_num)
+ self.token_stat[f'p{self.token_stat_percent*100}_token_num'] = round(
+ np.percentile(self.token_sort, self.token_stat_percent*100), 2)
+ self.token_stat['count'] = len(self.token_sort)
+ self.token_stat['mean_token'] = round(np.mean(self.token_sort), 2)
+ self.token_stat['max_token'] = np.max(self.token_sort)
+
+ @abstractmethod
+ def _creat_client(self, *args: Any, **kwds: Any) -> Any:
+ raise NotImplementedError
+
+ @abstractmethod
+ def __call__(self, *args: Any, **kwds: Any) -> Any:
+ raise NotImplementedError
+
+
+class OpenAIGPTModel(Model):
+ _global_azure_endpoint = GLOBAL_AZURE_ENDPOINT
+ _api_version = "2023-05-15"
+
+ def __init__(self, model_name='gpt-4', ak='', log_prob=0.01, if_global=False, token_stat_percent=0.99) -> None:
+ self.ak = OPEN_API_KEY if not ak else ak
+ self.if_global = if_global
+ self.ak_state = {}
+ self.ak_state_succ = {}
+ self.log_prob = log_prob
+ self.start_time = time.time()
+ super().__init__(model_name, ak, token_stat_percent)
+
+ def _creat_client(self, model_name, ak):
+ client = openai.AzureOpenAI(
+ azure_endpoint=OpenAIGPTModel._global_azure_endpoint,
+ api_version=OpenAIGPTModel._api_version,
+ api_key=ak,
+ )
+ client.temp_model_name = model_name
+ client.temp_ak = ak
+ self.ak_state[ak[:5]] = 0
+ self.ak_state_succ[ak[:5]] = 0
+ return client
+
+ def __call__(self, prompt="hello", system_prompt=None, max_tokens=1000, return_output_token_length=False):
+ client = random.choice(self.clients)
+ ak = client.temp_ak
+ self.ak_state[ak[:5]] += 1
+ if random.random() < self.log_prob:
+ for ak in self.ak_state:
+ print(
+ f"ak: {ak} 请求数：{self.ak_state[ak]}, 成功数：{self.ak_state_succ[ak]}, 成功率：{self.ak_state_succ[ak]/self.ak_state[ak]*100:.2f}%, 速度：{self.ak_state_succ[ak]/(time.time()-self.start_time)*60:.2f} 个/分钟")
+ print(f"token_stat：{self.token_stat}")
+
+ messages = []
+ if system_prompt is not None:
+ messages.append({"role": "system", "content": system_prompt})
+ messages.append({"role": "user", "content": prompt})
+
+ completion = client.chat.completions.create(
+ extra_headers={"X-TT-LOGID": "lizhe.xyz"}, # 请务必带上此header，方便定位问题
+ model=client.temp_model_name,
+ messages=messages,
+ max_tokens=max_tokens
+ )
+ self.ak_state_succ[ak[:5]] += 1
+ if self.token_stat_percent is not None:
+ # 输出token统计
+ self._update(completion.usage.completion_tokens)
+
+ if return_output_token_length:
+ # 允许输出token长度，便于筛选因输出达到max_tokens而被截断的数据
+ return completion.choices[0].message.content, completion.usage.completion_tokens
+ return completion.choices[0].message.content
+
+
+# better prompt
+system_prompt = '''You are a prompt engineer, aiming to rewrite user inputs into high-quality prompts for better video generation without affecting the original meaning.\n''' \
+'''Task requirements:\n''' \
+'''1. If no subject appearance description provided in user inputs, add some descriptions about appearance (e.g., clothing, gender);\n''' \
+'''2. If there is no camera movement information in the user inputs and the entire scene is very static, you can appropriately add some simple camera movement descriptions;\n''' \
+'''3. If there is no description of the setting, add some visual details about the scene, but there should not be a change of the scene;\n''' \
+'''4. If there is no description of shot scale in the user's input, you can appropriately add some information about the shot scale;\n''' \
+'''5. If the character's actions take too little time, you can appropriately add some detailed actions during or after the original action so that the whole prompt described can last for 5 seconds in total. The newly added actions need to be dynamic rather than static (for example, looking into the distance or standing still). Add the main subject's actions in chronological order, and include detailed descriptions of their physical movements;\n''' \
+'''6. If you are describing an action, describe the entire process, for example, "pick up the clothes, put your hands into the sleeves, and put on the clothes.";\n''' \
+'''7. The word count for each action description should be evenly distributed throughout the entire output, and there should not be any action descriptions that are significantly longer or shorter than the others;\n''' \
+'''8. The entire sequence of actions needs to be engaging and impressive;\n''' \
+'''9. If there are multiple subjects, describe the actions of each subject;\n''' \
+'''10. All the descriptions must be about appearance or action, and no sensory, feeling or highlighting descriptions are allowed;\n''' \
+'''11. Do not include any emphatic or highlighted descriptions such as demonstrating, showcasing, highlighting, emphasizing anything in the rewritten text;\n''' \
+'''12. All descriptions should be as objective as possible;\n''' \
+'''13. Unless the subject refers to multiple objects, use he/she/it instead of they to refer to the subject;\n''' \
+'''14. In describing actions, do not add adverbs before verbs, do not add prepositional phrase (e.g., with something) after verbs.\n''' \
+'''15. Output the entire prompt in English, retaining original text in quotes and titles, and preserve key input information;\n''' \
+'''16. If the original input is purely a landscape description, do not add additional subjects such as people;\n''' \
+'''17. Who does what action? Clarify the subject at the beginning of the sentence;\n''' \
+'''18. If you describe using the actions "Rolling", "raising hands", "lifting legs", etc., clarity whether they are forward or backward;\n''' \
+'''19. If the prompts specify the direction of movement, do not adjust it at will;\n''' \
+'''20. If describing a situation with multiple people, such as a group, state the number of people, but not too many, usually less than five;\n''' \
+'''21. If you are describing a person rotating, if you are describing the head turning, you should also describe the body turning as well;\n''' \
+'''22. When you describe actions, background and camera movement, you need to write them separately, don't mix them together;\n''' \
+'''23. If the original input is related to anime, you need to add "This video showcases an animated scene" at the beginning of your rewritten prompt. If there is a requirement for 2D or 3D animation, you should also include that information;\n''' \
+'''24. The revised prompt should be between 50-140 words long and always use simple and direct words.\n''' \
+
+'''Original user inputs and the examples after rewriting:\n''' \
+
+'''1.
+- **Original user inputs**: A woman is swimming underwater in a pool, extending her arms forward and kicking her legs. The pool has colorful lane dividers in the background. She performs a somersault, rotating her body in the water. After the somersault, she continues swimming forward with a streamlined body position.
+- **Examples after rewriting**: The video captures a female swimmer underwater, showing a sequence of movements that demonstrate various swimming techniques. Initially, the swimmer's body is horizontal, with the arms extended forward and the legs straightened, indicating the start of a stroke cycle. As the video progresses, the head turns slightly to the side, the arms begin to bend, and the legs begin to move in a rhythmic dolphin kick, creating a powerful propulsion mechanism. Throughout the frame, the swimmer maintains an streamlined position with minimal movement of the hands and feet to maintain balance and continue forward momentum. The swimmer is wearing a white bathing suit with pink floral patterns, her hair tied back to avoid obstructions. Bubbles can be seen around her body, indicating that she is moving through the water with fluid movements. In the background is a swimming pool lane marked by red and yellow lane lines that extend across the pool. The clear blue water reflects the sunlight, creating a serene and vibrant atmosphere. Another person is visible in the distance, partially obscured by the water's surface, swimming in the same lane. The swimmer's movements create small splashes and bubbles as she moves through the water. The camera follows her in front, highlighting the technique and form of the swimmer. The overall scene conveys a sense of focus and athleticism, highlighting the skill and grace of the swimmer in the water. The camera moves with the female athlete, keeping her in the center of the frame.\n''' \
+
+'''2.
+- **Original user inputs**: In an indoor tennis court, a man prepares to serve a tennis ball. He tosses the ball into the air and swings his racket to hit it. The opponent moves to the right to intercept the ball. The ball hits the net and falls to the ground on the opponent's side. The man in the white shirt follows through with his serve and then moves to the left side of the court.
+- **Examples after rewriting**: The video captures a well-lit indoor tennis court with high ceilings and large windows that allow natural light to filter in. The court has a standard blue surface with white borders and a net in the middle. A male player in a white T-shirt and black shorts can be seen tossing a ball upward with his left hand and swinging his racket at the ball with his right hand. The opposing player in a white top and dark shorts then moves to the right of the frame, opens his racket, runs after the ball, and returns a powerful forehand shot that hits the net and lands. The background includes an advertisement for tennis equipment brand Babolat and a bench with practice tennis balls. The overall atmosphere reflects focused training in a professional environment. The camera focuses on the man, moving first to the left and then to the right.\n''' \
+
+'''3.
+- **Original user inputs**: A man wearing black athletic clothing and bright orange running shoes is running on a paved track. The background features a large, green grassy field with scattered trees and a few buildings in the distance. The sky is clear and blue, indicating a sunny day. The man maintains a steady pace throughout the sequence, with his arms bent at the elbows and his legs moving in a rhythmic motion.
+- **Examples after rewriting**: The video captures a male runner in motion in an outdoor setting. The man is shirtless, wearing black running pants and bright orange running shoes. He has a black headband on his head and his hair is closely cropped. He runs on the road, swinging his hands back and forth at his sides as he runs, and his left and right legs alternate between landing and lifting on the road. The man maintains a consistent posture and runs at a constant speed, showing a continuous running posture. The background is a large green lawn that looks like a park or sports field, with trees scattered around the edges. In the distance, buildings that look like houses or apartments can be seen, indicating that the location is close to a residential area. The sky is clear and blue, indicating that it is a sunny day with good weather. The camera follows the man as he moves to the left.\n''' \
+
+'''4.
+- **Original user inputs**: In a large indoor arena with many people and banners, two individuals dressed in fencing gear are engaged in a fencing match. They are positioned on a white mat with a blue mat nearby. The fencers are seen lunging and parrying at each other with their swords. After a series of movements, they begin to separate and walk towards the right side of the frame. The fencer on the right raises their arm and points towards a man dressed in black standing near a table with a 'naked' banner.
+- **Examples after rewriting**: The video captures an intense fencing match between two male fencers in a large indoor arena. The fencers are dressed in traditional white fencing gear, including masks, jackets, gloves, and trousers, with one fencer wearing a light-colored top and light-colored pants, a yellow and black socks and shoes, and a mask with a blue logo on it, while the other fencer has a mask with a red logo on it. They are both holding fencing foils in their hands and positioned on a white mat with a blue mat nearby, with bags and equipment scattered nearby. The male fencer on the left side of the frame, wearing a mask with a blue logo on it, leans back with his legs bent, his left hand extended forward holding the sword, and then he stands up straight and continues to attack, while the male fencer on the right side of the frame, wearing a mask with a red logo on it, stands there with his left foot in front and his right foot behind, his right hand extended forward holding the sword, and then both of them stand up straight and continue to attack and defend. The fencer on the right side of the frame actively moves forward with his right foot in front and his left foot behind, while the fencer on the left side of the frame extends his left leg and swings his left hand to attack, while the fencer on the right side of the frame swings his right hand to defend. The fencer on the left side of the frame then retracts his left hand and turns to leave, while the fencer on the right side of the frame turns around and walks away while raising his right hand and looking at the referee dressed in black near the table with the "Naked" banner. The background shows a busy arena with spectators sitting in the stands and other fencers or staff near the barriers. The banners on the fence read "USA Fencing" and "Naked", indicating that this is part of an official event. The high ceiling is supported by structural pillars, and bright lights illuminate the entire space. The atmosphere is focused and competitive, which is typical of competitive fencing events. The camera lens follows the two fencers as they move.\n''' \
+
+'''5.
+- **Original user inputs**: Players in red and white uniforms are actively engaged in the game, with the player in white number 5 dribbling the ball. The player in white number 5 continues to dribble and moves towards the basket, closely followed by players in red uniforms. The player in white number 5 attempts a shot while being closely guarded by the players in red. The player in white number 5 successfully makes a basket, and the players in red attempt to block the shot. The ball goes through the hoop, scoring a point. The players in red and white continue to move around the court, with the player in white number 5 preparing to make another move.
+- **Examples after rewriting**: The video records a lively men's basketball game, which is played in an indoor gymnasium. The gymnasium has high ceilings and exposed metal beams. The court is marked with regulation lines, and the walls are painted white and decorated with colorful banners and a large red banner with yellow Chinese characters. A group of 10 players in red and white jerseys are actively engaged in the game. A player wearing a white No. 5 jersey is moving towards the basket with the ball, while his opponent in a red jersey is in hot pursuit. As the play progresses, the player with the ball fakes first to the right and then to the left, bypassing the defender and shooting towards the basket. The ball arcs through the air and enters the basket, while other players move to grab rebounds or assists. Another player in a red jersey jumps up to try to block the shot. Players on the sidelines watch the game intently, with referees or coaches nearby, with some of whom sit at tables holding referee's referee documents. The bright gym lights ensure that the game is clearly visible, highlighting the competitive nature of the sport and the strategic interactions between players. The camera follows the movement of player No. 5.\n''' \
+
+'''6.
+- **Original user inputs**: In an indoor badminton court with green walls and a balcony, two men are engaged in a badminton game. The man in black hits the shuttlecock, and the man in white returns it.
+- **Examples after rewriting**: The video captures a dynamic game of badminton in an indoor court with a green floor and white boundary lines. The court is surrounded by green walls and has wooden floors and fixed badminton nets. There is a blue banner with white Chinese characters on the left wall and a brand logo on the right. Initially, the male player near the camera, wearing a black shirt and shorts, holds the racket in his right hand behind him and prepares to hit the ball, while the male player wearing a white shirt and dark shorts stands in a ready position on the opposite court. Both players are focused on the game, moving quickly and strategically. The black player hits the shuttlecock with the racket in his right hand extended forward with force, and the white player hits the shuttlecock with the racket in his right hand extended forward with force. Following the same pattern, the intensity of the game keeps them fully engaged, as they move across the court to return the shuttlecock and maintain their positions. Towards the end of the video, the shuttlecock appears to have been hit out of the court, indicated by the players' continued movements and gestures. In the background, there are several stationary exercise bikes against the wall and a few other people sitting or standing, perhaps observing or resting. The bright lights in the room highlight the action, emphasizing the speed and precision required for each shot. The video ends with the black player running to the left of the frame and the white player preparing to hit the shuttlecock with the racket in his right hand. The camera perspective does not change. The overall atmosphere reflects a competitive yet friendly sporting environment.\n''' \
+
+'''7.
+- **Original user inputs**: A man stands on a stone platform by the river. The man releases his grip and tucks his body into a forward roll. The man dives into the river, creating a splash upon entry.
+- **Examples after rewriting**: The video captures a man diving into a stone pool from a stone platform. The man is shirtless, wearing black shorts with white stripes and a black hat. Initially, the man stands on one leg on the edge of the platform, stretching his arms upward, ready to stretch. He then lowers his arms and prepares to jump, swinging his arms back and forth as he begins to move. The man leaps forward, leaning forward, stretching his arms forward, and his feet alternately downward, completing a somersault before entering the water, with his arms extended upward at the top and your legs extended upward at the bottom. As he enters the water, he creates a splash. The other man, also shirtless and wearing black shorts, stands nearby, observing the diver. The pool is surrounded by stone walls and has a traditional architectural style, with trees and buildings visible in the background. The calm water reflects the sunlight and ripples, highlighting the contrast between stillness and the splash created by the diver's entry. The camera remains stationary, capturing the action as it unfolds against this serene yet dynamic backdrop.\n''' \
+
+'''8.
+- **Original user inputs**: In an indoor table tennis facility with multiple tables, a man and a woman are engaged in a game of table tennis.
+- **Examples after rewriting**: The video captures an intense game of table tennis in an indoor gymnasium. The venue is spacious, with several tables and chairs neatly arranged in the background. The floor is covered with a red non-woven fabric, and the walls are painted beige and white, with overhead lights evenly illuminating the field. In the video, a man in a yellow shirt and black pants is playing against a woman wearing a white jacket, beige pants and white sneakers. At first, the man in yellow stands on the left side of the screen, leaning forward, holding the racket in right hand and stretching his arms forward, while the woman in white stands on the right side of the screen, ready to receive the ball. The man in yellow serves, and the woman in white receives the ball. Then the man in yellow and the woman in white both stand up straight and prepare to hit back. The two exchange blows successfully, and the game becomes fierce. The man in yellow hits a powerful ball, but the woman in white didn't miss it, and then she hits a powerful ball. The man in yellow hits the ball again, and the woman in white jumps up and hits a powerful ball, making the man in yellow raise his arm and give up. The fierce competitive atmosphere reflects in their focused expressions and agile movements, highlighting the dynamic nature of this sport. The camera angle remains fixed, providing a clear perspective of the action as it unfolds.\n''' \
+
+'''9.
+- **Original user inputs**: A woman stands on a blue gymnastics mat. She performs a series of spins and jumps, maintaining her balance and poise.
+- **Examples after rewriting**: The video captures a female gymnast performing on a blue mat during a competition. The gymnast wears a black leotard with flowing pink patterns, which contrasts sharply against the blue mat. Her hair is tied into two small pigtails, showing her concentration and focus. First, she stretches her arms horizontally and stands in a squatting position, showing her flexibility and control. Then she turns her body while raising one arm and bending the other, showing her agility and grace. She leaps into the air, maintaining a dynamic pose with one leg extended forward and the other backward, before returning to a squatting position and completing a roll, rolling into a front flip in the air, spinning twice to land steadily. She then stands up straight and stretches her arms horizontally to complete her performance with a confident hand gesture and a smile. The venue is an indoor gymnasium, and spectators can be seen sitting in the stands in the background. Judges sit at a table with a banner that reads "OLAY", observing and evaluating the performance. The gymnast runs energetically across the mat, leaps into the air with precision, and transitions into complex movements with great skill. The audience remains focused throughout, highlighting the competitive atmosphere of the event. The camera follows her movements, capturing her in action.''' \
+
diff --git a/Meissonic/InfinityStar/tools/run_infinity.py b/Meissonic/InfinityStar/tools/run_infinity.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f93872d30c10d226ae44b0c777f325161844f11
--- /dev/null
+++ b/Meissonic/InfinityStar/tools/run_infinity.py
@@ -0,0 +1,339 @@
+# Copyright (c) 2025 FoundationVision
+# SPDX-License-Identifier: MIT
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+import os.path as osp
+from typing import List
+import time
+import hashlib
+import shutil
+import re
+import json
+from typing import Dict
+
+import cv2
+import numpy as np
+import torch
+torch._dynamo.config.cache_size_limit=64
+from transformers import AutoTokenizer
+from PIL import Image, ImageEnhance
+import torch.nn.functional as F
+from torch.cuda.amp import autocast
+from timm.models import create_model
+import imageio
+
+from infinity.models.infinity import Infinity
+from infinity.utils.load import load_visual_tokenizer
+from infinity.models.basic import *
+import PIL.Image as PImage
+from torchvision.transforms.functional import to_tensor
+from huggingface_hub import split_torch_state_dict_into_shards
+from safetensors.torch import save_file as safe_save_file
+
+
+def split_state_dict(state_dict: Dict[str, torch.Tensor], save_directory: str, max_shard_size='8GB'):
+ state_dict_split = split_torch_state_dict_into_shards(state_dict, max_shard_size=max_shard_size)
+ for filename, tensors in state_dict_split.filename_to_tensors.items():
+ shard = {tensor: state_dict[tensor] for tensor in tensors}
+ safe_save_file(
+ shard,
+ os.path.join(save_directory, filename),
+ metadata={"format": "pt"},
+ )
+ if state_dict_split.is_sharded:
+ index = {
+ "metadata": state_dict_split.metadata,
+ "weight_map": state_dict_split.tensor_to_filename,
+ }
+ with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f:
+ f.write(json.dumps(index, indent=2))
+
+def extract_key_val(text):
+ pattern = r'<(.+?):(.+?)>'
+ matches = re.findall(pattern, text)
+ key_val = {}
+ for match in matches:
+ key_val[match[0]] = match[1].lstrip()
+ return key_val
+
+def encode_prompt(t5_path, text_tokenizer, text_encoder, prompt, enable_positive_prompt=False, low_vram_mode=False):
+ if enable_positive_prompt:
+ pass
+ print(f'prompt={prompt}')
+ captions = [prompt]
+ if 'flan-t5' in t5_path:
+ tokens = text_tokenizer(text=captions, max_length=512, padding='max_length', truncation=True, return_tensors='pt')
+ input_ids = tokens.input_ids.cuda(non_blocking=True)
+ mask = tokens.attention_mask.cuda(non_blocking=True)
+ text_features = text_encoder(input_ids=input_ids, attention_mask=mask)['last_hidden_state'].float()
+ lens: List[int] = mask.sum(dim=-1).tolist()
+ cu_seqlens_k = F.pad(mask.sum(dim=-1).to(dtype=torch.int32).cumsum_(0), (1, 0))
+ Ltext = max(lens)
+ kv_compact = []
+ for len_i, feat_i in zip(lens, text_features.unbind(0)):
+ kv_compact.append(feat_i[:len_i])
+ kv_compact = torch.cat(kv_compact, dim=0)
+ text_cond_tuple = (kv_compact, lens, cu_seqlens_k, Ltext)
+ else:
+ text_features = text_encoder(captions, 'cuda')
+ lens = [len(item) for item in text_features]
+ cu_seqlens_k = [0]
+ for len_i in lens:
+ cu_seqlens_k.append(cu_seqlens_k[-1] + len_i)
+ cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32)
+ Ltext = max(lens)
+ kv_compact = torch.cat(text_features, dim=0).float()
+ text_cond_tuple = (kv_compact, lens, cu_seqlens_k, Ltext)
+ return text_cond_tuple
+
+def gen_one_example(
+ infinity_test,
+ vae,
+ text_tokenizer,
+ text_encoder,
+ prompt,
+ cfg_list=[],
+ tau_list=[],
+ negative_prompt='',
+ scale_schedule=None,
+ top_k=900,
+ top_p=0.97,
+ cfg_sc=3,
+ cfg_exp_k=0.0,
+ cfg_insertion_layer=-5,
+ vae_type=0,
+ gumbel=0,
+ softmax_merge_topk=-1,
+ gt_leak=-1,
+ gt_ls_Bl=None,
+ g_seed=None,
+ sampling_per_bits=1,
+ enable_positive_prompt=0,
+ input_use_interplote_up=False,
+ low_vram_mode=False,
+ args=None,
+ get_visual_rope_embeds=None,
+ context_info=None,
+ noise_list=None,
+ return_summed_code_only=False,
+ mode='',
+ former_clip_features=None,
+ first_frame_features=None,
+):
+ sstt = time.time()
+ if not isinstance(cfg_list, list):
+ cfg_list = [cfg_list] * len(scale_schedule)
+ if not isinstance(tau_list, list):
+ tau_list = [tau_list] * len(scale_schedule)
+ text_cond_tuple = encode_prompt(args.text_encoder_ckpt, text_tokenizer, text_encoder, prompt, enable_positive_prompt, low_vram_mode=low_vram_mode)
+ if negative_prompt:
+ negative_label_B_or_BLT = encode_prompt(args.text_encoder_ckpt, text_tokenizer, text_encoder, negative_prompt, low_vram_mode=low_vram_mode)
+ else:
+ negative_label_B_or_BLT = None
+ print(f'cfg: {cfg_list}, tau: {tau_list}')
+ with torch.cuda.amp.autocast(enabled=True, dtype=torch.bfloat16, cache_enabled=True):
+ stt = time.time()
+ out = infinity_test.autoregressive_infer(
+ vae=vae,
+ scale_schedule=scale_schedule,
+ label_B_or_BLT=text_cond_tuple, g_seed=g_seed,
+ B=1, negative_label_B_or_BLT=negative_label_B_or_BLT, force_gt_Bhw=None,
+ cfg_sc=cfg_sc, cfg_list=cfg_list, tau_list=tau_list, top_k=top_k, top_p=top_p,
+ returns_vemb=1, ratio_Bl1=None, gumbel=gumbel, norm_cfg=False,
+ cfg_exp_k=cfg_exp_k, cfg_insertion_layer=cfg_insertion_layer,
+ vae_type=vae_type, softmax_merge_topk=softmax_merge_topk,
+ ret_img=True, trunk_scale=1000,
+ gt_leak=gt_leak, gt_ls_Bl=gt_ls_Bl, inference_mode=True,
+ sampling_per_bits=sampling_per_bits,
+ input_use_interplote_up=input_use_interplote_up,
+ low_vram_mode=low_vram_mode,
+ args=args,
+ get_visual_rope_embeds=get_visual_rope_embeds,
+ context_info=context_info,
+ noise_list=noise_list,
+ return_summed_code_only=return_summed_code_only,
+ mode=mode,
+ former_clip_features=former_clip_features,
+ first_frame_features=first_frame_features,
+ )
+ if return_summed_code_only:
+ return out
+ else:
+ pred_multi_scale_bit_labels, img_list = out
+
+ print(f"cost: {time.time() - sstt}, infinity cost={time.time() - stt}")
+ img = img_list[0]
+ return img, pred_multi_scale_bit_labels
+
+def get_prompt_id(prompt):
+ md5 = hashlib.md5()
+ md5.update(prompt.encode('utf-8'))
+ prompt_id = md5.hexdigest()
+ return prompt_id
+
+def save_slim_model(infinity_model_path, save_file=None, device='cpu', key='gpt_fsdp'):
+ print('[Save slim model]')
+ full_ckpt = torch.load(infinity_model_path, map_location=device)
+ infinity_slim = full_ckpt['trainer'][key]
+ # ema_state_dict = cpu_d['trainer'].get('gpt_ema_fsdp', state_dict)
+ if not save_file:
+ save_file = osp.splitext(infinity_model_path)[0] + '-slim.pth'
+ print(f'Save to {save_file}')
+ torch.save(infinity_slim, save_file)
+ print('[Save slim model] done')
+ return save_file
+
+def load_tokenizer(t5_path =''):
+ print(f'[Loading tokenizer and text encoder]')
+ if 'flan-t5' in t5_path:
+ from transformers import AutoTokenizer, T5EncoderModel, T5TokenizerFast
+ text_tokenizer: T5TokenizerFast = AutoTokenizer.from_pretrained(t5_path, revision=None, legacy=True)
+ # text_encoder: T5EncoderModel = T5EncoderModel.from_pretrained(t5_path, torch_dtype=torch.bfloat16)
+ text_encoder: T5EncoderModel = T5EncoderModel.from_pretrained(t5_path, torch_dtype=torch.float16)
+ text_encoder.to('cuda')
+ text_encoder.eval()
+ text_encoder.requires_grad_(False)
+ else:
+ raise ValueError(f'Not support t5_path: {t5_path}')
+ return text_tokenizer, text_encoder
+
+def transform(pil_img, tgt_h, tgt_w):
+ width, height = pil_img.size
+ if width / height <= tgt_w / tgt_h:
+ resized_width = tgt_w
+ resized_height = int(tgt_w / (width / height))
+ else:
+ resized_height = tgt_h
+ resized_width = int((width / height) * tgt_h)
+ pil_img = pil_img.resize((resized_width, resized_height), resample=PImage.LANCZOS)
+ # crop the center out
+ arr = np.array(pil_img)
+ crop_y = (arr.shape[0] - tgt_h) // 2
+ crop_x = (arr.shape[1] - tgt_w) // 2
+ im = to_tensor(arr[crop_y: crop_y + tgt_h, crop_x: crop_x + tgt_w])
+ return im.add(im).add_(-1)
+
+
+def load_transformer(vae, args):
+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+ model_path = args.model_path
+
+ print(f'[Loading Infinity]')
+ with torch.cuda.amp.autocast(enabled=True, dtype=torch.bfloat16, cache_enabled=True), torch.no_grad():
+ infinity_test: Infinity = create_model(
+ args.model_type,
+ vae_local=vae, text_channels=args.text_channels, text_maxlen=512,
+ raw_scale_schedule=None,
+ checkpointing='full-block',
+ pad_to_multiplier=128,
+ use_flex_attn=args.use_flex_attn,
+ add_lvl_embeding_on_first_block=0,
+ num_of_label_value=args.num_of_label_value,
+ rope2d_each_sa_layer=args.rope2d_each_sa_layer,
+ rope2d_normalized_by_hw=args.rope2d_normalized_by_hw,
+ pn=args.pn,
+ apply_spatial_patchify=args.apply_spatial_patchify,
+ inference_mode=True,
+ train_h_div_w_list=[0.571, 1.0],
+ video_frames=args.video_frames,
+ other_args=args,
+ ).to(device=device)
+ print(f'[you selected Infinity with {args.model_type}] model size: {sum(p.numel() for p in infinity_test.parameters())/1e9:.2f}B, bf16={args.bf16}')
+ if args.bf16:
+ for block in infinity_test.unregistered_blocks:
+ block.bfloat16()
+ infinity_test.eval()
+ infinity_test.requires_grad_(False)
+ infinity_test.cuda()
+ torch.cuda.empty_cache()
+
+ if not model_path:
+ return infinity_test
+
+ print(f'============== [Load Infinity weights] ==============')
+ if args.checkpoint_type == 'torch':
+ state_dict = torch.load(model_path, map_location=device)
+ if 'trainer' in state_dict:
+ print(infinity_test.load_state_dict(state_dict['trainer']['gpt_fsdp']))
+ else:
+ print(infinity_test.load_state_dict(state_dict))
+ elif args.checkpoint_type == 'torch_shard':
+ from transformers.modeling_utils import load_sharded_checkpoint
+ print(load_sharded_checkpoint(infinity_test, model_path, strict=False))
+ elif args.checkpoint_type == 'omnistore':
+ from infinity.utils.save_and_load import merge_ckpt
+ if args.enable_model_cache and osp.exists(args.cache_dir):
+ local_model_dir = osp.abspath(osp.join(args.cache_dir, 'tmp', model_path.replace('/', '_')))
+ else:
+ local_model_dir = osp.abspath(model_path)
+ print(f'load checkpoint from {local_model_dir}')
+ state_dict = merge_ckpt(local_model_dir, osp.join(local_model_dir, 'ouput'), save=False, fsdp_save_flatten_model=args.fsdp_save_flatten_model)
+ print(infinity_test.load_state_dict(state_dict))
+ infinity_test.rng = torch.Generator(device=device)
+ return infinity_test
+
+def save_video(ndarray_image_list, fps=24, save_filepath='tmp.mp4'):
+ if len(ndarray_image_list) == 1:
+ save_filepath = save_filepath.replace('.mp4', '.jpg')
+ cv2.imwrite(save_filepath, ndarray_image_list[0])
+ print(f"Image saved as {osp.abspath(save_filepath)}")
+ else:
+ h, w = ndarray_image_list[0].shape[:2]
+ os.makedirs(osp.dirname(save_filepath), exist_ok=True)
+ imageio.mimsave(save_filepath, ndarray_image_list[:, :, :, ::-1], fps=fps,)
+ print(f"Video saved as {osp.abspath(save_filepath)}")
+
+def read_video_as_frames(video_path):
+ if video_path.endswith('.jpg'):
+ return cv2.imread(video_path)[None, ...]
+ cap = cv2.VideoCapture(video_path)
+ if not cap.isOpened():
+ print(f"Error: Unable to open video file {video_path}")
+ return None
+ frames = []
+ frame_count = 0
+ while True:
+ ret, frame = cap.read()
+ if not ret:
+ break
+ frames.append(frame)
+ frame_count += 1
+ cap.release()
+ frames = np.stack(frames)
+ return frames
+
+def add_common_arguments(parser):
+ parser.add_argument('--cfg', type=str, default='3')
+ parser.add_argument('--tau', type=float, default=1)
+ parser.add_argument('--pn', type=str, required=True, choices=['0.06M', '0.25M', '0.40M', '0.90M'])
+ parser.add_argument('--model_path', type=str, default='')
+ parser.add_argument('--cfg_insertion_layer', type=int, default=0)
+ parser.add_argument('--vae_type', type=int, default=64)
+ parser.add_argument('--vae_path', type=str, default='')
+ parser.add_argument('--add_lvl_embeding_on_first_block', type=int, default=0, choices=[0,1])
+ parser.add_argument('--num_of_label_value', type=int, default=2)
+ parser.add_argument('--model_type', type=str, default='infinity_2b')
+ parser.add_argument('--rope2d_each_sa_layer', type=int, default=1, choices=[0,1])
+ parser.add_argument('--rope2d_normalized_by_hw', type=int, default=2, choices=[0,1,2])
+ parser.add_argument('--use_scale_schedule_embedding', type=int, default=0, choices=[0,1])
+ parser.add_argument('--sampling_per_bits', type=int, default=1, choices=[1,2,4,8,16])
+ parser.add_argument('--text_encoder_ckpt', type=str, default='')
+ parser.add_argument('--text_channels', type=int, default=2048)
+ parser.add_argument('--apply_spatial_patchify', type=int, default=0, choices=[0,1])
+ parser.add_argument('--h_div_w_template', type=float, default=1.000)
+ parser.add_argument('--use_flex_attn', type=int, default=0, choices=[0,1])
+ parser.add_argument('--enable_positive_prompt', type=int, default=0, choices=[0,1])
+ parser.add_argument('--cache_dir', type=str, default='/dev/shm')
+ parser.add_argument('--enable_model_cache', type=int, default=0, choices=[0,1])
+ parser.add_argument('--checkpoint_type', type=str, default='torch')
+ parser.add_argument('--seed', type=int, default=0)
+ parser.add_argument('--bf16', type=int, default=1, choices=[0,1])
+ parser.add_argument('--dynamic_scale_schedule', type=str, default='13_hand_craft')
+ parser.add_argument('--video_frames', type=int, default=81)
+ parser.add_argument('--videovae', type=int, default=10)
+ parser.add_argument('--fake_vae_input', type=int, default=0, choices=[0,1])
+ parser.add_argument('--casual_multi_scale', type=int, default=0, choices=[0,1])
+ parser.add_argument('--scale_embeds_num', type=int, default=128)
+ parser.add_argument('--train_h_div_w_list', type=float, default=None, nargs='+')
+ parser.add_argument('--mask_type', type=str, default='infinity_elegant_clip20frames_v2')
+ parser.add_argument('--context_frames', type=int, default=1000)
diff --git a/Meissonic/InfinityStar/tools/save_dataset_features.py b/Meissonic/InfinityStar/tools/save_dataset_features.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a525105981a6424e93f27333f650997d84a598a
--- /dev/null
+++ b/Meissonic/InfinityStar/tools/save_dataset_features.py
@@ -0,0 +1,209 @@
+# Copyright (c) 2025 FoundationVision
+# SPDX-License-Identifier: MIT
+import os
+import os.path as osp
+import time
+import gc
+import json
+import math
+import random
+import sys
+import argparse
+import copy
+import traceback
+import collections
+from collections import deque
+from contextlib import nullcontext
+from functools import partial
+from typing import List, Optional, Tuple
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+os.environ['XFORMERS_FORCE_DISABLE_TRITON'] = '1'
+import threading
+
+import numpy as np
+import torch
+from torch.nn import functional as F
+from torch.utils.data import DataLoader
+import torch.distributed as tdist
+import tqdm
+
+from tools.run_infinity import *
+from infinity.dataset.dataset_joint_vi import JointViIterableDataset
+from infinity.schedules.dynamic_resolution import get_dynamic_resolution_meta
+
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ add_common_arguments(parser)
+ parser.add_argument('--reweight_loss_by_scale', type=int, default=1, choices=[0,1])
+ parser.add_argument('--vis_model_flop_param', type=int, default=0, choices=[0,1])
+ parser.add_argument('--image_data_path', type=str, default='')
+ parser.add_argument('--video_data_path', type=str, default='')
+ parser.add_argument('--video_batch_size', type=int, default=1)
+ parser.add_argument('--image_batch_size', type=int, default=1)
+ parser.add_argument('--dataloader_workers', type=int, default=12)
+ parser.add_argument('--noise_apply_layers', type=int, default=20)
+ parser.add_argument('--noise_apply_requant', type=int, default=1, choices=[0,1])
+ parser.add_argument('--noise_apply_strength', type=float, default=0.2)
+ parser.add_argument('--debug_bsc', type=int, default=0, choices=[0,1])
+ parser.add_argument('--log_freq', type=int, default=10)
+ parser.add_argument('--video_fps', type=int, default=24)
+ parser.add_argument('--steps_per_frame', type=int, default=4)
+ parser.add_argument('--video_tower_style', type=str, default='bottom')
+ parser.add_argument('--use_slice', type=int, default=1, choices=[0,1])
+ parser.add_argument('--use_vae_token_cache', type=int, default=1, choices=[0,1])
+ parser.add_argument('--allow_online_vae_feature_extraction', type=int, default=1, choices=[0,1])
+ parser.add_argument('--use_text_token_cache', type=int, default=1, choices=[0,1])
+ parser.add_argument('--image_batches_multiply', type=float, default=1)
+ parser.add_argument('--token_cache_dir', type=str, default='/mnt/bn/genai-data2/hanjian.thu123/vae_features')
+ parser.add_argument('--down_size_limit', type=int, default=10000)
+ parser.add_argument('--addition_pn006M', type=int, default=1, choices=[0,1])
+ parser.add_argument('--addition_pn025M', type=int, default=1, choices=[0,1])
+ parser.add_argument('--video_caption_type', type=str, default='tarsier2_caption')
+ parser.add_argument('--only_images4extract_feats', type=int, default=1, choices=[0,1])
+ parser.add_argument('--temporal_compress_rate', type=int, default=4)
+ parser.add_argument('--cached_video_frames', type=int, default=81)
+ parser.add_argument('--duration_resolution', type=float, default=0.005)
+ parser.add_argument('--train_max_token_len', type=int, default=20480)
+ parser.add_argument('--cache_check_mode', type=int, default=0)
+ parser.add_argument('--seq_pack_bucket', type=int, default=1)
+ parser.add_argument('--drop_long_video', type=int, default=0)
+ parser.add_argument('--append_duration2caption', type=int, default=0)
+ parser.add_argument('--min_video_frames', type=int, default=32)
+ parser.add_argument('--addition_pn_list', type=str, default='[]')
+ parser.add_argument('--semantic_scale_dim', type=int, default=16)
+ parser.add_argument('--detail_scale_dim', type=int, default=64)
+ parser.add_argument('--use_learnable_dim_proj', type=int, default=0)
+ parser.add_argument('--detail_scale_min_tokens', type=int, default=80)
+ parser.add_argument('--semantic_scales', type=int, default=80)
+
+ parser.add_argument('--tlen', type=int, default=512)
+ parser.add_argument('--manual_parallel', action="store_true")
+ parser.add_argument('--num_replicas', type=int, default=-1) # only valid when manual_parallel is True
+ parser.add_argument('--rank', type=int, default=-1) # only valid when manual_parallel is True
+ parser.add_argument('--restrict_data_size', type=int, default=-1)
+ parser.add_argument('--allow_less_one_elem_in_seq', type=int, default=1)
+ parser.add_argument('--use_feat_proj', type=int, default=2)
+ parser.add_argument('--use_two_stage_lfq', type=int, default=1)
+ parser.add_argument('--epoch', type=int, default=0)
+
+ args = parser.parse_args()
+
+ if args.manual_parallel:
+ device = "cuda:0"
+ num_replicas = args.num_replicas
+ rank = args.rank
+ assert num_replicas > 0 and rank >= 0
+ else:
+ tdist.init_process_group(backend='nccl')
+ device = torch.device(tdist.get_rank() % torch.cuda.device_count())
+ num_replicas = tdist.get_world_size()
+ rank=tdist.get_rank()
+ args.device = device
+ args.text_tokenizer = None
+ args.duration_resolution = 4 / args.video_fps
+
+ # load vae
+ vae = load_visual_tokenizer(args, device=device)
+
+ dataset = JointViIterableDataset(
+ image_meta_folder=args.image_data_path,
+ video_meta_folder=args.video_data_path,
+ max_caption_len=512,
+ short_prob=0.0,
+ load_vae_instead_of_image=False,
+ pn=args.pn,
+ seed=args.seed,
+ video_fps=args.video_fps,
+ num_frames=args.video_frames,
+ online_t5=True,
+ num_replicas=num_replicas, # 1,
+ rank=rank, # 0
+ dataloader_workers=args.dataloader_workers,
+ dynamic_resolution_across_gpus=0,
+ enable_dynamic_length_prompt=0,
+ dynamic_scale_schedule=args.dynamic_scale_schedule,
+ add_motion_score2caption=0,
+ other_args=args,
+ )
+ dataloader = DataLoader(dataset, batch_size=None, num_workers=args.dataloader_workers, pin_memory=True)
+ print(f'len(dataloader): {len(dataloader)}, len(dataset): {len(dataset)}')
+ t1 = time.time()
+ dataloader.dataset.set_epoch(0)
+ pbar = tqdm.tqdm(total=len(dataloader))
+ accumulate_res = collections.defaultdict(list)
+ dynamic_resolution_h_w, h_div_w_templates = get_dynamic_resolution_meta(args.dynamic_scale_schedule, args.video_frames)
+
+ print(device)
+ vae.to(device)
+
+ def save_token():
+ while True:
+ try:
+ raw_features, feature_cache_files4images = save_token_queue.get()
+ for i in range(len(feature_cache_files4images)):
+ if not osp.exists(feature_cache_files4images[i]):
+ os.makedirs(osp.dirname(feature_cache_files4images[i]), exist_ok=True)
+ torch.save(raw_features[i], feature_cache_files4images[i])
+ print(f'Save to {feature_cache_files4images[i]}')
+ else:
+ print(f'{feature_cache_files4images[i]} exists, skip')
+ except Exception as e:
+ print(f"Error saving token: {e}")
+ finally:
+ save_token_queue.task_done()
+
+ import queue
+ save_token_queue = queue.Queue()
+ saver = threading.Thread(target=save_token, daemon=True)
+ saver.start()
+
+ data_time = time.time()
+ iter_time = time.time()
+
+ pn_list = [args.pn] + json.loads(args.addition_pn_list)
+ pn_list = list(set(pn_list))
+
+ for i, data in enumerate(iter(dataloader)):
+ pbar.update(1)
+ # print(f"[step {i}]: iter time: {time.time() - iter_time:.2f}s, data time:{time.time() - data_time:.2f}s")
+ iter_time = time.time()
+ # print("data time", iter_time - data_time)
+
+ captions, feature_cache_files4images, raw_features_bcthw = data['captions'], data['feature_cache_files4images'], data['raw_features_bcthw']
+ # print(len(feature_cache_files4images))
+ if args.only_images4extract_feats:
+ assert len(raw_features_bcthw) == 0
+ if not len(feature_cache_files4images):
+ continue
+
+ for pn_ind, pn in enumerate(pn_list):
+ if pn == args.pn:
+ inp_B3HW = data['images']
+ else:
+ inp_B3HW = data['addition_pn_images'][f'img_T3HW_{pn}']
+ try:
+ # print(f"args.pn:{args.pn}, pn:{pn}")
+ cur_feature_cache_files4images = [item.replace(f'pn_{args.pn}', f'pn_{pn}') for item in feature_cache_files4images]
+ except Exception as e:
+ import pdb; pdb.set_trace()
+ assert len(inp_B3HW) == len(cur_feature_cache_files4images)
+
+ for images_CTHW, feature_save_file in zip(inp_B3HW, cur_feature_cache_files4images):
+ try:
+ pt = images_CTHW.shape[-3]
+ h_div_w = images_CTHW.shape[-2] / images_CTHW.shape[-1]
+ h_div_w_templates = np.array(list(dynamic_resolution_h_w.keys()))
+ h_div_w_template = h_div_w_templates[np.argmin(np.abs(h_div_w-h_div_w_templates))]
+ # [forward]
+ with torch.amp.autocast('cuda', enabled=False):
+ with torch.no_grad():
+ raw_features, _, _ = vae.encode_for_raw_features(images_CTHW.unsqueeze(0).to(device), scale_schedule=None, slice=args.use_slice)
+ raw_features = raw_features.cpu().data
+ save_token_queue.put((raw_features, [feature_save_file]))
+ except Exception as e:
+ print(e)
+ data_time = time.time()
+ # print("iter time", data_time - iter_time)
+ save_token_queue.join()
diff --git a/Meissonic/InfinityStar/train.py b/Meissonic/InfinityStar/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d7b131b82ae6ee4054045a5c6a34dcdadf1f76f
--- /dev/null
+++ b/Meissonic/InfinityStar/train.py
@@ -0,0 +1,466 @@
+# Copyright (c) 2025 FoundationVision
+# SPDX-License-Identifier: MIT
+import gc
+import json
+import math
+import os
+import os.path as osp
+import random
+import sys
+import time
+import traceback
+from collections import deque
+from contextlib import nullcontext
+from functools import partial
+from distutils.util import strtobool
+from typing import List, Optional, Tuple
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+os.environ['XFORMERS_FORCE_DISABLE_TRITON'] = '1'
+# os.environ["TORCH_LOGS"] = "+dynamo"
+# os.environ["TORCHDYNAMO_VERBOSE"] = '1'
+
+import numpy as np
+import torch
+torch._dynamo.config.cache_size_limit = 64
+from torch.nn import functional as F
+from torch.profiler import record_function
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer, T5EncoderModel, T5TokenizerFast
+import torch.distributed as tdist
+
+import infinity.utils.dist as dist
+from infinity.dataset.build import build_joint_dataset
+from infinity.utils.save_and_load import CKPTSaver, omnistoreCheckpoint, auto_resume, omnistore_auto_resume
+from infinity.models.ema import get_ema_model
+from infinity.utils import arg_util, misc, wandb_utils
+from infinity.trainer import get_trainer
+# from infinity.utils.mfu.mfu import mfutool
+
+def build_everything_from_args(args: arg_util.Args, saver):
+ # set seed
+ args.set_initial_seed(benchmark=True)
+ # build tokenizer
+ print(f'Loading T5 from {args.t5_path}...')
+ if 'flan-t5' in args.t5_path:
+ from transformers import T5EncoderModel, T5TokenizerFast
+ text_tokenizer: T5TokenizerFast = AutoTokenizer.from_pretrained(args.t5_path, revision=None, legacy=True) # text_tokenizer.model_max_length is 512
+ text_tokenizer.model_max_length = args.tlen
+ text_encoder: T5EncoderModel = T5EncoderModel.from_pretrained(args.t5_path, torch_dtype=torch.float16)
+ text_encoder.to(args.device)
+ text_encoder.eval()
+ text_encoder.requires_grad_(False)
+ args.text_tokenizer_type = 'flan_t5'
+ args.text_tokenizer = text_tokenizer
+ else: # umt5
+ raise ValueError("Only flan-t5 is supported now.")
+
+ # build models. Note that here gpt is the causal VAR transformer which performs next scale prediciton with text guidance
+ vae_local, gpt_uncompiled, gpt_wo_ddp, gpt_ddp, gpt_wo_ddp_ema, gpt_ddp_ema, gpt_optim = build_model_optimizer(args)
+
+ # IMPORTANT: import heavy package `InfinityTrainer` after the Dataloader object creation/iteration to avoid OOM
+ InfinityTrainer = get_trainer(args)
+ # build trainer
+ trainer = InfinityTrainer(
+ device=args.device,
+ raw_scale_schedule=args.scale_schedule,
+ vae_local=vae_local,
+ gpt_wo_ddp=gpt_wo_ddp, gpt=gpt_ddp,
+ gpt_opt=gpt_optim,
+ label_smooth=args.label_smooth,
+ zero=args.zero,
+ vae_type=args.vae_type,
+ reweight_loss_by_scale=args.reweight_loss_by_scale,
+ gpt_wo_ddp_ema=gpt_wo_ddp_ema,
+ gpt_ema=gpt_ddp_ema,
+ use_fsdp_model_ema=args.use_fsdp_model_ema,
+ other_args=args,
+ )
+
+ # auto resume from broken experiment
+ global_it = 0
+ if args.checkpoint_type == 'torch':
+ auto_resume_info, start_ep, global_it, acc_str, _, trainer_state, _ = auto_resume(args, 'global_step_*')
+ if trainer_state is not None and len(trainer_state):
+ trainer.load_state_dict(trainer_state, strict=False, skip_vae=True)
+ elif args.checkpoint_type == 'omnistore':
+ resume_path, info = omnistore_auto_resume(args, 'global_step_*')
+ if not resume_path and args.rush_omnistore_resume:
+ resume_path = args.rush_omnistore_resume
+ if resume_path:
+ print(f"omnistore resume from {resume_path}", flush=True)
+ args_state, start_ep, start_it, global_it, acc_str, eval_milestone = saver.load(resume_path, fsdp_object=trainer.gpt, optimizer_object=trainer.gpt_opt.optimizer)
+ dist.barrier()
+ if args.rush_omnistore_resume == resume_path:
+ global_it = 0
+ auto_resume_info, acc_str, eval_milestone, trainer_state, args_state = info, '[no acc str]', [], {}, {}
+
+ del vae_local, gpt_uncompiled, gpt_wo_ddp, gpt_ddp, gpt_wo_ddp_ema, gpt_ddp_ema, gpt_optim
+ dist.barrier()
+ return text_tokenizer, text_encoder, trainer, global_it
+
+
+def build_model_optimizer(args):
+ from torch.nn.parallel import DistributedDataParallel as DDP
+ from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+ from infinity.models.infinity import Infinity, MultipleLayers
+ from infinity.models.init_param import init_weights
+ from infinity.utils.amp_opt import AmpOptimizer
+ from infinity.utils.lr_control import filter_params
+ from infinity.utils.load import build_vae_gpt
+
+ # disable builtin initialization for speed
+ setattr(torch.nn.Linear, 'reset_parameters', lambda self: None)
+ setattr(torch.nn.LayerNorm, 'reset_parameters', lambda self: None)
+ vae_local, gpt_wo_ddp = build_vae_gpt(args, device=args.model_init_device)
+ count_p = lambda m: sum(p.numel() for p in m.parameters()) / 1e6
+ num_para = count_p(gpt_wo_ddp)
+ if num_para/1000 < 20: # < 20B
+ gpt_wo_ddp = gpt_wo_ddp.to('cuda')
+
+ if args.tini < 0:
+ args.tini = math.sqrt(1 / gpt_wo_ddp.C / 3)
+ init_weights(gpt_wo_ddp, other_std=args.tini)
+ gpt_wo_ddp.special_init()
+ if args.use_fsdp_model_ema:
+ gpt_wo_ddp_ema = get_ema_model(gpt_wo_ddp)
+ else:
+ gpt_wo_ddp_ema = None
+
+ if args.rush_resume:
+ print(f"{args.rush_resume=}")
+ cpu_d = torch.load(args.rush_resume, 'cpu')
+ if 'trainer' in cpu_d:
+ state_dict = cpu_d['trainer']['gpt_fsdp']
+ ema_state_dict = cpu_d['trainer'].get('gpt_ema_fsdp', state_dict)
+ else:
+ state_dict = cpu_d
+ ema_state_dict = state_dict
+ def drop_unfit_weights(state_dict):
+ if 'word_embed.weight' in state_dict and (state_dict['word_embed.weight'].shape[1] != gpt_wo_ddp.word_embed.in_features):
+ print(f'[rush_resume] drop word_embed.weight')
+ del state_dict['word_embed.weight']
+ if 'head.weight' in state_dict and (state_dict['head.weight'].shape[0] != gpt_wo_ddp.head.out_features):
+ print(f'[rush_resume] drop head.weight')
+ del state_dict['head.weight']
+ if 'head.bias' in state_dict and (state_dict['head.bias'].shape[0] != gpt_wo_ddp.head.bias.shape[0]):
+ print(f'[rush_resume] drop head.bias')
+ del state_dict['head.bias']
+ if 'text_proj_for_sos.ca.mat_kv.weight' in state_dict and \
+ (state_dict['text_proj_for_sos.ca.mat_kv.weight'].shape != gpt_wo_ddp.text_proj_for_sos.ca.mat_kv.weight.shape):
+ print(f'[rush_resume] drop cfg_uncond')
+ del state_dict['cfg_uncond']
+ for key in list(state_dict.keys()):
+ if 'text' in key:
+ del state_dict[key]
+ if 'semantic_head.weight' in state_dict:
+ print(f'[rush_resume] replace semantic_head with semantic_head2')
+ state_dict['semantic_head2.weight'] = state_dict['semantic_head.weight']
+ state_dict['semantic_head2.bias'] = state_dict['semantic_head.bias']
+ del state_dict['semantic_head.weight']
+ del state_dict['semantic_head.bias']
+ if 'semantic_head2.weight' in state_dict and (state_dict['semantic_head2.weight'].shape[0] != gpt_wo_ddp.semantic_head2.out_features):
+ print(f'[rush_resume] drop semantic_head2.weight, semantic_head2.bias')
+ del state_dict['semantic_head2.weight']
+ del state_dict['semantic_head2.bias']
+ return state_dict
+ print(gpt_wo_ddp.load_state_dict(drop_unfit_weights(state_dict), strict=False))
+ if args.use_fsdp_model_ema:
+ gpt_wo_ddp_ema.load_state_dict(drop_unfit_weights(ema_state_dict), strict=False)
+ elif args.torchshard_resume:
+ from transformers.modeling_utils import load_sharded_checkpoint
+ load_sharded_checkpoint(gpt_wo_ddp, args.torchshard_resume, strict=False)
+
+ ndim_dict = {name: para.ndim for name, para in gpt_wo_ddp.named_parameters() if para.requires_grad}
+
+ print(f'[PT] GPT model = {gpt_wo_ddp}\n\n')
+ print(f'[PT][#para], GPT={num_para:.2f}\n\n')
+
+ gpt_uncompiled = gpt_wo_ddp
+
+ gpt_ddp_ema = None
+ if args.zero:
+ from torch.distributed.fsdp import ShardingStrategy
+ from torch.distributed.fsdp.wrap import ModuleWrapPolicy
+ from torch.distributed.device_mesh import init_device_mesh
+
+ # use mix prec: https://github.com/pytorch/pytorch/issues/76607
+ if gpt_wo_ddp.num_block_chunks == 1: # no chunks
+ auto_wrap_policy = ModuleWrapPolicy([type(gpt_wo_ddp.unregistered_blocks[0]), ])
+ else:
+ auto_wrap_policy = ModuleWrapPolicy([MultipleLayers, ])
+
+ if args.enable_hybrid_shard:
+ sharding_strategy = ShardingStrategy.HYBRID_SHARD if args.zero == 3 else ShardingStrategy._HYBRID_SHARD_ZERO2
+ world_size = dist.get_world_size()
+ assert world_size % args.inner_shard_degree == 0
+ assert args.inner_shard_degree > 1 and args.inner_shard_degree < world_size
+ device_mesh = init_device_mesh('cuda', (world_size // args.inner_shard_degree, args.inner_shard_degree))
+ else:
+ sharding_strategy = ShardingStrategy.FULL_SHARD if args.zero == 3 else ShardingStrategy.SHARD_GRAD_OP
+ device_mesh = None
+ print(f'{">" * 45 + " " * 5} FSDP INIT with {args.zero=} {sharding_strategy=} {auto_wrap_policy=} {" " * 5 + "<" * 45}', flush=True)
+
+ if args.fsdp_init_device == 'cpu':
+ gpt_wo_ddp = gpt_wo_ddp.cpu()
+
+ gpt_ddp: FSDP = FSDP(
+ gpt_wo_ddp,
+ device_id=dist.get_local_rank(),
+ sharding_strategy=sharding_strategy,
+ mixed_precision=None,
+ auto_wrap_policy=auto_wrap_policy,
+ use_orig_params=True,
+ sync_module_states=True,
+ limit_all_gathers=True,
+ device_mesh=device_mesh,
+ ).to(args.device)
+
+ if args.use_fsdp_model_ema:
+ gpt_wo_ddp_ema = gpt_wo_ddp_ema.to(args.device)
+ gpt_ddp_ema: FSDP = FSDP(
+ gpt_wo_ddp_ema,
+ device_id=dist.get_local_rank(),
+ sharding_strategy=sharding_strategy,
+ mixed_precision=None,
+ auto_wrap_policy=auto_wrap_policy,
+ use_orig_params=args.fsdp_orig,
+ sync_module_states=True,
+ limit_all_gathers=True,
+ )
+ else:
+ ddp_class = DDP if dist.initialized() else misc.NullDDP
+ gpt_ddp: DDP = ddp_class(gpt_wo_ddp, device_ids=[dist.get_local_rank()], find_unused_parameters=False, broadcast_buffers=False)
+ torch.cuda.synchronize()
+
+ # =============== build optimizer ===============
+ nowd_keys = set()
+ if args.disable_weight_decay:
+ nowd_keys |= {
+ 'cls_token', 'start_token', 'task_token', 'cfg_uncond',
+ 'pos_embed', 'pos_1LC', 'pos_start', 'start_pos', 'lvl_embed',
+ 'gamma', 'beta',
+ 'ada_gss', 'moe_bias',
+ 'scale_mul',
+ 'text_proj_for_sos.ca.mat_q',
+ }
+ names, paras, para_groups = filter_params(gpt_ddp if args.zero else gpt_wo_ddp, ndim_dict, nowd_keys=nowd_keys)
+ del ndim_dict
+ if '_' in args.ada:
+ beta0, beta1 = map(float, args.ada.split('_'))
+ else:
+ beta0, beta1 = float(args.ada), -1
+
+ opt_clz = {
+ 'sgd': partial(torch.optim.SGD, momentum=beta0, nesterov=True),
+ 'adam': partial(torch.optim.AdamW, betas=(beta0, beta1), fused=args.fused_adam),
+ 'adamw': partial(torch.optim.AdamW, betas=(beta0, beta1), fused=args.fused_adam),
+ }[args.opt]
+ opt_kw = dict(lr=args.tlr, weight_decay=0)
+ if args.adam_eps: opt_kw['eps'] = args.adam_eps
+ print(f'[vgpt] optim={opt_clz}, opt_kw={opt_kw}\n')
+ gpt_optim = AmpOptimizer('gpt', args.fp16, opt_clz(params=para_groups, **opt_kw), gpt_ddp if args.zero else gpt_wo_ddp, args.r_accu, args.grad_clip, args.zero)
+ del names, paras, para_groups
+ return vae_local, gpt_uncompiled, gpt_wo_ddp, gpt_ddp, gpt_wo_ddp_ema, gpt_ddp_ema, gpt_optim
+
+
+def build_dataset(args):
+ train_dataset = build_joint_dataset(
+ args,
+ args.data_path,
+ args.video_data_path,
+ max_caption_len=args.tlen,
+ short_prob=args.short_cap_prob,
+ load_vae_instead_of_image=False
+ )
+ return train_dataset
+
+def main_train(args: arg_util.Args):
+ if args.checkpoint_type == 'torch':
+ saver = CKPTSaver(dist.is_master(), eval_milestone=None)
+ elif args.checkpoint_type == 'omnistore':
+ saver = omnistoreCheckpoint(eval_milestone=None)
+ else:
+ raise ValueError(f'{args.checkpoint_type=}')
+ ret = build_everything_from_args(args, saver)
+
+ if ret is None:
+ return
+
+ text_tokenizer, text_encoder, trainer, start_global_it = ret
+ gc.collect(), torch.cuda.empty_cache()
+ seg5 = np.linspace(1, args.epoch, 5+1, dtype=int).tolist()
+
+ time.sleep(3), gc.collect(), torch.cuda.empty_cache(), time.sleep(3)
+ ep_lg = max(1, args.epoch // 10) if args.epoch <= 100 else max(1, args.epoch // 20)
+
+ # ============================================= epoch loop begins =============================================
+ # build wandb logger
+ if dist.is_master():
+ wandb_utils.wandb.init(project=args.project_name, name=args.exp_name, config={})
+ for ep in range(args.epoch):
+ # build data at each epoch to ensure read meta take effects for each dataloader worker
+ args.epoch = ep
+
+ if ep == 0:
+ train_dataset = build_dataset(args)
+ iters_train = len(train_dataset)
+ start_ep = start_global_it // iters_train
+ start_it = start_global_it % iters_train
+ print(f'[PT info] from ep{start_ep} it{start_it} {iters_train=}=======> bed: {args.bed} <=======\n')
+
+ if ep < start_ep:
+ continue
+ if ep > start_ep:
+ train_dataset = build_dataset(args)
+ iters_train = len(train_dataset)
+
+ # [train one epoch]
+ train_dataloader = DataLoader(dataset=train_dataset, num_workers=args.workers, pin_memory=True, batch_size=None)
+ stats = train_one_epoch(
+ epoch=ep,
+ is_first_ep=ep == start_ep,
+ start_it=start_it if ep == start_ep else 0,
+ start_global_it=start_global_it,
+ me=None,
+ saver=saver,
+ args=args,
+ dataloader_iter=iter(train_dataloader),
+ iters_train=iters_train,
+ text_tokenizer=text_tokenizer, text_encoder=text_encoder,
+ trainer=trainer,
+ )
+
+ del stats, train_dataset, train_dataloader
+ return
+
+
+g_speed_ls = deque(maxlen=128)
+def train_one_epoch(
+ epoch: int, is_first_ep: bool, start_it: int, start_global_it: int, me: misc.MetricLogger,
+ saver: CKPTSaver, args: arg_util.Args, dataloader_iter, iters_train: int,
+ text_tokenizer: T5TokenizerFast, text_encoder: T5EncoderModel, trainer,
+):
+ # IMPORTANT: import heavy packages after the Dataloader object creation/iteration to avoid OOM
+ step_cnt = 0
+ header = f'[Ep]: [{epoch:4d}/{args.epoch}]'
+
+ last_touch = time.time()
+ g_it, max_it = epoch * iters_train, args.epoch * iters_train
+
+ doing_profiling = args.prof and epoch == 0 and (args.profall or dist.is_master())
+ maybe_record_function = record_function if doing_profiling else nullcontext
+ trainer.gpt_wo_ddp.maybe_record_function = maybe_record_function
+
+ last_t_perf = time.time()
+ speed_ls: deque = g_speed_ls
+ FREQ = min(args.prof_freq, iters_train//2-1)
+ NVIDIA_IT_PLUS_1 = set(FREQ*i for i in (1, 2, 3, 4, 6, 8))
+ ranges = set([2 ** i for i in range(20)])
+ if epoch <= 1: ranges |= {1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 32, 40}
+ PRINTABLE_IT_PLUS_1 = set(FREQ*i for i in ranges)
+
+ me = misc.MetricLogger()
+ [me.add_meter(x, misc.SmoothedValue(window_size=1, fmt='{value:.2g}')) for x in ['tlr']]
+ [me.add_meter(x, misc.SmoothedValue(window_size=1, fmt='{median:.2f} ({global_avg:.2f})')) for x in ['tnm']]
+ [me.add_meter(x, misc.SmoothedValue(window_size=1, fmt='{median:.3f} ({global_avg:.3f})')) for x in ['L', 'L_i', 'L_v']]
+ [me.add_meter(x, misc.SmoothedValue(window_size=1, fmt='{median:.2f} ({global_avg:.2f})')) for x in ['Acc', 'Acc_i', 'Acc_v']]
+ [me.add_meter(x, misc.SmoothedValue(window_size=1, fmt='{median:.2f} ({global_avg:.2f})')) for x in ['seq_usage']]
+ # ============================================= iteration loop begins =============================================
+ for it, data in me.log_every(start_it, iters_train, dataloader_iter, args.log_freq, args.log_every_iter, header, args):
+ g_it = epoch * iters_train + it
+ # mfutool.step()
+ # mfu_val = mfutool.get_mfu() * 100 # to percent
+ # print(f"[MFU] step={g_it}, mfu={mfu_val:.2f} %, mfu.iter_time = {mfutool.iter_time():.4f} s")
+
+
+ if (it+1) % FREQ == 0:
+ speed_ls.append((time.time() - last_t_perf) / FREQ)
+ last_t_perf = time.time()
+
+ if (g_it+1) % args.save_model_iters_freq == 0:
+ if args.checkpoint_type == 'torch':
+ saver.sav(args=args, g_it=(g_it+1), next_ep=epoch, next_it=it+1, trainer=trainer, acc_str=f'[todo]', eval_milestone=None, also_save_to=None, best_save_to=None)
+ elif args.checkpoint_type == 'omnistore':
+ saver.sav(args=args, global_it=(g_it+1), next_ep=epoch, next_it=it+1, fsdp_object=trainer.gpt, optimizer_object=trainer.gpt_opt.optimizer, acc_str=None, eval_milestone=None)
+
+ with maybe_record_function('before_train'):
+ # [get data]
+ images, captions, raw_features_bcthw, feature_cache_files4images, media = data['images'], data['captions'], data['raw_features_bcthw'], data['feature_cache_files4images'], data['media']
+
+ # # [prepare text features]
+ if args.text_tokenizer_type == 'flan_t5':
+ tokens = text_tokenizer(text=captions, max_length=text_tokenizer.model_max_length, padding='max_length', truncation=True, return_tensors='pt') # todo: put this into dataset
+ input_ids = tokens.input_ids.cuda(non_blocking=True)
+ mask = tokens.attention_mask.cuda(non_blocking=True)
+ text_features = text_encoder(input_ids=input_ids, attention_mask=mask)['last_hidden_state'].float()
+ lens: List[int] = mask.sum(dim=-1).tolist()
+ cu_seqlens_k = F.pad(mask.sum(dim=-1).to(dtype=torch.int32).cumsum_(0), (1, 0))
+ Ltext = max(lens)
+ kv_compact = []
+ for text_ind, (len_i, feat_i) in enumerate(zip(lens, text_features.unbind(0))):
+ kv_compact.append(feat_i[:len_i])
+ kv_compact = torch.cat(kv_compact, dim=0)
+ text_cond_tuple: Tuple[torch.FloatTensor, List[int], torch.LongTensor, int] = (kv_compact, lens, cu_seqlens_k, Ltext)
+ else:
+ text_features = text_encoder(captions, args.device)
+ lens = [len(item) for item in text_features]
+ cu_seqlens_k = [0]
+ for len_i in lens:
+ cu_seqlens_k.append(cu_seqlens_k[-1] + len_i)
+ cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32)
+ Ltext = max(lens)
+ kv_compact = torch.cat(text_features, dim=0).float()
+ text_cond_tuple = (kv_compact, lens, cu_seqlens_k, Ltext)
+
+ if len(images):
+ images = [item.to(args.device, non_blocking=True) for item in images]
+ if len(raw_features_bcthw):
+ raw_features_bcthw = [item.to(args.device, non_blocking=True) for item in raw_features_bcthw]
+
+ # [logging]
+ if dist.is_local_master() and (it >= start_it + 10) and (time.time() - last_touch > 90):
+ args.dump_log()
+ last_touch = time.time()
+
+ # [get scheduled hyperparameters]
+ progress = g_it / (max_it - 1)
+ clip_decay_ratio = (0.3 ** (20 * progress) + 0.2) if args.cdec else 1
+
+ stepping = (g_it + 1) % args.ac == 0
+ step_cnt += int(stepping)
+
+ with maybe_record_function('in_training'):
+ grad_norm_t, scale_log2_t = trainer.train_step(
+ epoch=epoch,
+ it=it,
+ g_it=g_it,
+ stepping=stepping,
+ clip_decay_ratio=clip_decay_ratio,
+ metric_lg=me,
+ inp_B3HW=images,
+ raw_features_bcthw=raw_features_bcthw,
+ feature_cache_files4images=feature_cache_files4images,
+ text_cond_tuple=text_cond_tuple,
+ media=media,
+ args=args,
+ )
+
+ with maybe_record_function('after_train'):
+ me.update(tlr=args.tlr)
+ # ============================================= iteration loop ends =============================================
+
+ me.synchronize_between_processes()
+ return {k: meter.global_avg for k, meter in me.meters.items()}
+
+
+def main():
+ args: arg_util.Args = arg_util.init_dist_and_get_args()
+ main_train(args)
+ print(f'final args:\n\n{str(args)}')
+ args.dump_log()
+ if isinstance(sys.stdout, dist.BackupStreamToFile) and isinstance(sys.stderr, dist.BackupStreamToFile):
+ sys.stdout.close(), sys.stderr.close()
+ dist.barrier()
+
+if __name__ == '__main__':
+ main()
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/comparison.mp4 b/Meissonic/InfinityStar/vae_reconstruction_test/comparison.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2f8af955e72b0c0884efc16bc69d83f98740cf42
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/comparison.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c179a456120edfa8df2d54bf438270503f3210101efbc0acf096ef32e6a85e26
+size 596649
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/comparison_grid.png b/Meissonic/InfinityStar/vae_reconstruction_test/comparison_grid.png
new file mode 100644
index 0000000000000000000000000000000000000000..965824044f75d3638f6efaaad0cfcf816f06a826
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/comparison_grid.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d026c6309def71e22a232ddc202461dc2276a3022880cbfc45c9b9090b9871bb
+size 6041734
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/frame_000_comparison.png b/Meissonic/InfinityStar/vae_reconstruction_test/frame_000_comparison.png
new file mode 100644
index 0000000000000000000000000000000000000000..ac667fe706da3f88167748b9bccc97ac9e2bc117
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/frame_000_comparison.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290d7c3bf99413b965e42022bedc6eb3115486ea7bd7c21015e8f93e7fbbee61
+size 935646
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/frame_001_comparison.png b/Meissonic/InfinityStar/vae_reconstruction_test/frame_001_comparison.png
new file mode 100644
index 0000000000000000000000000000000000000000..ed808da5610da2d6875ab686cd08beacea8f84e5
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/frame_001_comparison.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741e12e178a1b026e39c451ccddcbe0f55a95eb9e66287628e5d04fd77d75b0c
+size 968011
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/frame_002_comparison.png b/Meissonic/InfinityStar/vae_reconstruction_test/frame_002_comparison.png
new file mode 100644
index 0000000000000000000000000000000000000000..cb70027393b03421cf502cd1408de7eb5fe397bc
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/frame_002_comparison.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dd72eef725688bd6a113c118e4dff27c060d3f5dfc25cd5041033f32ee04b03
+size 979803
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/frame_003_comparison.png b/Meissonic/InfinityStar/vae_reconstruction_test/frame_003_comparison.png
new file mode 100644
index 0000000000000000000000000000000000000000..01788f8439bc410452a431db4626a7d3a7e6cf92
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/frame_003_comparison.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc239ef47a83a0e511ee22bd704309f2e26a93ac1c4711875ae582504888df2e
+size 984296
diff --git a/Meissonic/InfinityStar/vae_reconstruction_test/frame_004_comparison.png b/Meissonic/InfinityStar/vae_reconstruction_test/frame_004_comparison.png
new file mode 100644
index 0000000000000000000000000000000000000000..95e67f1d240b781ed079e0e585d543c91c836df1
--- /dev/null
+++ b/Meissonic/InfinityStar/vae_reconstruction_test/frame_004_comparison.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:808e9e50cf6ebc2b2c8c0ddf4381bd94038a29c6b93ff210d396c24cab97a843
+size 986298
diff --git a/Meissonic/LICENSE b/Meissonic/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64
--- /dev/null
+++ b/Meissonic/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/Meissonic/README.md b/Meissonic/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4f9e69eaa3761b7a62ae13df9d2fa61a9814e316
--- /dev/null
+++ b/Meissonic/README.md
@@ -0,0 +1,222 @@
+# Meissonic: Revitalizing Masked Generative Transformers for Efficient High-Resolution Text-to-Image Synthesis
+
+
+
+
+[](https://arxiv.org/abs/2410.08261)
+[](https://huggingface.co/MeissonFlow/Meissonic)
+[](https://github.com/viiika/Meissonic)
+[](https://www.youtube.com/watch?v=PlmifElhr6M)
+[](https://www.youtube.com/watch?v=rJDrf49wF64)
+[](https://huggingface.co/spaces/MeissonFlow/meissonic)
+[](https://replicate.com/chenxwh/meissonic)
+
+[](https://huggingface.co/Collov-Labs/Monetico)
+[](https://huggingface.co/spaces/Collov-Labs/Monetico)
+
+[](https://arxiv.org/abs/2411.10781)
+
+[](https://arxiv.org/abs/2503.15457)
+[](https://huggingface.co/Yuanzhi/DiMO)
+[](https://github.com/yuanzhi-zhu/DiMO)
+
+
+[](https://arxiv.org/abs/2505.23606)
+[](https://huggingface.co/MeissonFlow/Muddit)
+[](https://github.com/M-E-AGI-Lab/Muddit)
+[](https://huggingface.co/spaces/MeissonFlow/muddit)
+
+[](https://arxiv.org/abs/2507.04947)
+
+[](https://arxiv.org/abs/2508.10684)
+
+[](https://arxiv.org/abs/2509.19244)
+[](https://arxiv.org/abs/2509.23919)
+[](https://arxiv.org/abs/2509.25171)
+
+[](https://arxiv.org/abs/2510.06308)
+
+[](https://arxiv.org/abs/2510.20668) [](https://github.com/M-E-AGI-Lab/Awesome-World-Models)
+
+
+
+
"A pillow with a picture of a Husky on it."
+
+
+
"A white coffee mug, a solid black background"
+
+
+ 
+
+
+
+
+ Made with ❤️ by the MeissonFlow Research
+
diff --git a/Meissonic/VidTok/.gitignore b/Meissonic/VidTok/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..81849eac5c147c73d4202463658bd4b49571d163
--- /dev/null
+++ b/Meissonic/VidTok/.gitignore
@@ -0,0 +1,167 @@
+amlt
+.amltconfig
+checkpoints
+logs
+wandb
+tmp
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
\ No newline at end of file
diff --git a/Meissonic/VidTok/CODE_OF_CONDUCT.md b/Meissonic/VidTok/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000000000000000000000000000000000000..f9ba8cf65f3e3104dd061c178066ec8247811f33
--- /dev/null
+++ b/Meissonic/VidTok/CODE_OF_CONDUCT.md
@@ -0,0 +1,9 @@
+# Microsoft Open Source Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+
+Resources:
+
+- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
+- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
+- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
diff --git a/Meissonic/VidTok/LICENSE b/Meissonic/VidTok/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..9e841e7a26e4eb057b24511e7b92d42b257a80e5
--- /dev/null
+++ b/Meissonic/VidTok/LICENSE
@@ -0,0 +1,21 @@
+ MIT License
+
+ Copyright (c) Microsoft Corporation.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE
diff --git a/Meissonic/VidTok/README.md b/Meissonic/VidTok/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..edd9df5e2319e6dab35604d0d67192381dfacabf
--- /dev/null
+++ b/Meissonic/VidTok/README.md
@@ -0,0 +1,472 @@
+
+
+# VidTok
A Family of Versatile and State-Of-The-Art Video Tokenizers
+
+[](https://arxiv.org/pdf/2412.13061) [](https://github.com/microsoft/VidTok) [](https://huggingface.co/microsoft/VidTok)
+
+
+---
+
+
+
+ We introduce VidTok, a cutting-edge family of video tokenizers that excels in both continuous and discrete tokenizations. VidTok incorporates several key advancements over existing approaches:
+ * ⚡️ **Efficient Architecture**. Separate spatial and temporal sampling reduces computational complexity without sacrificing quality.
+ * 🔥 **Advanced Quantization**. Finite Scalar Quantization (FSQ) addresses training instability and codebook collapse in discrete tokenization.
+ * 💥 **Enhanced Training**. A two-stage strategy—pre-training on low-res videos and fine-tuning on high-res—boosts efficiency. Reduced frame rates improve motion dynamics representation.
+
+VidTok, trained on a large-scale video dataset, outperforms previous models across all metrics, including PSNR, SSIM, LPIPS, and FVD.
+
+https://github.com/user-attachments/assets/a3341037-130d-4a83-aba6-c3daeaf66932
+
+## 🔥 News
+- August, 2025: 🚀 Introduced spatial tiling for large resolutions (>256), reducing GPU memory usage to ~6 GB when encoding and decoding a 17 × 768 × 768 video.
+* March, 2025: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) has been accepted by CVPR 2025, and the [checkpoint](https://huggingface.co/microsoft/vidtwin) was released!
+* March, 2025: 🚀 [VidTok v1.1](#-updates-in-vidtok-v11) was released! We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. [Relevant checkpoints](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints/vidtok_v1_1) are continuously updating.
+* December, 2024: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) was released!
+* December, 2024: 🚀 [VidTok](https://github.com/microsoft/vidtok) was released!
+
+
+## 💥 Updates in VidTok v1.1
+> VidTok v1.1 is an update for causal models. We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. See performance [here](#v11-performance).
+
+### v1.1: Long Video Reconstruction
+Run the following inference script to [reconstruct an input video](#reconstruct-an-input-video):
+```bash
+python scripts/inference_reconstruct.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --output_video_dir OUTPUT_DIR --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+and run the following inference script to [evaluate the reconstruction performance](#performance-evaluation):
+```bash
+python scripts/inference_evaluate.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+
+For an easy usage of VidTok v1.1 models, refer to [this script](#easy-usage) and make the following revision:
+```python
+# Use VidTok v1.1 models
+cfg_path = "configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml"
+ckpt_path = "checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt"
+
+...
+
+model.to('cuda').eval()
+# Using tiling inference to save memory usage
+model.use_tiling = True
+model.t_chunk_enc = 16
+model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+model.use_overlap = True
+# random input: long video
+x_input = (torch.rand(1, 3, 129, 256, 256) * 2 - 1).to('cuda')
+
+...
+
+if x_recon.shape[2] != x_input.shape[2]:
+ x_recon = x_recon[:, :, -x_input.shape[2]:, ...]
+```
+
+### v1.1: Long Video Fine-tuning
+Follow this [training guidance](#fine-tune-on-custom-data) to fine-tune on your custom long video data and note that:
+- Compared to VidTok v1.0, we tend to use longer sequences to fine-tune the model (for example, setting `NUM_FRAMES_1` to 33, 49, or larger).
+- The resolution and the sequence length of training data should be adjusted according to GPU memory.
+
+### v1.1: Performance
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.13 | 0.941 | 0.049 | 87.4 |
+| [vidtok_kl_causal_41616_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x16x16 | 29.61 | 0.854 | 0.113 | 162.7 |
+| [vidtok_kl_causal_288_8chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt) | KL-8chn | ✔️ | 2x8x8 | 34.59 | 0.935 | 0.051 | 78.2 |
+| [vidtok_fsq_causal_488_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.39 | 0.856 | 0.114 | 168.5 |
+| [vidtok_fsq_causal_888_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 8x8x8 | 27.95 | 0.817 | 0.142 | 293.2 |
+- This is the evaluation result of long video reconstruction conducted on each complete video in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30 and a resolution of `256x256`.
+
+
+## 🔧 Setup
+1. Clone this repository and navigate to VidTok folder:
+```bash
+git clone https://github.com/microsoft/VidTok
+cd VidTok
+```
+2. We provide an `environment.yaml` file for setting up a Conda environment. Conda's installation instructions are available [here](https://docs.anaconda.com/miniconda/index.html).
+```bash
+# 1. Prepare conda environment
+conda env create -f environment.yaml
+# 2. Activate the environment
+conda activate vidtok
+```
+
+We recommend using 1+ high-end GPU for training and inference. We have done all testing and development using A100 and MI300X GPUs. For convenience, we also provide prebuilt [Docker](https://hub.docker.com/) images with required dependencies. You can use it as follows:
+
+```bash
+# NVIDIA GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-cuda12.1-python3.10-pytorch2.5:orig-vidtok bash
+# AMD GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-rocm6.2.4-python3.10-pytorch2.5:orig-vidtok bash
+```
+
+## 🎈 Checkpoints
+Download pre-trained models [here](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints), and put them in `checkpoints` folder, like:
+```
+└── checkpoints
+ ├── vidtok_v1_1
+ │ ├── vidtok_kl_causal_488_16chn_v1_1.ckpt
+ │ └── ...
+ ├── vidtok_fsq_causal_41616_262144.ckpt
+ ├── vidtok_fsq_causal_488_262144.ckpt
+ ├── vidtok_fsq_causal_488_32768.ckpt
+ ├── vidtok_fsq_causal_488_4096.ckpt
+ ├── vidtok_fsq_noncausal_41616_262144.ckpt
+ ├── vidtok_fsq_noncausal_488_262144.ckpt
+ ├── vidtok_kl_causal_288_8chn.ckpt
+ ├── vidtok_kl_causal_41616_4chn.ckpt
+ ├── vidtok_kl_causal_444_4chn.ckpt
+ ├── vidtok_kl_causal_488_16chn.ckpt
+ ├── vidtok_kl_causal_488_4chn.ckpt
+ ├── vidtok_kl_causal_488_8chn.ckpt
+ ├── vidtok_kl_noncausal_41616_16chn.ckpt
+ ├── vidtok_kl_noncausal_41616_4chn.ckpt
+ ├── vidtok_kl_noncausal_488_16chn.ckpt
+ └── vidtok_kl_noncausal_488_4chn.ckpt
+```
+Each checkpoint has a corresponding config file with the same name in `configs` folder.
+
+
+## 🔆 Performance
+
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_4chn.ckpt) | KL-4chn | ✔️ | 4x8x8 | 29.64 | 0.852| 0.114| 194.2|
+| [vidtok_kl_causal_488_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_8chn.ckpt) | KL-8chn | ✔️ |4x8x8 | 31.83 | 0.897| 0.083| 109.3|
+| [vidtok_kl_causal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_16chn.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.04 |0.942 |0.047 | 78.9|
+| [vidtok_kl_causal_288_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_288_8chn.ckpt) | KL-8chn | ✔️ | 2x8x8 | 33.86 | 0.928 |0.057 | 80.7 |
+| [vidtok_kl_causal_444_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_444_4chn.ckpt) | KL-4chn | ✔️ | 4x4x4 | 34.78 | 0.941 | 0.051| 87.2|
+| [vidtok_kl_causal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_41616_4chn.ckpt) | KL-4chn | ✔️ | 4x16x16 | 25.05 | 0.711| 0.228| 549.1|
+| [vidtok_kl_noncausal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_4chn.ckpt) | KL-4chn | ✖️ | 4x8x8 | 30.60 | 0.876 | 0.098| 157.9|
+| [vidtok_kl_noncausal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_16chn.ckpt) | KL-16chn | ✖️ | 4x8x8 | 36.13 | 0.950 | 0.044| 60.5|
+| [vidtok_kl_noncausal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt) | KL-4chn | ✖️ | 4x16x16 | 26.06 | 0.751 | 0.190|423.2 |
+| [vidtok_kl_noncausal_41616_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt) | KL-16chn | ✖️ | 4x16x16 | 30.69 | 0.878 | 0.095| 147.1|
+| [vidtok_fsq_causal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_262144.ckpt) | FSQ-262,144 | ✔️ | 4x8x8 | 29.82 | 0.867 |0.106 | 160.1|
+| [vidtok_fsq_causal_488_32768](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_32768.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.16 | 0.854 | 0.117| 196.9|
+| [vidtok_fsq_causal_488_4096](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_4096.ckpt) | FSQ-4096 | ✔️ | 4x8x8 | 28.36 | 0.832 | 0.133| 218.1|
+| [vidtok_fsq_causal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_41616_262144.ckpt) | FSQ-262,144 | ✔️ | 4x16x16 | 25.38 | 0.738 |0.206 | 430.1|
+| [vidtok_fsq_noncausal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_488_262144.ckpt) | FSQ-262,144 | ✖️ | 4x8x8 | 30.78 | 0.889| 0.091| 132.1|
+| [vidtok_fsq_noncausal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt) | FSQ-262,144 | ✖️ | 4x16x16 | 26.37 | 0.772| 0.171| 357.0|
+
+- `VCR` indicates the video compression ratio `TxHxW`.
+- The above table shows model performance evaluated on 30 test videos in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30. The input size is `17x256x256` for causal models and `16x256x256` for non-causal models.
+
+## 🔛 Training
+
+### Data Preparation
+1. Put all training videos under `DATA_DIR`:
+```
+└── DATA_DIR
+ ├── subset1
+ │ ├── videoname11.mp4
+ │ └── videoname12.mp4
+ ├── subset2
+ │ ├── videoname21.mp4
+ │ ├── videoname22.mp4
+ │ └── subsubset1
+ │ ├── videoname211.mp4
+ │ └── videoname212.mp4
+ └── ...
+```
+2. Prepare a `.csv` meta file to record the relative paths of these videos with respect to `DATA_DIR`, like:
+```
+videos
+subset1/videoname11.mp4
+subset2/videoname21.mp4
+subset2/subsubset1/videoname211.mp4
+```
+
+> Validation data is also prepared following the above steps.
+
+### Fine-tune on Custom Data
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and modify the following parameters:
+ - Specify the `ckpt_path` parameter to initialize the model with pre-trained checkpoint parameters:
+ ```yaml
+ model:
+ params:
+ ckpt_path: PATH_TO_CHECKPOINT # train from existing checkpoint
+ ```
+ - Specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+ - Set `fix_encoder` and `fix_decoder` to be `False` to enable full model fine-tuning:
+ ```yaml
+ model:
+ params:
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+ ```
+ - Other hyperparameters according to your needs.
+
+3. Run the following command to start training:
+```bash
+python main.py -b CONFIG --logdir LOGDIR
+
+# You can also use `torchrun` to start the training code.
+```
+Training logs and checkpoints are saved in `LOGDIR`.
+
+It is recommended to use [Weights & Biases](https://wandb.ai/site) as the data visualization tool ([TensorBoard](https://www.tensorflow.org/tensorboard) by default). Use `wandb login` to log in first, and then run:
+```bash
+python main.py -b CONFIG --logdir LOGDIR --wandb --wandb_entity ENTITY --wandb_project PROJECT
+```
+
+
+### Train from Scratch
+
+Two-stage Training
+We adopt a two-stage training strategy to improve training efficiency: initially pre-training the full model on low-resolution videos, followed by fine-tuning only the decoder on high-resolution videos.
+
+| First Stage | Second Stage | Fix encoder | PSNR | SSIM | LPIPS | GPU Hours|
+|------|------|------|------|------|------|------|
+| 256 x 256 | - | - | 29.19 | 0.843 | 0.127| 3,072|
+| 128 x 128 | 256 x 256 | ✔️ | 29.21 | 0.843 | 0.125| 1,536|
+
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1 # vary in different training stages
+ input_width: INPUT_WIDTH_1 # vary in different training stages
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+
+3. Start the first stage of training. First, revise the `CONFIG` file to enable full model training with low-resolution data:
+```yaml
+model:
+ params:
+ # ckpt_path: # disable this parameter so as to train from scratch
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 128
+ input_width: 128
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 50,000 steps with batch size 16 in this stage.
+
+4. Start the second stage of training. First, revise the `CONFIG` file to enable the fine-tuning of the decoder with high-resolution data:
+```yaml
+model:
+ params:
+ ckpt_path: CKPT_PATH # path to the saved checkpoint after the first stage of training
+ encoder_config:
+ params:
+ fix_encoder: true
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 256
+ input_width: 256
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 30,000 steps with batch size 8 in this stage.
+
+
+
+## 🚀 Inference
+
+### Easy Usage
+We provide the following example for a quick usage of our models. It works for both continuous and discrete tokenization and both causal and non-causal models.
+Just provide the path to the configuration file `cfg_path` and checkpoint file `ckpt_path`.
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+# model forward
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+assert x_input.shape == x_recon.shape
+```
+If you want to directly infer from latent tokens, run the following code:
+```python
+z, reg_log = model.encode(x_input, return_reg_log=True)
+# infer from continuous latent space
+x_recon = model.decode(z)
+# infer from discrete latent tokens
+x_recon = model.decode(reg_log['indices'], decode_from_indices=True)
+```
+
+### Use Torch Compile to Speed Up Inference
+Use compiled components in VidTok can speed up inference by as much as 2X. The following code snippet demonstrates how to compile our models.
+
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+torch._inductor.config.cpp.weight_prepack=True
+torch._inductor.config.freezing=True
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+model.encoder = torch.compile(model.encoder)
+model.decoder = torch.compile(model.decoder)
+
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+
+# Warm Up
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+
+torch.cuda.synchronize()
+import time
+start = time.time()
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ for i in range(10):
+ _, x_recon, _ = model(x_input)
+torch.cuda.synchronize()
+print(f"Average inference time: {(time.time() - start)/10 :.4f} seconds")
+```
+
+### Reconstruct an Input Video
+```bash
+python scripts/inference_reconstruct.py --config CONFIG --ckpt CKPT --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --output_video_dir OUTPUT_DIR
+```
+- Specify `VIDEO_PATH` to the path of your test video. We provide an example video in `assets/example.mp4`.
+- The reconstructed video is saved in `OUTPUT_DIR`.
+- For causal models, you can choose to add `--pad_gen_frames` to the command line, which may improve the smoothness of the reconstructed video.
+
+### Performance Evaluation
+We also provide a manuscript `scripts/inference_evaluate.py` to evaluate the video reconstruction performance in PSNR, SSIM and LPIPS.
+
+1. Put all of your test videos under `DATA_DIR`.
+2. Run the following command, and all `.mp4` videos under `DATA_DIR` will be tested:
+```bash
+python scripts/inference_evaluate.py --config CONFIG --ckpt CKPT --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30
+```
+(Optional) If you only want to test certain videos under `DATA_DIR`, you need to prepare a `.csv` meta file
+to indicate the video files to be tested (refer to [Data Preparation](#data-preparation)). And add `--meta_path META_PATH` to the above command to specify the path to the `.csv` meta file.
+
+## 💡 Intended Uses
+
+We are sharing our model with the research community to foster further research in this area:
+* Training your own video tokenizers for research purpose.
+* Video tokenization with various compression rates.
+
+
+## 🪧 Out-of-scope Uses
+
+Our models are not specifically designed or evaluated for all downstream purposes. Developers should consider common limitations of video tokenizers (e.g., performance degradation on out-of-domain data) as they select use cases, and evaluate and mitigate for privacy, safety, and fairness before using within a specific downstream use case, particularly for high-risk scenarios.
+
+Developers should be aware of and adhere to applicable laws or regulations (including privacy, trade compliance laws, etc.) that are relevant to their use case.
+
+
+## 🤖️ Risks and Limitations
+
+Some of the limitations of this model to be aware of include:
+* VidTok may lose detailed information on the reconstructed content.
+* VidTok inherits any biases, errors, or omissions characteristic of its training data.
+* VidTok was developed for research and experimental purposes. Further testing and validation are needed before considering its application in commercial or real-world scenarios.
+
+
+## 🤗 Acknowledgments
+
+This codebase borrows code from [generative-models](https://github.com/Stability-AI/generative-models). We thank Stability AI for its efforts and innovations, which have made the development process more efficient and convenient.
+
+Thank you to everyone who contributed their wisdom and efforts to this project.
+
+## ✏️ BibTeX
+
+```bibtex
+@article{tang2024vidtok,
+ title={VidTok: A Versatile and Open-Source Video Tokenizer},
+ author={Tang, Anni and He, Tianyu and Guo, Junliang and Cheng, Xinle and Song, Li and Bian, Jiang},
+ year={2024},
+ journal={arXiv preprint arXiv:2412.13061},
+}
+```
+
+## ☎️ Contact
+
+We welcome feedback and collaboration from our audience. If you have suggestions, questions, or observe unexpected/offensive behavior in our technology, please contact us at tianyuhe@microsoft.com.
+
+## 📄 Contributing
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+
+## 📍 Trademarks
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
+trademarks or logos is subject to and must follow
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.
diff --git a/Meissonic/VidTok/SECURITY.md b/Meissonic/VidTok/SECURITY.md
new file mode 100644
index 0000000000000000000000000000000000000000..b3c89efc852e22f71eabf5dfbc6ac62493425eb6
--- /dev/null
+++ b/Meissonic/VidTok/SECURITY.md
@@ -0,0 +1,41 @@
+
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+ * Full paths of source file(s) related to the manifestation of the issue
+ * The location of the affected source code (tag/branch/commit or direct URL)
+ * Any special configuration required to reproduce the issue
+ * Step-by-step instructions to reproduce the issue
+ * Proof-of-concept or exploit code (if possible)
+ * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd).
+
+
diff --git a/Meissonic/VidTok/SUPPORT.md b/Meissonic/VidTok/SUPPORT.md
new file mode 100644
index 0000000000000000000000000000000000000000..291d4d43733f4c15a81ff598ec1c99fd6c18f64c
--- /dev/null
+++ b/Meissonic/VidTok/SUPPORT.md
@@ -0,0 +1,25 @@
+# TODO: The maintainer of this repo has not yet edited this file
+
+**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
+
+- **No CSS support:** Fill out this template with information about how to file issues and get help.
+- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
+- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
+
+*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
+
+# Support
+
+## How to file issues and get help
+
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates. For new issues, file your bug or
+feature request as a new Issue.
+
+For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
+FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
+CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
+
+## Microsoft Support Policy
+
+Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
diff --git a/Meissonic/VidTok/assets/example.mp4 b/Meissonic/VidTok/assets/example.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..21be26c8a8f5dbb76de5a225e5b284fdbb024904
--- /dev/null
+++ b/Meissonic/VidTok/assets/example.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:588ca89fae7320a079d4f77cf963f88075959f06310594ab35d3e04b844c4d50
+size 540937
diff --git a/Meissonic/VidTok/assets/gemini.png b/Meissonic/VidTok/assets/gemini.png
new file mode 100644
index 0000000000000000000000000000000000000000..fe326dc4b4b4b6db82575b15e9b4bf2f3b63e63d
Binary files /dev/null and b/Meissonic/VidTok/assets/gemini.png differ
diff --git a/Meissonic/VidTok/assets/radar.png b/Meissonic/VidTok/assets/radar.png
new file mode 100644
index 0000000000000000000000000000000000000000..6eff9338928faf73c0e39e859237bc6c588ec183
--- /dev/null
+++ b/Meissonic/VidTok/assets/radar.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cfdef783e26264ff671f81845b54471b237f5cd7df0dbd63642fde0c20f935e
+size 424581
diff --git a/Meissonic/VidTok/assets/vidtwin.png b/Meissonic/VidTok/assets/vidtwin.png
new file mode 100644
index 0000000000000000000000000000000000000000..a7ae41105384d077a7d128d1d687e3ee8e17fbd8
--- /dev/null
+++ b/Meissonic/VidTok/assets/vidtwin.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e58210f7ca7784a4df737f8a2ece2b5e9f85fff3bf423b24aa6aeeb0b196cef
+size 549933
diff --git a/Meissonic/VidTok/assets/vidtwin_demo.png b/Meissonic/VidTok/assets/vidtwin_demo.png
new file mode 100644
index 0000000000000000000000000000000000000000..ca79126104a7cbfb67c0acb70f03c6e66546675c
--- /dev/null
+++ b/Meissonic/VidTok/assets/vidtwin_demo.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af8e15c46050cc7957bc5d334bd1902d58b9be9aa7669df82f2ae9ba08c90585
+size 6371846
diff --git a/Meissonic/VidTok/checkpoints/lpips/vgg.pth b/Meissonic/VidTok/checkpoints/lpips/vgg.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f57dcf5cc764d61c8a460365847fb2137ff0a62d
--- /dev/null
+++ b/Meissonic/VidTok/checkpoints/lpips/vgg.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868
+size 7289
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml b/Meissonic/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..80b2edbebf97fd747933000d32ccbdfe0e135702
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_causal_488_262144.yaml b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f9c3e94ed8b64a681b2ed3887929109b6ccc53
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_causal_488_32768.yaml b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e661c533b367c7be4eea8de629e074b395e1684
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_32768.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_causal_488_4096.yaml b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9bf654bb21c1bd67ddde3f4878497c6ad780503a
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_4096.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8] # codebook size: 8*8*8*8=4096
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml b/Meissonic/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e400ff097155465e2a477bfbc3ca32f346a0ca12
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml b/Meissonic/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6c8731af435aae0efbde9716ee696ce3efd30d25
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_288_8chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4c20bf1aaf2aa97f67d19d0de1dbac47cf2d55ac
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_288_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1db41cd612f5e6b1390bebda33e559e347b60907
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_444_4chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bcbbd11cba27e60bd2f44866655adcfff91f91a6
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_444_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ spatial_ds: [1, 2]
+ spatial_us: [1, 2]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_488_16chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a7527679fe0766616df45223fe1e9595101fadc5
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_488_4chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d14b0d75435b24affad90095e4d6e42c6525a8d
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_causal_488_8chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..336cbac8319dc3f9c232c02889e598fdf098a777
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b32064f1589eb3b21cb978f808a95fd6ade31ff9
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bdd33a5ebc635cbf9a04fb94006d14a473d87a3c
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..47a1090f42b07655626f0d522d756157fdeb1d6a
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml b/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05bb8c54df1374c0f67c79d256195dd6451acfbd
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..74364fa3209abbda65b0c23311d92ad0975570dd
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..334749091ff2c561208f169029eae4704d4213e3
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2966072ff2a073d404d0cb438674acfc92319033
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
@@ -0,0 +1,122 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [0, 1, 2]
+ tempo_us: [1, 2, 3]
+ time_downsample_factor: 8
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..deefcc29ecd8212e8bd3edd9d9870d8c64079db7
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
@@ -0,0 +1,116 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..965b243859e513244d3d3fd9cc68aa27aee887da
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f68342b48dd5fcae8e5bfcd2fcb5cc5bd1ab1c
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b4badafed742c03fd7850f197cdb8207c59b992e
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml b/Meissonic/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8bcd59650c229ac57560602854015262969d658b
--- /dev/null
+++ b/Meissonic/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
@@ -0,0 +1,154 @@
+model:
+ base_learning_rate: 1.6e-4
+ target: vidtwin.models.vidtwin_ae.VidAutoEncoderQformerCompactSymVidVAE
+ params:
+ input_key: jpg
+ monitor: val/rec_loss
+ ckpt_path: PATH_TO_CHECKPOINT
+ ignore_keys: []
+ expect_ch: 8
+ cont_num_blocks: 1
+ downsample_motion: True
+ motion_num_blocks: 1
+ d_dim: 8
+
+ temporal_qformer_config:
+ target: vidtwin.modules.qformer.MyQformerInterface
+ params:
+ num_query_tokens: 16
+ query_hidden_size: 64
+ encoder_hidden_size: 768
+
+ encoder_config:
+ target: vidtwin.modules.st_transformer.STTEncoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ decoder_config:
+ target: vidtwin.modules.st_transformer.STTDecoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ perceptual_weight: 0.05
+ disc_start: 20001
+ disc_weight: 0.05
+ learn_logvar: True
+ dims: 3
+ disc_type: 2d
+ regularization_weights:
+ kl_loss: 0.001
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+ params:
+ sample: True
+
+
+ lr_scheduler_config_d:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 1.5e-05
+ lr_start: 1.0e-05
+ warmup_steps: 5000
+ lr_scheduler_config_g:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 3.0e-05
+ lr_start: 0
+ warmup_steps: 5000
+ optimizer_config:
+ target: torch.optim.AdamW
+ params:
+ betas:
+ - 0
+ - 0.9
+ weight_decay: 0.0001
+ lr_scheduler_config:
+ target: inverse_sqrt
+ params:
+ num_warmup_steps: 2000
+ frequency: 1
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: True
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 2
+
+
+
+ trainer:
+ # precision: bf16-mixed # 16-mixed
+ benchmark: True
+ devices: 4
+ num_sanity_val_steps: 10
+ val_check_interval: 5000
+ accumulate_grad_batches: 1
+ max_epochs: 10
diff --git a/Meissonic/VidTok/environment.yaml b/Meissonic/VidTok/environment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6abf1dcc7cefdab27b272681c0395791b3d432f
--- /dev/null
+++ b/Meissonic/VidTok/environment.yaml
@@ -0,0 +1,114 @@
+name: vidtok
+channels:
+ - defaults
+dependencies:
+ - _libgcc_mutex=0.1=main
+ - _openmp_mutex=5.1=1_gnu
+ - bzip2=1.0.8=h5eee18b_6
+ - ca-certificates=2024.11.26=h06a4308_0
+ - ld_impl_linux-64=2.40=h12ee557_0
+ - libffi=3.4.4=h6a678d5_1
+ - libgcc-ng=11.2.0=h1234567_1
+ - libgomp=11.2.0=h1234567_1
+ - libstdcxx-ng=11.2.0=h1234567_1
+ - libuuid=1.41.5=h5eee18b_0
+ - ncurses=6.4=h6a678d5_0
+ - openssl=3.0.15=h5eee18b_0
+ - pip=24.2=py310h06a4308_0
+ - python=3.10.15=he870216_1
+ - readline=8.2=h5eee18b_0
+ - setuptools=75.1.0=py310h06a4308_0
+ - sqlite=3.45.3=h5eee18b_0
+ - tk=8.6.14=h39e8969_0
+ - wheel=0.44.0=py310h06a4308_0
+ - xz=5.4.6=h5eee18b_1
+ - zlib=1.2.13=h5eee18b_1
+ - pip:
+ - absl-py==2.1.0
+ - aiohappyeyeballs==2.4.4
+ - aiohttp==3.11.9
+ - aiosignal==1.3.1
+ - antlr4-python3-runtime==4.9.3
+ - appdirs==1.4.4
+ - async-timeout==5.0.1
+ - attrs==24.2.0
+ - av==12.0.0
+ - beartype==0.18.2
+ - certifi==2024.8.30
+ - charset-normalizer==3.4.0
+ - click==8.1.7
+ - contourpy==1.3.1
+ - cycler==0.12.1
+ - decord==0.6.0
+ - docker-pycreds==0.4.0
+ - einops==0.8.0
+ - filelock==3.16.1
+ - fonttools==4.55.1
+ - frozenlist==1.5.0
+ - fsspec==2024.10.0
+ - gitdb==4.0.11
+ - gitpython==3.1.43
+ - grpcio==1.68.1
+ - idna==3.10
+ - imageio==2.34.0
+ - jinja2==3.1.4
+ - kiwisolver==1.4.7
+ - lightning==2.2.4
+ - lightning-utilities==0.11.9
+ - markdown==3.7
+ - markdown-it-py==3.0.0
+ - markupsafe==3.0.2
+ - matplotlib==3.8.4
+ - mdurl==0.1.2
+ - mpmath==1.3.0
+ - multidict==6.1.0
+ - natsort==8.4.0
+ - networkx==3.4.2
+ - numpy==1.26.4
+ - nvidia-cublas-cu12==12.1.3.1
+ - nvidia-cuda-cupti-cu12==12.1.105
+ - nvidia-cuda-nvrtc-cu12==12.1.105
+ - nvidia-cuda-runtime-cu12==12.1.105
+ - nvidia-cudnn-cu12==8.9.2.26
+ - nvidia-cufft-cu12==11.0.2.54
+ - nvidia-curand-cu12==10.3.2.106
+ - nvidia-cusolver-cu12==11.4.5.107
+ - nvidia-cusparse-cu12==12.1.0.106
+ - nvidia-nccl-cu12==2.19.3
+ - nvidia-nvjitlink-cu12==12.6.85
+ - nvidia-nvtx-cu12==12.1.105
+ - omegaconf==2.3.0
+ - opencv-python==4.6.0.66
+ - packaging==24.2
+ - pandas==2.1.4
+ - pillow==11.0.0
+ - propcache==0.2.1
+ - protobuf==4.25.5
+ - psutil==6.1.0
+ - pygments==2.18.0
+ - pyparsing==3.2.0
+ - python-dateutil==2.9.0.post0
+ - pytorch-lightning==2.2.4
+ - pytz==2024.2
+ - pyyaml==6.0.2
+ - requests==2.32.3
+ - rich==13.5.3
+ - safetensors==0.4.2
+ - sentry-sdk==2.19.0
+ - setproctitle==1.3.4
+ - six==1.17.0
+ - smmap==5.0.1
+ - sympy==1.13.3
+ - tensorboard==2.16.2
+ - tensorboard-data-server==0.7.2
+ - torch==2.2.2
+ - torchmetrics==1.6.0
+ - torchvision==0.17.2
+ - tqdm==4.67.1
+ - triton==2.2.0
+ - typing-extensions==4.12.2
+ - tzdata==2024.2
+ - urllib3==2.2.3
+ - wandb==0.16.6
+ - werkzeug==3.1.3
+ - yarl==1.18.3
diff --git a/Meissonic/VidTok/main.py b/Meissonic/VidTok/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..e03f0de8a1c683fbdde7177083c87a0a3df85f83
--- /dev/null
+++ b/Meissonic/VidTok/main.py
@@ -0,0 +1,1124 @@
+import argparse
+import datetime
+import pytz
+import glob
+import inspect
+import os
+import re
+import sys
+import numpy as np
+import warnings
+warnings.filterwarnings("ignore")
+from rich import print
+from inspect import Parameter
+from typing import Union
+from matplotlib import pyplot as plt
+from natsort import natsorted
+from omegaconf import OmegaConf
+from packaging import version
+from PIL import Image
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch import seed_everything
+from lightning.pytorch.trainer import Trainer
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from vidtok.modules.util import (exists, instantiate_from_config, isheatmap,
+ print0, seed_anything)
+
+MULTINODE_HACKS = True
+
+
+def default_trainer_args():
+ argspec = dict(inspect.signature(Trainer.__init__).parameters)
+ argspec.pop("self")
+ default_args = {
+ param: argspec[param].default
+ for param in argspec
+ if argspec[param] != Parameter.empty
+ }
+ return default_args
+
+
+def get_step_value(folder_name):
+ match = re.search(r"step=(\d+)", folder_name)
+ if match:
+ return int(match.group(1))
+ return 0
+
+
+def get_parser(**parser_kwargs):
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser(**parser_kwargs)
+ parser.add_argument(
+ "-n",
+ "--name",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="postfix for logdir",
+ )
+ parser.add_argument(
+ "--no_date",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="if True, skip date generation for logdir and only use naming via opt.base or opt.name (+ opt.postfix, optionally)",
+ )
+ parser.add_argument(
+ "-r",
+ "--resume",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="resume from logdir or checkpoint in logdir",
+ )
+ parser.add_argument(
+ "-b",
+ "--base",
+ nargs="*",
+ metavar="base_config.yaml",
+ help="paths to base configs. Loaded from left-to-right. "
+ "Parameters can be overwritten or added with command-line options of the form `--key value`.",
+ default=list(),
+ )
+ parser.add_argument(
+ "-t",
+ "--train",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="train",
+ )
+ parser.add_argument(
+ "--no-test",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="disable test",
+ )
+ parser.add_argument(
+ "-p", "--project", help="name of new or path to existing project"
+ )
+ parser.add_argument(
+ "-d",
+ "--debug",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="enable post-mortem debugging",
+ )
+ parser.add_argument(
+ "-s",
+ "--seed",
+ type=int,
+ default=23,
+ help="seed for seed_everything",
+ )
+ parser.add_argument(
+ "--seed_rank",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="reset seed every rank on fit start",
+ )
+ parser.add_argument(
+ "-f",
+ "--postfix",
+ type=str,
+ default="",
+ help="post-postfix for default name",
+ )
+ parser.add_argument(
+ "-l",
+ "--logdir",
+ type=str,
+ default="logs",
+ help="directory for logging dat shit",
+ )
+ parser.add_argument(
+ "--scale_lr",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="scale base-lr by ngpu * batch_size * n_accumulate",
+ )
+ parser.add_argument(
+ "--legacy_naming",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="name run based on config file name if true, else by whole path",
+ )
+ parser.add_argument(
+ "--enable_tf32",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="enables the TensorFloat32 format both for matmuls and cuDNN for pytorch 1.12",
+ )
+ parser.add_argument(
+ "--startup",
+ type=str,
+ default=None,
+ help="Startuptime from distributed script",
+ )
+ parser.add_argument(
+ "--wandb",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="log to wandb",
+ )
+ parser.add_argument(
+ "--wandb_entity",
+ type=str,
+ default="",
+ help="Wandb entity name string",
+ )
+ parser.add_argument(
+ "--wandb_key",
+ type=str,
+ default="",
+ help="Wandb key",
+ )
+ parser.add_argument(
+ "--wandb_project",
+ type=str,
+ default="vidtok",
+ )
+ parser.add_argument(
+ "--wandb_id",
+ type=str,
+ default=None,
+ help="automatically resume from the same wandb id"
+ "must be used in combination with --wandb_auto_resume False",
+ )
+ parser.add_argument(
+ "--wandb_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest run id in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--checkpoint_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest checkpoint in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--no_base_name",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False, # TODO: later default to True
+ help="log to wandb",
+ )
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ parser.add_argument(
+ "--resume_from_checkpoint",
+ type=str,
+ default=None,
+ help="single checkpoint file to resume from",
+ )
+ default_args = default_trainer_args()
+ for key in default_args:
+ # parameters in the pl.Trainer are passed as --key value
+ parser.add_argument("--" + key, default=default_args[key])
+ return parser
+
+
+def get_checkpoint_name(logdir):
+ ckpt = os.path.join(logdir, "checkpoints", "last**.ckpt")
+ ckpt = natsorted(glob.glob(ckpt))
+ print0('available "last" checkpoints:')
+ print0(ckpt)
+ if len(ckpt) > 1:
+ print0("got most recent checkpoint")
+ ckpt = sorted(ckpt, key=lambda x: os.path.getmtime(x))[-1]
+ print0(f"Most recent ckpt is {ckpt}")
+ with open(os.path.join(logdir, "most_recent_ckpt.txt"), "w") as f:
+ f.write(ckpt + "\n")
+ try:
+ version = int(ckpt.split("/")[-1].split("-v")[-1].split(".")[0])
+ except Exception as e:
+ print0("version confusion but not bad")
+ print0(e)
+ version = 1
+ # version = last_version + 1
+ else:
+ # in this case, we only have one "last.ckpt"
+ ckpt = ckpt[0]
+ version = 1
+ melk_ckpt_name = f"last-v{version}.ckpt"
+ print0(f"Current melk ckpt name: {melk_ckpt_name}")
+ return ckpt, melk_ckpt_name
+
+
+class SetupCallback(Callback):
+ def __init__(
+ self,
+ resume,
+ now,
+ logdir,
+ ckptdir,
+ cfgdir,
+ config,
+ lightning_config,
+ debug,
+ save_ckpt_on_exception=False,
+ ckpt_name=None,
+ seed=None,
+ seed_rank=False,
+ ):
+ super().__init__()
+ self.resume = resume
+ self.now = now
+ self.logdir = logdir
+ self.ckptdir = ckptdir
+ self.cfgdir = cfgdir
+ self.config = config
+ self.lightning_config = lightning_config
+ self.debug = debug
+ self.save_ckpt_on_exception = save_ckpt_on_exception
+ self.ckpt_name = ckpt_name
+ self.seed = seed
+ self.seed_rank = seed_rank
+
+ def on_exception(self, trainer: pl.Trainer, pl_module, exception):
+ if self.save_ckpt_on_exception and (not self.debug) and (trainer.global_rank == 0):
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Saving checkpoint to {self.ckptdir}")
+ if self.ckpt_name is None:
+ ckpt_path = os.path.join(self.ckptdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(self.ckptdir, self.ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def on_fit_start(self, trainer, pl_module):
+ if self.seed_rank:
+ # current_seed = torch.initial_seed()
+ seed_anything(self.seed + trainer.global_rank)
+ print(f"[bold red]\[main][SetupCallback][/bold red] Rank {trainer.global_rank}: Reset GLOBAL seed to {self.seed + trainer.global_rank}")
+ elif hasattr(pl_module, "set_seed") and callable(pl_module.set_seed):
+ pl_module.set_seed(self.seed)
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Set pl_module seed to {self.seed} with pl_module.set_seed")
+ if trainer.global_rank == 0:
+ # Create logdirs and save configs
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Creating logdir: {self.logdir}, ckptdir: {self.ckptdir}, cfgdir: {self.cfgdir}")
+ os.makedirs(self.logdir, exist_ok=True)
+ os.makedirs(self.ckptdir, exist_ok=True)
+ os.makedirs(self.cfgdir, exist_ok=True)
+
+ if "callbacks" in self.lightning_config:
+ if (
+ "metrics_over_trainsteps_checkpoint"
+ in self.lightning_config["callbacks"]
+ ):
+ os.makedirs(
+ os.path.join(self.ckptdir, "trainstep_checkpoints"),
+ exist_ok=True,
+ )
+ print0("[bold red]\[main][SetupCallback][/bold red] Project config")
+ print0(OmegaConf.to_yaml(self.config))
+ if MULTINODE_HACKS and not self.debug:
+ import time
+ time.sleep(5)
+ OmegaConf.save(
+ self.config,
+ os.path.join(self.cfgdir, "{}-project.yaml".format(self.now)),
+ )
+
+ print0("[bold red]\[main][SetupCallback][/bold red] Lightning config")
+ print0(OmegaConf.to_yaml(self.lightning_config))
+ OmegaConf.save(
+ OmegaConf.create({"lightning": self.lightning_config}),
+ os.path.join(self.cfgdir, "{}-lightning.yaml".format(self.now)),
+ )
+
+ else:
+ # ModelCheckpoint callback created log directory --- remove it
+ if not MULTINODE_HACKS and not self.resume and os.path.exists(self.logdir):
+ dst, name = os.path.split(self.logdir)
+ dst = os.path.join(dst, "child_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ try:
+ os.rename(self.logdir, dst)
+ except FileNotFoundError:
+ pass
+
+
+class ImageLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ rescale=True,
+ disabled=True,
+ log_on_batch_idx=False,
+ log_first_step=False,
+ log_images_kwargs=None,
+ log_before_first_step=False,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ # TODO: support wandb
+ else:
+ grid = torchvision.utils.make_grid(images[k], nrow=4)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ self.check_frequency(check_idx)
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ gpu_autocast_kwargs = {
+ "enabled": self.enable_autocast, # torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad(), torch.cuda.amp.autocast(**gpu_autocast_kwargs):
+ images = pl_module.log_images(
+ batch, split=split, **self.log_images_kwargs
+ )
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ print0("[bold red]\[main][ImageLogger][/bold red]", e)
+ pass
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ print0(f"[bold red]\[main][ImageLogger][/bold red] {self.__class__.__name__}: logging before training")
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+@rank_zero_only
+def init_wandb(save_dir, opt, config, group_name, name_str):
+ print0(f"[bold red]\[main][init_wandb][/bold red] Creating WANDB_DIR: {save_dir}")
+ os.makedirs(save_dir, exist_ok=True)
+
+ # os.environ["WANDB_DIR"] = save_dir
+ gitcmd = f'git config --global --add safe.directory {os.path.dirname(os.path.abspath(__file__))}'
+ os.system(gitcmd)
+ print0(f"[bold red]\[main][init_wandb][/bold red] wandb_id is set to {opt.wandb_id}")
+ wandb_id = opt.wandb_id if opt.wandb_id is not None else name_str
+
+ if not wandb.api.api_key:
+ wandb.login(key=opt.wandb_key)
+ if opt.debug:
+ wandb.init(project=opt.wandb_project, mode="offline", group=group_name)
+ else:
+ wandb.init(
+ project=opt.wandb_project,
+ entity=opt.wandb_entity,
+ config=dict(config),
+ group=group_name,
+ name=name_str,
+ resume='auto',
+ id=wandb_id,
+ )
+
+
+if __name__ == "__main__":
+ # custom parser to specify config files, train, test and debug mode,
+ # postfix, resume.
+ # `--key value` arguments are interpreted as arguments to the trainer.
+ # `nested.key=value` arguments are interpreted as config parameters.
+ # configs are merged from left-to-right followed by command line parameters.
+
+ # model:
+ # base_learning_rate: float
+ # target: path to lightning module
+ # params:
+ # key: value
+ # data:
+ # target: main.DataModuleFromConfig
+ # params:
+ # batch_size: int
+ # wrap: bool
+ # train:
+ # target: path to train dataset
+ # params:
+ # key: value
+ # validation:
+ # target: path to validation dataset
+ # params:
+ # key: value
+ # test:
+ # target: path to test dataset
+ # params:
+ # key: value
+ # lightning: (optional, has sane defaults and can be specified on cmdline)
+ # trainer:
+ # additional arguments to trainer
+ # logger:
+ # logger to instantiate
+ # modelcheckpoint:
+ # modelcheckpoint to instantiate
+ # callbacks:
+ # callback1:
+ # target: importpath
+ # params:
+ # key: value
+
+ now = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+
+ # add cwd for convenience and to make classes in this file available when
+ # running as `python main.py`
+ # (in particular `main.DataModuleFromConfig`)
+ sys.path.append(os.getcwd())
+
+ parser = get_parser()
+
+ opt, unknown = parser.parse_known_args()
+
+ if opt.name and opt.resume:
+ raise ValueError(
+ "-n/--name and -r/--resume cannot be specified both."
+ "If you want to resume training in a new log folder, "
+ "use -n/--name in combination with --resume_from_checkpoint"
+ )
+ melk_ckpt_name = None
+ name = None
+ if opt.resume:
+ if not os.path.exists(opt.resume):
+ raise ValueError("Cannot find {}".format(opt.resume))
+ if os.path.isfile(opt.resume):
+ paths = opt.resume.split("/")
+ # idx = len(paths)-paths[::-1].index("logs")+1
+ # logdir = "/".join(paths[:idx])
+ logdir = "/".join(paths[:-2])
+ ckpt = opt.resume
+ _, melk_ckpt_name = get_checkpoint_name(logdir)
+ else:
+ assert os.path.isdir(opt.resume), opt.resume
+ logdir = opt.resume.rstrip("/")
+ ckpt, melk_ckpt_name = get_checkpoint_name(logdir)
+
+ print0("-" * 80)
+ print0(f'[bold red][main][/bold red] Resuming from checkpoint "{ckpt}"')
+
+ opt.resume_from_checkpoint = ckpt
+ base_configs = sorted(glob.glob(os.path.join(logdir, "configs/*.yaml")))
+ opt.base = base_configs + opt.base
+ _tmp = logdir.split("/")
+ nowname = _tmp[-1]
+ else:
+ if opt.name:
+ name = "_" + opt.name
+ elif opt.base:
+ if opt.no_base_name:
+ name = ""
+ else:
+ if opt.legacy_naming:
+ cfg_fname = os.path.split(opt.base[0])[-1]
+ cfg_name = os.path.splitext(cfg_fname)[0]
+ else:
+ assert "configs" in os.path.split(opt.base[0])[0], os.path.split(
+ opt.base[0]
+ )[0]
+ cfg_path = os.path.split(opt.base[0])[0].split(os.sep)[
+ os.path.split(opt.base[0])[0].split(os.sep).index("configs")
+ + 1 :
+ ] # cut away the first one (we assert all configs are in "configs")
+ cfg_name = os.path.splitext(os.path.split(opt.base[0])[-1])[0]
+ cfg_name = "-".join(cfg_path) + f"-{cfg_name}"
+ name = "_" + cfg_name
+ else:
+ name = ""
+ # automatic resume last checkpoint if available
+ if os.path.exists(opt.logdir):
+ auto_resumed = False
+ for sub_dir in sorted(os.listdir(opt.logdir)):
+ if sub_dir.endswith(name + opt.postfix):
+ ## checkpoint resume
+ if opt.checkpoint_auto_resume and not opt.debug:
+ checkpoint_dir = os.path.join(opt.logdir, sub_dir, "checkpoints")
+ # Use the max step checkpoint file
+ ckpt_files1 = glob.glob(os.path.join(checkpoint_dir, "*/*.ckpt"))
+ ckpt_files2 = glob.glob(os.path.join(checkpoint_dir, "*.ckpt"))
+ ckpt_files = ckpt_files1 + ckpt_files2
+ ckpt_files.sort(key=get_step_value, reverse=True)
+ if ckpt_files:
+ ckpt = ckpt_files[0]
+ else:
+ # If no checkpoint files found, use a random initialized model
+ ckpt = None
+ if ckpt is not None and os.path.isfile(ckpt):
+ opt.resume_from_checkpoint = ckpt
+ auto_resumed = True
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous log dir and checkpoint: {ckpt}")
+ ## wandb resume
+ if opt.wandb_auto_resume:
+ wandb_dir = Path(os.path.join(opt.logdir, sub_dir)) / "wandb"
+ if wandb_dir.exists() and any((wandb_dir / "latest-run").iterdir()):
+ # Parse unique `run_id` from the `.wandb.` file...
+ wandb_fns = [f.name for f in (wandb_dir / "latest-run").iterdir() if f.name.endswith(".wandb")]
+ assert len(wandb_fns) == 1, f"There should only be 1 `.wandb.` file... found {len(wandb_fns)}!"
+ # Regex Match on `run-{id}.wandb`
+ opt.wandb_id = re.search("run-(.+?).wandb", wandb_fns[0]).group(1)
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous wandb run id: {opt.wandb_id}")
+ if auto_resumed:
+ print0(f"[bold red]\[main][/bold red] Auto-resuming from checkpoint: {opt.resume_from_checkpoint} and wandb id: {opt.wandb_id}")
+ ckpt_basename = os.path.basename(opt.resume_from_checkpoint)
+ seed_str = ''.join(re.findall(r'\d+', ckpt_basename))
+ if len(seed_str) > 0:
+ opt.seed = int(seed_str)
+ print0(f"[bold red]\[main][/bold red] Auto-reseting seed to {opt.seed} from checkpoint name")
+
+ if not opt.no_date:
+ nowname = now + name + opt.postfix
+ else:
+ nowname = name + opt.postfix
+ if nowname.startswith("_"):
+ nowname = nowname[1:]
+ logdir = os.path.join(opt.logdir, nowname)
+ print0(f"[bold red]\[main][/bold red] LOGDIR: {logdir}")
+
+ ckptdir = os.path.join(logdir, "checkpoints")
+ cfgdir = os.path.join(logdir, "configs")
+ if not opt.seed_rank:
+ seed_everything(opt.seed, workers=True) # torch.initial_seed()
+
+ # move before model init, in case a torch.compile(...) is called somewhere
+ if opt.enable_tf32:
+ # pt_version = version.parse(torch.__version__)
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.backends.cudnn.allow_tf32 = True
+ print0(f"[bold red]\[main][/bold red] Enabling TF32 for PyTorch {torch.__version__}")
+ else:
+ print0(f"[bold red]\[main][/bold red] Using default TF32 settings for PyTorch {torch.__version__}:")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cuda.matmul.allow_tf32={torch.backends.cuda.matmul.allow_tf32}")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cudnn.allow_tf32={torch.backends.cudnn.allow_tf32}")
+
+ try:
+ # init and save configs
+ configs = [OmegaConf.load(cfg) for cfg in opt.base]
+ # deal with the unknown args, e.g., --model.base_learning_rate=1.0e-4
+ for i, u in enumerate(unknown):
+ if u.startswith("--"):
+ unknown[i] = u[2:]
+ # merge all configs and cli args
+ cli = OmegaConf.from_dotlist(unknown)
+ config = OmegaConf.merge(*configs, cli)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged input config: {config}")
+ lightning_config = config.pop("lightning", OmegaConf.create())
+ # merge trainer cli with config
+ trainer_config = lightning_config.get("trainer", OmegaConf.create())
+
+ # debug: default to one node
+ if opt.debug:
+ trainer_config["num_nodes"] = 1
+
+ # default profiler
+ trainer_config["profiler"] = None if not opt.debug else "simple"
+
+ # default to gpu
+ trainer_config["accelerator"] = "gpu"
+ #
+ standard_args = default_trainer_args()
+ for k in standard_args:
+ if getattr(opt, k) != standard_args[k]:
+ trainer_config[k] = getattr(opt, k)
+
+ if not "devices" in trainer_config and trainer_config["accelerator"] != "gpu":
+ del trainer_config["accelerator"]
+ cpu = True
+ else:
+ gpuinfo = trainer_config["devices"]
+ print0(f"[bold red]\[main][/bold red] Running on {gpuinfo} GPUs")
+ cpu = False
+ trainer_opt = argparse.Namespace(**trainer_config)
+ lightning_config.trainer = trainer_config
+
+ # model
+ model = instantiate_from_config(config.model)
+
+ # trainer and callbacks
+ trainer_kwargs = dict()
+
+ # default logger configs
+ default_logger_cfgs = {
+ "wandb": {
+ "target": "lightning.pytorch.loggers.WandbLogger",
+ "params": {
+ "name": nowname,
+ "save_dir": logdir,
+ "offline": opt.debug,
+ "id": nowname,
+ "project": opt.wandb_project,
+ "log_model": False,
+ "entity": opt.wandb_entity,
+ },
+ },
+ "csv": {
+ "target": "lightning.pytorch.loggers.CSVLogger",
+ "params": {
+ "name": "testtube", # hack for sbord fanatics
+ "save_dir": logdir,
+ },
+ },
+ "tensorboard": {
+ "target": "lightning.pytorch.loggers.TensorBoardLogger",
+ "params": {
+ "save_dir": logdir,
+ "name": 'tensorboard',
+ "version": nowname,
+ }
+ },
+ }
+ default_logger_cfg = default_logger_cfgs["wandb" if opt.wandb else "tensorboard"]
+ if opt.wandb:
+ # change once leaving "swiffer" config directory
+ try:
+ group_name = nowname.split(now)[-1].split("-")[1]
+ except:
+ group_name = nowname
+ default_logger_cfg["params"]["group"] = group_name
+
+ wandb_save_dir = os.path.join(os.getcwd(), logdir)
+ os.environ["WANDB_DIR"] = wandb_save_dir
+
+ init_wandb(
+ wandb_save_dir,
+ opt=opt,
+ group_name=group_name,
+ config=config,
+ name_str=nowname,
+ )
+ if "logger" in lightning_config:
+ logger_cfg = lightning_config.logger
+ else:
+ logger_cfg = OmegaConf.create()
+ logger_cfg = OmegaConf.merge(default_logger_cfg, logger_cfg)
+ trainer_kwargs["logger"] = instantiate_from_config(logger_cfg)
+
+ ckpt_resume_path = opt.resume_from_checkpoint
+
+ # modelcheckpoint - use TrainResult/EvalResult(checkpoint_on=metric) to
+ # specify which metric is used to determine best models
+ default_modelckpt_cfg = {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": ckptdir,
+ "filename": "{epoch:04}-{step:08}", # "epoch={epoch:06}-step={step:07}"
+ "verbose": True,
+ "save_last": True,
+ "auto_insert_metric_name": True,
+ },
+ }
+ if hasattr(model, "monitor"):
+ print0(f"[bold red]\[main][/bold red] Monitoring {model.monitor} as checkpoint metric.")
+ default_modelckpt_cfg["params"]["monitor"] = model.monitor
+ default_modelckpt_cfg["params"]["save_top_k"] = 3
+
+ if "modelcheckpoint" in lightning_config:
+ modelckpt_cfg = lightning_config.modelcheckpoint
+ else:
+ modelckpt_cfg = OmegaConf.create()
+ modelckpt_cfg = OmegaConf.merge(default_modelckpt_cfg, modelckpt_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged modelckpt-cfg: {modelckpt_cfg}")
+
+ # https://pytorch-lightning.readthedocs.io/en/stable/extensions/strategy.html
+ # default to ddp if not further specified
+ default_strategy_config = {"target": "lightning.pytorch.strategies.DDPStrategy"}
+
+ if "strategy" in lightning_config:
+ strategy_cfg = lightning_config.strategy
+ else:
+ strategy_cfg = OmegaConf.create()
+ default_strategy_config["params"] = {
+ "find_unused_parameters": False,
+ # "static_graph": True,
+ # "ddp_comm_hook": default.fp16_compress_hook # experiment with this, also for DDPSharded
+ }
+ strategy_cfg = OmegaConf.merge(default_strategy_config, strategy_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] strategy config: {strategy_cfg}")
+ trainer_kwargs["strategy"] = instantiate_from_config(strategy_cfg)
+ if hasattr(trainer_kwargs["strategy"], "_timeout"):
+ trainer_kwargs["strategy"]._timeout = datetime.timedelta(seconds=5400) # 3600s = 1h
+
+ # add callback which sets up log directory
+ default_callbacks_cfg = {
+ "setup_callback": {
+ "target": "main.SetupCallback",
+ "params": {
+ "resume": opt.resume,
+ "now": now,
+ "logdir": logdir,
+ "ckptdir": ckptdir,
+ "cfgdir": cfgdir,
+ "config": config,
+ "lightning_config": lightning_config,
+ "debug": opt.debug,
+ "ckpt_name": melk_ckpt_name,
+ "seed": opt.seed,
+ "seed_rank": opt.seed_rank
+ },
+ },
+ "image_logger": {
+ "target": "main.ImageLogger",
+ "params": {"batch_frequency": 1000, "max_samples": 4, "clamp": True},
+ },
+ "learning_rate_logger": {
+ "target": "lightning.pytorch.callbacks.LearningRateMonitor",
+ "params": {
+ "logging_interval": "step",
+ # "log_momentum": True
+ },
+ },
+ }
+ if version.parse(pl.__version__) >= version.parse("1.4.0"):
+ default_callbacks_cfg.update({"checkpoint_callback": modelckpt_cfg})
+
+ if "callbacks" in lightning_config:
+ callbacks_cfg = lightning_config.callbacks
+ else:
+ callbacks_cfg = OmegaConf.create()
+
+ if "metrics_over_trainsteps_checkpoint" in callbacks_cfg:
+ print0(
+ "[bold red]\[main][/bold red] Caution: Saving checkpoints every n train steps without deleting. This might require some free space."
+ )
+ default_metrics_over_trainsteps_ckpt_dict = {
+ "metrics_over_trainsteps_checkpoint": {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": os.path.join(ckptdir, "trainstep_checkpoints"),
+ "filename": "{epoch:04}-{step:08}", # "{epoch:06}-{step:09}"
+ "verbose": True,
+ "save_top_k": -1,
+ "every_n_train_steps": 10000,
+ "save_weights_only": True,
+ },
+ }
+ }
+ default_callbacks_cfg.update(default_metrics_over_trainsteps_ckpt_dict)
+
+ callbacks_cfg = OmegaConf.merge(default_callbacks_cfg, callbacks_cfg)
+ if "ignore_keys_callback" in callbacks_cfg and ckpt_resume_path is not None:
+ callbacks_cfg.ignore_keys_callback.params["ckpt_path"] = ckpt_resume_path
+ elif "ignore_keys_callback" in callbacks_cfg:
+ del callbacks_cfg["ignore_keys_callback"]
+
+ trainer_kwargs["callbacks"] = [
+ instantiate_from_config(callbacks_cfg[k]) for k in callbacks_cfg
+ ]
+ if not "plugins" in trainer_kwargs:
+ trainer_kwargs["plugins"] = list()
+
+ # cmd line trainer args (which are in trainer_opt) have always priority over config-trainer-args (which are in trainer_kwargs)
+ trainer_opt = vars(trainer_opt)
+ trainer_kwargs = {
+ key: val for key, val in trainer_kwargs.items() if key not in trainer_opt
+ }
+ trainer = Trainer(**trainer_opt, **trainer_kwargs)
+
+ trainer.logdir = logdir
+
+ # data
+ if ((not opt.train) or opt.debug) and hasattr(config.data.params, "validation"):
+ config.data.params.train = config.data.params.validation
+ print0("[bold red]\[main][/bold red] Using validation data as training data for fast loading.")
+ data = instantiate_from_config(config.data)
+ # NOTE according to https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html
+ # calling these ourselves should not be necessary but it is.
+ # lightning still takes care of proper multiprocessing though
+ data.prepare_data()
+ # data.setup()
+ try:
+ for k in data.datasets:
+ print0(
+ f"[bold red]\[main][/bold red] {k}, {data.datasets[k].__class__.__name__}, {len(data.datasets[k])}"
+ )
+ except:
+ print0("[bold red]\[main][/bold red] datasets not yet initialized.")
+
+ # configure learning rate
+ if "batch_size" in config.data.params:
+ bs, base_lr = config.data.params.batch_size, config.model.base_learning_rate
+ else:
+ bs, base_lr = (
+ config.data.params.train.loader.batch_size,
+ config.model.base_learning_rate,
+ )
+ if not cpu:
+ # add for different device input type
+ if isinstance(lightning_config.trainer.devices, int):
+ ngpu = lightning_config.trainer.devices
+ elif isinstance(lightning_config.trainer.devices, list):
+ ngpu = len(lightning_config.trainer.devices)
+ elif isinstance(lightning_config.trainer.devices, str):
+ ngpu = len(lightning_config.trainer.devices.strip(",").split(","))
+ else:
+ ngpu = 1
+ if "accumulate_grad_batches" in lightning_config.trainer:
+ accumulate_grad_batches = lightning_config.trainer.accumulate_grad_batches
+ else:
+ accumulate_grad_batches = 1
+ print0(f"[bold red]\[main][/bold red] accumulate_grad_batches = {accumulate_grad_batches}")
+ lightning_config.trainer.accumulate_grad_batches = accumulate_grad_batches
+
+ if opt.scale_lr:
+ model.learning_rate = accumulate_grad_batches * ngpu * bs * base_lr
+ print0(
+ "[bold red]\[main][/bold red] Setting learning rate to {:.2e} = {} (accumulate_grad_batches) * {} (num_gpus) * {} (batchsize) * {:.2e} (base_lr)".format(
+ model.learning_rate, accumulate_grad_batches, ngpu, bs, base_lr
+ )
+ )
+ else:
+ model.learning_rate = base_lr
+ print0("[bold red]\[main][/bold red] NOT using learning rate scaling")
+ print0(f"[bold red]\[main][/bold red] Setting learning rate to {model.learning_rate:.2e}")
+
+ # allow checkpointing via USR1
+ def melk(*args, **kwargs):
+ # run all checkpoint hooks
+ if trainer.global_rank == 0:
+ melkdir = os.path.join(logdir, "melk")
+ os.makedirs(melkdir, exist_ok=True)
+ print0(f"[bold red]\[main][/bold red] Saving checkpoint to {melkdir}")
+ if melk_ckpt_name is None:
+ ckpt_path = os.path.join(melkdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(melkdir, melk_ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def divein(*args, **kwargs):
+ if trainer.global_rank == 0:
+ import pudb
+ pudb.set_trace()
+
+ import signal
+ signal.signal(signal.SIGUSR1, melk)
+ signal.signal(signal.SIGUSR2, divein)
+
+ # run
+ if opt.train:
+ try:
+ trainer.fit(model, data, ckpt_path=ckpt_resume_path)
+ print0(f"[bold red]\[main][/bold red] Finish training with logdir: {logdir}")
+ except Exception as e:
+ print(f"")
+ print(f"[bold red]\[main][/bold red] Exception: {e}")
+ print(f"[bold red]\[main][/bold red] Beijing Time {datetime.datetime.now(tz=pytz.timezone('Asia/Shanghai'))}")
+ if not opt.debug:
+ melk()
+ raise
+ else:
+ trainer.validate(model, data, ckpt_path=ckpt_resume_path)
+ exit()
+ if not opt.no_test and not trainer.interrupted:
+ trainer.test(model, data)
+ except RuntimeError as err:
+ if MULTINODE_HACKS:
+ import datetime
+ import os
+ import socket
+ import requests
+
+ device = os.environ.get("CUDA_VISIBLE_DEVICES", "?")
+ hostname = socket.gethostname()
+ ts = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+ resp = requests.get("http://169.254.169.254/latest/meta-data/instance-id")
+ print(
+ f"[bold red]\[main][/bold red] ERROR at {ts} on {hostname}/{resp.text} (CUDA_VISIBLE_DEVICES={device}): {type(err).__name__}: {err}",
+ flush=True,
+ )
+ raise err
+ except Exception:
+ if opt.debug and trainer.global_rank == 0:
+ try:
+ import pudb as debugger
+ except ImportError:
+ import pdb as debugger
+ # debugger.post_mortem()
+ raise
+ finally:
+ # move newly created debug project to debug_runs
+ if opt.debug and not opt.resume and trainer.global_rank == 0:
+ dst, name = os.path.split(logdir)
+ dst = os.path.join(dst, "debug_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ os.rename(logdir, dst)
+
+ if opt.wandb:
+ wandb.finish()
+
+ # clean up
+ # dist.barrier()
+ # torch.cuda.empty_cache()
+ dist.destroy_process_group()
+
+ if trainer.global_rank == 0 and opt.debug:
+ print0(f"[bold red]\[main][/bold red] Current logdir: {logdir}")
+ # print0(f"[bold red]\[main][/bold red] Profiler summary:")
+ # print(trainer.profiler.summary())
+ print0(f"[bold red]\[main][/bold red] Memory summary:")
+ num_params = sum([p.numel() for p in model.parameters()])
+ print0(f"[bold red]\[main][/bold red] Expected bf16 memory usage from params: {num_params * 2 / 1e9:.2f} GB")
+ print0(f"[bold red]\[main][/bold red] Current memory usage with model on device {torch.cuda.max_memory_allocated() / 1e9:.2f} GB")
+ # trainer.print(torch.cuda.memory_summary())
diff --git a/Meissonic/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc b/Meissonic/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9dc3ee2b98d3f51e021fb953effcd638ef0ca8ec
Binary files /dev/null and b/Meissonic/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/scripts/inference_evaluate.py b/Meissonic/VidTok/scripts/inference_evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4152046727305af7b32e0a3e5728072b7085ca4
--- /dev/null
+++ b/Meissonic/VidTok/scripts/inference_evaluate.py
@@ -0,0 +1,198 @@
+import argparse
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from torch import autocast
+from tqdm import tqdm
+
+import numpy as np
+import torch
+from einops import rearrange
+from lightning.pytorch import seed_everything
+
+from vidtok.data.vidtok import VidTokValDataset
+from vidtok.modules.lpips import LPIPS
+from vidtok.modules.util import (compute_psnr, compute_ssim,
+ instantiate_from_config, print0)
+
+
+def load_model_from_config(config, ckpt, ignore_keys=[], verbose=False):
+ config = OmegaConf.load(config)
+ config.model.params.ckpt_path = ckpt
+ config.model.params.ignore_keys = ignore_keys
+ config.model.params.verbose = verbose
+ model = instantiate_from_config(config.model)
+ return model
+
+
+class MultiVideoDataset(VidTokValDataset):
+ def __init__(
+ self,
+ data_dir,
+ meta_path=None,
+ input_height=256,
+ input_width=256,
+ sample_fps=30,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ super().__init__(
+ data_dir=data_dir,
+ meta_path=meta_path,
+ video_params={
+ "input_height": input_height,
+ "input_width": input_width,
+ "sample_num_frames": chunk_size + 1 if is_causal else chunk_size,
+ "sample_fps": sample_fps,
+ },
+ pre_load_frames=True,
+ last_frames_handle="repeat",
+ read_long_video=read_long_video,
+ chunk_size=chunk_size,
+ is_causal=is_causal,
+ )
+
+ def __getitem__(self, idx):
+ frames = super().__getitem__(idx)["jpg"]
+ return frames
+
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--data_dir",
+ type=str,
+ default="./",
+ help="root folder",
+ )
+ parser.add_argument(
+ "--meta_path",
+ type=str,
+ default=None,
+ help="path to the .csv meta file",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.enable_tiling()
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+
+ if args.input_width > 256:
+ model.enable_tiling()
+
+ dataset = MultiVideoDataset(
+ data_dir=args.data_dir,
+ meta_path=args.meta_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ perceptual_loss = LPIPS().eval()
+ perceptual_loss = perceptual_loss.to(device)
+
+ psnrs, ssims, lpipss = [], [], []
+
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+ _, output, reg_log = model(input)
+ output = output.clamp(-1, 1)
+ input, output = map(lambda x: (x + 1) / 2, (input, output))
+
+ if input.dim() == 5:
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ assert output.dim() == 5
+ output = rearrange(output, "b c t h w -> (b t) c h w")
+
+ for inp, out in zip(torch.split(input, 16), torch.split(output, 16)):
+ psnrs += [compute_psnr(inp, out).item()] * inp.shape[0]
+ ssims += [compute_ssim(inp, out).item()] * inp.shape[0]
+ lpipss += [perceptual_loss(inp * 2 - 1, out * 2 - 1).mean().item()] * inp.shape[0]
+
+ toc = time.time()
+ print0(
+ f"[bold red]\[scripts.inference_evaluate][/bold red] PSNR: {np.mean(psnrs):.4f}, SSIM: {np.mean(ssims):.4f}, LPIPS: {np.mean(lpipss):.4f}"
+ )
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/VidTok/scripts/inference_reconstruct.py b/Meissonic/VidTok/scripts/inference_reconstruct.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a26b9475339f6675e01fd052637c8465ca37caf
--- /dev/null
+++ b/Meissonic/VidTok/scripts/inference_reconstruct.py
@@ -0,0 +1,246 @@
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import argparse
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from pathlib import Path
+from tqdm import tqdm
+
+import numpy as np
+import torch
+import decord
+from einops import rearrange
+from lightning.pytorch import seed_everything
+from torch import autocast
+from torchvision import transforms
+from torchvision.io import write_video
+
+from vidtok.modules.util import print0
+from scripts.inference_evaluate import load_model_from_config
+
+
+class SingleVideoDataset(torch.utils.data.Dataset):
+ def __init__(
+ self,
+ video_path,
+ input_height=128,
+ input_width=128,
+ sample_fps=8,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ decord.bridge.set_bridge("torch")
+ self.video_path = video_path
+ self.transform = transforms.Compose(
+ [
+ transforms.Resize(input_height, antialias=True),
+ transforms.CenterCrop((input_height, input_width)),
+ transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
+ ]
+ )
+
+ self.video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(self.video_reader)
+ fps = self.video_reader.get_avg_fps() # float
+
+ interval = round(fps / sample_fps)
+ frame_ids = list(range(0, total_frames, interval))
+ self.frame_ids_batch = []
+ if read_long_video:
+ video_length = len(frame_ids)
+ if is_causal and video_length > chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * ((video_length - 1) // chunk_size) + 1])
+ elif not is_causal and video_length >= chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * (video_length // chunk_size)])
+ else:
+ num_frames_per_batch = chunk_size + 1 if is_causal else chunk_size
+ for x in range(0, len(frame_ids), num_frames_per_batch):
+ if len(frame_ids[x : x + num_frames_per_batch]) == num_frames_per_batch:
+ self.frame_ids_batch.append(frame_ids[x : x + num_frames_per_batch])
+
+ def __len__(self):
+ return len(self.frame_ids_batch)
+
+ def __getitem__(self, idx):
+ frame_ids = self.frame_ids_batch[idx]
+ frames = self.video_reader.get_batch(frame_ids).permute(0, 3, 1, 2).float() / 255.0
+ frames = self.transform(frames).permute(1, 0, 2, 3)
+ return frames
+
+
+def tensor_to_uint8(tensor):
+ tensor = torch.clamp(tensor, -1.0, 1.0)
+ tensor = (tensor + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ tensor = (tensor.cpu().numpy() * 255).astype(np.uint8)
+ return tensor
+
+
+def main():
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--output_video_dir",
+ type=str,
+ default="tmp",
+ help="path to save the outputs",
+ )
+ parser.add_argument(
+ "--input_video_path",
+ type=str,
+ default="assets/example.mp4",
+ help="path to the input video",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+ parser.add_argument(
+ "--pad_gen_frames",
+ action="store_true",
+ help="Used only in causal mode. If True, pad frames generated in the last batch, else replicate the first frame instead",
+ )
+ parser.add_argument(
+ "--concate_input",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="",
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+ config = OmegaConf.load(args.config)
+
+ os.makedirs(args.output_video_dir, exist_ok=True)
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.use_tiling = True
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+ model.use_overlap = True
+
+ dataset = SingleVideoDataset(
+ video_path=args.input_video_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ inputs = []
+ outputs = []
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+
+ if model.is_causal and not args.read_long_video and args.pad_gen_frames:
+ if i == 0:
+ _, xrec, _ = model(input)
+ else:
+ _, xrec, _ = model(torch.cat([last_gen_frames, input], dim=2))
+ xrec = xrec[:, :, -input.shape[2]:].clamp(-1, 1)
+ last_gen_frames = xrec[:, :, (1 - model.encoder.time_downsample_factor):, :, :]
+ else:
+ _, xrec, _ = model(input)
+
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ inputs.append(input)
+ xrec = rearrange(xrec.clamp(-1, 1), "b c t h w -> (b t) c h w")
+ outputs.append(xrec)
+
+ toc = time.time()
+
+ # save the outputs as videos
+ inputs = tensor_to_uint8(torch.cat(inputs, dim=0))
+ inputs = rearrange(inputs, "t c h w -> t h w c")
+ outputs = tensor_to_uint8(torch.cat(outputs, dim=0))
+ outputs = rearrange(outputs, "t c h w -> t h w c")
+ min_len = min(inputs.shape[0], outputs.shape[0])
+ final = np.concatenate([inputs[:min_len], outputs[:min_len]], axis=2) if args.concate_input else outputs[:min_len]
+
+ output_video_path = os.path.join(args.output_video_dir, f"{Path(args.input_video_path).stem}_reconstructed.mp4")
+ write_video(output_video_path, final, args.sample_fps)
+
+ print0(f"[bold red]Results saved in: {output_video_path}[/bold red]")
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/VidTok/test_vidtok.py b/Meissonic/VidTok/test_vidtok.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4c82d5b7d1ff85d8713321354841413318ba6a5
--- /dev/null
+++ b/Meissonic/VidTok/test_vidtok.py
@@ -0,0 +1,758 @@
+#!/usr/bin/env python3
+"""
+Test script for VidTok tokenizer performance.
+
+This script:
+1. Loads a video from the training dataset
+2. Encodes it using VidTok tokenizer
+3. Decodes it back
+4. Computes metrics (PSNR, SSIM, MSE)
+5. Creates a side-by-side comparison video
+6. Saves the results
+
+Based on VidTok: https://github.com/microsoft/VidTok
+"""
+
+import argparse
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import torch
+import numpy as np
+from PIL import Image
+import cv2
+from torchvision import transforms
+from torchvision.utils import make_grid, save_image
+
+# VidTok imports - adjust path if needed
+VIDTOK_AVAILABLE = False
+VIDTOK_PATH = None
+
+def _setup_vidtok():
+ """Setup VidTok by trying to import or download from GitHub."""
+ global VIDTOK_AVAILABLE, VIDTOK_PATH
+
+ # Try to import from existing installation
+ try:
+ from scripts.inference_evaluate import load_model_from_config
+ VIDTOK_AVAILABLE = True
+ return load_model_from_config
+ except ImportError:
+ pass
+
+ # Try to find VidTok in common locations
+ vidtok_paths = [
+ "VidTok",
+ "../VidTok",
+ os.path.join(os.path.dirname(__file__), "../VidTok"),
+ os.path.expanduser("~/VidTok"),
+ ]
+
+ for vidtok_path in vidtok_paths:
+ if os.path.exists(vidtok_path) and os.path.exists(os.path.join(vidtok_path, "scripts")):
+ sys.path.insert(0, vidtok_path)
+ try:
+ from scripts.inference_evaluate import load_model_from_config
+ VIDTOK_AVAILABLE = True
+ VIDTOK_PATH = vidtok_path
+ print(f"Found VidTok at: {vidtok_path}")
+ return load_model_from_config
+ except ImportError:
+ if vidtok_path in sys.path:
+ sys.path.remove(vidtok_path)
+ continue
+
+ # Try to download from GitHub
+ print("VidTok not found locally. Attempting to download from GitHub...")
+ try:
+ import subprocess
+ import tempfile
+
+ # Create cache directory
+ cache_dir = os.path.join(os.getcwd(), "vidtok_cache")
+ vidtok_dir = os.path.join(cache_dir, "VidTok")
+
+ # Check if already downloaded
+ if os.path.exists(vidtok_dir) and os.path.exists(os.path.join(vidtok_dir, "scripts")):
+ sys.path.insert(0, vidtok_dir)
+ try:
+ from scripts.inference_evaluate import load_model_from_config
+ VIDTOK_AVAILABLE = True
+ VIDTOK_PATH = vidtok_dir
+ print(f"Using cached VidTok from: {vidtok_dir}")
+ return load_model_from_config
+ except ImportError:
+ if vidtok_dir in sys.path:
+ sys.path.remove(vidtok_dir)
+
+ # Download from GitHub
+ print("Downloading VidTok from GitHub...")
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Use git clone if available, otherwise download zip
+ if subprocess.run(["which", "git"], capture_output=True).returncode == 0:
+ # Clone repository
+ if os.path.exists(vidtok_dir):
+ import shutil
+ shutil.rmtree(vidtok_dir)
+
+ result = subprocess.run(
+ ["git", "clone", "--depth", "1", "https://github.com/microsoft/VidTok.git", vidtok_dir],
+ capture_output=True,
+ text=True
+ )
+ if result.returncode == 0:
+ sys.path.insert(0, vidtok_dir)
+ try:
+ from scripts.inference_evaluate import load_model_from_config
+ VIDTOK_AVAILABLE = True
+ VIDTOK_PATH = vidtok_dir
+ print(f"Successfully downloaded VidTok to: {vidtok_dir}")
+ return load_model_from_config
+ except ImportError as e:
+ if vidtok_dir in sys.path:
+ sys.path.remove(vidtok_dir)
+ print(f"Failed to import VidTok after download: {e}")
+ else:
+ # Fallback: download zip file
+ print("Git not available, trying to download zip file...")
+ import urllib.request
+ import zipfile
+
+ zip_url = "https://github.com/microsoft/VidTok/archive/refs/heads/main.zip"
+ zip_path = os.path.join(cache_dir, "VidTok-main.zip")
+
+ print(f"Downloading {zip_url}...")
+ urllib.request.urlretrieve(zip_url, zip_path)
+
+ # Extract zip
+ with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+ zip_ref.extractall(cache_dir)
+
+ # Rename extracted directory
+ extracted_dir = os.path.join(cache_dir, "VidTok-main")
+ if os.path.exists(vidtok_dir):
+ import shutil
+ shutil.rmtree(vidtok_dir)
+ os.rename(extracted_dir, vidtok_dir)
+
+ sys.path.insert(0, vidtok_dir)
+ try:
+ from scripts.inference_evaluate import load_model_from_config
+ VIDTOK_AVAILABLE = True
+ VIDTOK_PATH = vidtok_dir
+ print(f"Successfully downloaded VidTok to: {vidtok_dir}")
+ return load_model_from_config
+ except ImportError as e:
+ if vidtok_dir in sys.path:
+ sys.path.remove(vidtok_dir)
+ print(f"Failed to import VidTok after download: {e}")
+
+ except Exception as e:
+ print(f"Failed to download VidTok: {e}")
+
+ return None
+
+# Setup VidTok
+load_model_from_config = _setup_vidtok()
+sys.path.append("/mnt/Meissonic")
+
+
+from train.dataset_utils import OpenVid1MDataset
+from transformers import T5Tokenizer
+
+
+
+
+def calculate_psnr(img1, img2, max_val=1.0):
+ """Calculate PSNR between two images."""
+ # Ensure both tensors are on the same device (preferably CPU for metric calculation)
+ if img1.device != img2.device:
+ img1 = img1.to(img2.device)
+ # Move to CPU for metric calculation to avoid GPU memory issues
+ img1 = img1.cpu()
+ img2 = img2.cpu()
+
+ mse = torch.mean((img1 - img2) ** 2)
+ if mse == 0:
+ return float('inf')
+ psnr = 20 * torch.log10(max_val / torch.sqrt(mse))
+ return psnr.item()
+
+
+def calculate_mse(img1, img2):
+ """Calculate MSE between two images."""
+ # Ensure both tensors are on the same device (preferably CPU for metric calculation)
+ if img1.device != img2.device:
+ img1 = img1.to(img2.device)
+ # Move to CPU for metric calculation to avoid GPU memory issues
+ img1 = img1.cpu()
+ img2 = img2.cpu()
+
+ return torch.mean((img1 - img2) ** 2).item()
+
+
+def calculate_ssim(img1, img2, window_size=11):
+ """Calculate SSIM between two images (simplified version)."""
+ # Ensure both tensors are on the same device (preferably CPU for metric calculation)
+ if img1.device != img2.device:
+ img1 = img1.to(img2.device)
+ # Move to CPU for metric calculation to avoid GPU memory issues
+ img1 = img1.cpu()
+ img2 = img2.cpu()
+
+ # Simple SSIM approximation
+ C1 = 0.01 ** 2
+ C2 = 0.03 ** 2
+
+ mu1 = img1.mean()
+ mu2 = img2.mean()
+
+ sigma1_sq = img1.var()
+ sigma2_sq = img2.var()
+ sigma12 = ((img1 - mu1) * (img2 - mu2)).mean()
+
+ ssim = ((2 * mu1 * mu2 + C1) * (2 * sigma12 + C2)) / ((mu1**2 + mu2**2 + C1) * (sigma1_sq + sigma2_sq + C2))
+ return ssim.item()
+
+
+def video_to_numpy(video_tensor):
+ """
+ Convert video tensor [C, F, H, W] in [-1, 1] or [0, 1] to numpy array [F, H, W, C] in [0, 255] (RGB).
+ VidTok uses [-1, 1] range.
+ """
+ if isinstance(video_tensor, torch.Tensor):
+ # [C, F, H, W] -> [F, C, H, W] -> [F, H, W, C]
+ video_np = video_tensor.permute(1, 0, 2, 3).cpu().numpy() # [F, C, H, W]
+ video_np = np.transpose(video_np, (0, 2, 3, 1)) # [F, H, W, C]
+
+ # Normalize to [0, 1] range (VidTok uses [-1, 1])
+ if video_np.min() < 0:
+ # Assume range is [-1, 1]
+ video_np = (video_np + 1) / 2
+ else:
+ # Assume range is [0, 1]
+ video_np = np.clip(video_np, 0, 1)
+
+ # Convert to [0, 255]
+ video_np = (video_np * 255).astype(np.uint8)
+ else:
+ video_np = np.array(video_tensor)
+ return video_np
+
+
+def add_text_to_image(image_tensor, text, position=(10, 30)):
+ """
+ Add text label to an image tensor.
+
+ Args:
+ image_tensor: Image tensor [C, H, W] in [0, 1] or [-1, 1]
+ text: Text to add
+ position: (x, y) position for text
+ Returns:
+ Image tensor with text [C, H, W]
+ """
+ # Normalize to [0, 1] if needed
+ img_norm = image_tensor.clone()
+ if img_norm.min() < 0:
+ img_norm = (img_norm + 1) / 2
+ img_norm = torch.clamp(img_norm, 0, 1)
+
+ # Convert to PIL Image
+ image_np = img_norm.permute(1, 2, 0).cpu().numpy() # [H, W, C]
+ image_np = (image_np * 255).astype(np.uint8)
+ pil_image = Image.fromarray(image_np)
+
+ # Add text
+ from PIL import ImageDraw, ImageFont
+ draw = ImageDraw.Draw(pil_image)
+ try:
+ font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 24)
+ except:
+ try:
+ font = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 24)
+ except:
+ font = ImageFont.load_default()
+
+ # Draw white text with black outline
+ x, y = position
+ # Draw outline
+ for adj in [(-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1), (1, -1), (1, 0), (1, 1)]:
+ draw.text((x + adj[0], y + adj[1]), text, font=font, fill=(0, 0, 0))
+ # Draw main text
+ draw.text((x, y), text, font=font, fill=(255, 255, 255))
+
+ # Convert back to tensor (normalize to [0, 1])
+ image_tensor = transforms.ToTensor()(pil_image)
+ return image_tensor
+
+
+def create_side_by_side_video(original, reconstructed, output_path, fps=8):
+ """
+ Create a side-by-side comparison video.
+
+ Args:
+ original: Original video tensor [C, F, H, W] in [0, 1] or [-1, 1]
+ reconstructed: Reconstructed video tensor [C, F, H, W] in [0, 1] or [-1, 1]
+ output_path: Path to save the output video
+ fps: Frames per second
+ """
+ # Convert to numpy (RGB format: [F, H, W, C])
+ orig_np = video_to_numpy(original)
+ recon_np = video_to_numpy(reconstructed)
+
+ # Get dimensions
+ F, H, W, C = orig_np.shape
+ F_recon, H_recon, W_recon, C_recon = recon_np.shape
+
+ # Ensure same number of frames
+ F_min = min(F, F_recon)
+ orig_np = orig_np[:F_min]
+ recon_np = recon_np[:F_min]
+
+ # Resize if dimensions don't match
+ if H != H_recon or W != W_recon:
+ print(f"Resizing reconstructed video from ({H_recon}, {W_recon}) to ({H}, {W})")
+ recon_np_resized = np.zeros((F_min, H, W, C), dtype=np.uint8)
+ for f in range(F_min):
+ recon_np_resized[f] = cv2.resize(recon_np[f], (W, H), interpolation=cv2.INTER_LINEAR)
+ recon_np = recon_np_resized
+
+ # Add text labels to frames
+ from PIL import Image, ImageDraw, ImageFont
+ side_by_side_frames = []
+ for f in range(F_min):
+ # Original frame with label
+ orig_frame_pil = Image.fromarray(orig_np[f])
+ draw = ImageDraw.Draw(orig_frame_pil)
+ try:
+ font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 32)
+ except:
+ try:
+ font = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 32)
+ except:
+ font = ImageFont.load_default()
+ # Draw text with outline for visibility
+ text = "Original"
+ x, y = 20, 20
+ for adj in [(-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1), (1, -1), (1, 0), (1, 1)]:
+ draw.text((x + adj[0], y + adj[1]), text, font=font, fill=(0, 0, 0))
+ draw.text((x, y), text, font=font, fill=(255, 255, 255))
+ orig_frame = np.array(orig_frame_pil)
+
+ # Reconstructed frame with label
+ recon_frame_pil = Image.fromarray(recon_np[f])
+ draw = ImageDraw.Draw(recon_frame_pil)
+ text = "Reconstructed"
+ x, y = 20, 20
+ for adj in [(-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1), (1, -1), (1, 0), (1, 1)]:
+ draw.text((x + adj[0], y + adj[1]), text, font=font, fill=(0, 0, 0))
+ draw.text((x, y), text, font=font, fill=(255, 255, 0)) # Yellow text
+ recon_frame = np.array(recon_frame_pil)
+
+ # Concatenate horizontally
+ frame = np.concatenate([orig_frame, recon_frame], axis=1)
+ side_by_side_frames.append(frame)
+
+ # Write video using OpenCV (needs BGR format)
+ fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+ out = cv2.VideoWriter(output_path, fourcc, fps, (W * 2, H))
+
+ if not out.isOpened():
+ print(f"Warning: Could not open video writer with mp4v codec, trying XVID...")
+ fourcc = cv2.VideoWriter_fourcc(*'XVID')
+ out = cv2.VideoWriter(output_path, fourcc, fps, (W * 2, H))
+
+ for frame in side_by_side_frames:
+ # Convert RGB to BGR for OpenCV
+ frame_bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+ out.write(frame_bgr)
+
+ out.release()
+ print(f"Saved side-by-side video to: {output_path}")
+
+
+def create_comparison_grid(original, reconstructed, output_path, nrow=4):
+ """
+ Create a grid image comparing original and reconstructed frames.
+
+ Args:
+ original: Original video tensor [C, F, H, W] in [0, 1] or [-1, 1]
+ reconstructed: Reconstructed video tensor [C, F, H, W] in [0, 1] or [-1, 1]
+ output_path: Path to save the grid image
+ nrow: Number of frames per row
+ """
+ # Get number of frames
+ F = min(original.shape[1], reconstructed.shape[1])
+
+ # Select frames to display
+ num_frames_to_show = min(8, F)
+ frame_indices = np.linspace(0, F - 1, num_frames_to_show, dtype=int)
+
+ frames_list = []
+ for idx in frame_indices:
+ # Original frame with label
+ orig_frame = original[:, idx, :, :].clone() # [C, H, W]
+ orig_frame = add_text_to_image(orig_frame, "Original", position=(10, 10))
+ frames_list.append(orig_frame)
+
+ # Reconstructed frame with label
+ recon_frame = reconstructed[:, idx, :, :].clone() # [C, H, W]
+ recon_frame = add_text_to_image(recon_frame, "Reconstructed", position=(10, 10))
+ frames_list.append(recon_frame)
+
+ # Create grid
+ frames_tensor = torch.stack(frames_list, dim=0)
+ grid = make_grid(frames_tensor, nrow=nrow * 2, padding=2, pad_value=1.0)
+
+ save_image(grid, output_path)
+ print(f"Saved comparison grid to: {output_path}")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Test VidTok tokenizer performance")
+
+ parser.add_argument(
+ "--config",
+ type=str,
+ default=None,
+ help="Path to VidTok config file (e.g., configs/vidtok_kl_causal_488_4chn.yaml). "
+ "If not provided, will try to download from HuggingFace."
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default=None,
+ help="Path to VidTok checkpoint file or HuggingFace model ID. "
+ "If HuggingFace model ID (e.g., microsoft/VidTok), will download automatically."
+ )
+ parser.add_argument(
+ "--model_name",
+ type=str,
+ default="vidtok_kl_causal_488_4chn",
+ help="VidTok model name for HuggingFace download. "
+ "Options: vidtok_kl_causal_488_4chn, vidtok_kl_noncausal_488_4chn, "
+ "vidtok_fsq_causal_488_4096, etc. (default: vidtok_kl_causal_488_4chn)"
+ )
+ parser.add_argument(
+ "--csv_path",
+ type=str,
+ required=True,
+ help="Path to OpenVid1M CSV file"
+ )
+ parser.add_argument(
+ "--video_root_dir",
+ type=str,
+ default=None,
+ help="Root directory for videos (auto-detected if not provided)"
+ )
+ parser.add_argument(
+ "--video_index",
+ type=int,
+ default=0,
+ help="Index of video to test (default: 0)"
+ )
+ parser.add_argument(
+ "--num_frames",
+ type=int,
+ default=16,
+ help="Number of frames (use 17 for causal models, 16 for non-causal)"
+ )
+ parser.add_argument(
+ "--height",
+ type=int,
+ default=256,
+ help="Video height"
+ )
+ parser.add_argument(
+ "--width",
+ type=int,
+ default=256,
+ help="Video width"
+ )
+ parser.add_argument(
+ "--output_dir",
+ type=str,
+ default="./vidtok_test_output",
+ help="Output directory for results"
+ )
+ parser.add_argument(
+ "--device",
+ type=str,
+ default="cuda" if torch.cuda.is_available() else "cpu",
+ help="Device to use"
+ )
+ parser.add_argument(
+ "--use_continuous",
+ action="store_true",
+ help="Use continuous latent space for decoding (default: use discrete tokens)"
+ )
+
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+
+ # Create output directory
+ os.makedirs(args.output_dir, exist_ok=True)
+
+ # Set device
+ device = torch.device(args.device)
+ print(f"Using device: {device}")
+
+ # Determine config and checkpoint paths
+ config_path = args.config
+ ckpt_path = args.ckpt
+
+ # If checkpoint is a HuggingFace model ID, download from HuggingFace
+ if ckpt_path is None or ckpt_path.startswith("microsoft/VidTok") or "/" in ckpt_path and not os.path.exists(ckpt_path):
+ print(f"Downloading VidTok model from HuggingFace...")
+ try:
+ from huggingface_hub import hf_hub_download, snapshot_download
+ import tempfile
+
+ # Determine model ID
+ if ckpt_path and ckpt_path.startswith("microsoft/VidTok"):
+ repo_id = ckpt_path
+ else:
+ repo_id = "microsoft/VidTok"
+
+ # Download checkpoint
+ checkpoint_filename = "vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt"#f"{args.model_name}.ckpt"
+ print(f"Downloading checkpoint: {checkpoint_filename}")
+
+ # Create temporary directory for downloads
+ cache_dir = os.path.join(os.getcwd(), "vidtok_cache")
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Download checkpoint
+ ckpt_path = hf_hub_download(
+ repo_id=repo_id,
+ filename=f"checkpoints/{checkpoint_filename}",
+ cache_dir=cache_dir,
+ local_dir=os.path.join(cache_dir, repo_id.replace("/", "_")),
+ )
+ print(f"Downloaded checkpoint to: {ckpt_path}")
+
+ # Download config if not provided
+ config_path = "/mnt/Meissonic/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml" #local path to config, no need to download
+
+ except ImportError:
+ print("Error: huggingface_hub not installed. Install with: pip install huggingface_hub")
+ sys.exit(1)
+ except Exception as e:
+ print(f"Error downloading from HuggingFace: {e}")
+ print("Please provide --config and --ckpt paths, or install VidTok repository.")
+ sys.exit(1)
+
+ # Load VidTok model
+ if not VIDTOK_AVAILABLE or load_model_from_config is None:
+ print("Error: VidTok scripts not available. Please install VidTok:")
+ print(" git clone https://github.com/microsoft/VidTok")
+ print(" export PYTHONPATH=\"${PYTHONPATH}:$(pwd)/VidTok\"")
+ print("\nOr ensure you have git installed for automatic download.")
+ sys.exit(1)
+
+ print(f"Loading VidTok model from config: {config_path}")
+ print(f"Checkpoint: {ckpt_path}")
+ if VIDTOK_PATH:
+ print(f"Using VidTok from: {VIDTOK_PATH}")
+ model = load_model_from_config(config_path, ckpt_path)
+ model = model.to(device).eval()
+ # model.encoder = torch.compile(model.encoder)
+ # model.decoder = torch.compile(model.decoder)
+
+
+ # Check if model is causal
+ is_causal = getattr(model, 'is_causal', False)
+ print(f"Model is causal: {is_causal}")
+ if is_causal and args.num_frames == 16:
+ print("Warning: Causal models typically use 17 frames. Consider using --num_frames 17")
+
+ # Load dataset
+ print(f"Loading dataset from: {args.csv_path}")
+
+ # Auto-detect video_root_dir if not provided
+ video_root_dir = args.video_root_dir
+ if video_root_dir is None:
+ csv_dir = os.path.dirname(args.csv_path)
+ if os.path.exists(os.path.join(csv_dir, 'video_reorg')):
+ video_root_dir = os.path.join(csv_dir, 'video_reorg')
+ elif os.path.exists(os.path.join(os.path.dirname(csv_dir), 'video_reorg')):
+ video_root_dir = os.path.join(os.path.dirname(csv_dir), 'video_reorg')
+ else:
+ video_root_dir = csv_dir
+ print(f"Warning: Video directory not found, using CSV directory: {video_root_dir}")
+
+ # Initialize tokenizer for dataset (needed for OpenVid1MDataset)
+ tokenizer = T5Tokenizer.from_pretrained("google/umt5-base")
+
+ # Create dataset
+ dataset = OpenVid1MDataset(
+ csv_path=args.csv_path,
+ video_root_dir=video_root_dir,
+ tokenizer=tokenizer,
+ num_frames=args.num_frames,
+ height=args.height,
+ width=args.width,
+ text_encoder_architecture="umt5-base",
+ )
+
+ print(f"Dataset size: {len(dataset)}")
+
+ # Load video
+ if args.video_index >= len(dataset):
+ print(f"Error: video_index {args.video_index} >= dataset size {len(dataset)}")
+ return
+
+ print(f"Loading video at index {args.video_index}...")
+ sample = dataset[args.video_index]
+ original_video = sample["video"] # [C, F, H, W] in [0, 1]
+
+ # Get video info from dataset
+ row = dataset.data[args.video_index]
+ video_path = row.get('video', 'unknown')
+ caption = row.get('caption', 'no caption')
+
+ print(f"Video path: {video_path}")
+ print(f"Caption: {caption}")
+ print(f"Original video shape: {original_video.shape}")
+ print(f"Original video range: [{original_video.min():.3f}, {original_video.max():.3f}]")
+
+ # Convert to VidTok format: [B, C, T, H, W] in [-1, 1]
+ # Original video is [C, F, H, W] in [0, 1]
+ original_video_vidtok = original_video.unsqueeze(0) # [1, C, F, H, W]
+ original_video_vidtok = original_video_vidtok.permute(0, 1, 2, 3, 4) # [1, C, F, H, W] -> [1, C, T, H, W]
+ # Convert from [0, 1] to [-1, 1]
+ original_video_vidtok = original_video_vidtok * 2.0 - 1.0
+ original_video_vidtok = original_video_vidtok.to(device=device)
+
+ print(f"VidTok input shape: {original_video_vidtok.shape}")
+ print(f"VidTok input range: [{original_video_vidtok.min():.3f}, {original_video_vidtok.max():.3f}]")
+
+ # Encode
+ print("\nEncoding video...")
+ with torch.no_grad(), torch.autocast(device_type='cuda' if device.type == 'cuda' else 'cpu', dtype=torch.float16 if device.type == 'cuda' else torch.float32):
+ if args.use_continuous:
+ # Encode to continuous latent space
+ z, reg_log = model.encode(original_video_vidtok, return_reg_log=True)
+ print(f"Continuous latent shape: {z.shape}")
+ print(f"Discrete tokens shape: {reg_log['indices'].shape if 'indices' in reg_log else 'N/A'}")
+ else:
+ # Full forward pass to get reconstruction
+ _, reconstructed_video_vidtok, _ = model(original_video_vidtok)
+
+ # Decode
+ if args.use_continuous:
+ print("\nDecoding from continuous latent space...")
+ with torch.no_grad(), torch.autocast(device_type='cuda' if device.type == 'cuda' else 'cpu', dtype=torch.float16 if device.type == 'cuda' else torch.float32):
+ reconstructed_video_vidtok = model.decode(z)
+ else:
+ print("\nUsing reconstruction from forward pass...")
+
+ # Convert back to [C, F, H, W] format and [0, 1] range
+ reconstructed_video_vidtok = reconstructed_video_vidtok.squeeze(0) # [C, T, H, W]
+ reconstructed_video_vidtok = reconstructed_video_vidtok.permute(0, 1, 2, 3) # [C, T, H, W] -> [C, F, H, W]
+ # Convert from [-1, 1] to [0, 1]
+ reconstructed_video = (reconstructed_video_vidtok + 1.0) / 2.0
+ reconstructed_video = torch.clamp(reconstructed_video, 0, 1)
+
+ print(f"Reconstructed video shape: {reconstructed_video.shape}")
+ print(f"Reconstructed video range: [{reconstructed_video.min():.3f}, {reconstructed_video.max():.3f}]")
+
+ # Ensure same number of frames for comparison
+ F_orig = original_video.shape[1]
+ F_recon = reconstructed_video.shape[1]
+ F_min = min(F_orig, F_recon)
+
+ original_video = original_video[:, :F_min, :, :]
+ reconstructed_video = reconstructed_video[:, :F_min, :, :]
+
+ # Resize if spatial dimensions don't match
+ if original_video.shape[2:] != reconstructed_video.shape[2:]:
+ print(f"Resizing reconstructed video from {reconstructed_video.shape[2:]} to {original_video.shape[2:]}")
+ # Use interpolation to resize
+ reconstructed_video_resized = torch.zeros_like(original_video)
+ for f in range(F_min):
+ frame = reconstructed_video[:, f, :, :].unsqueeze(0) # [1, C, H, W]
+ frame_resized = torch.nn.functional.interpolate(
+ frame, size=original_video.shape[2:], mode='bilinear', align_corners=False
+ )
+ reconstructed_video_resized[:, f, :, :] = frame_resized.squeeze(0)
+ reconstructed_video = reconstructed_video_resized
+
+ # Calculate metrics
+ print("\nCalculating metrics...")
+
+ # Convert to float32 for metric calculation
+ orig_f32 = original_video.to(torch.float32).cpu()
+ recon_f32 = reconstructed_video.to(torch.float32).cpu()
+
+ # Frame-wise metrics
+ psnr_values = []
+ mse_values = []
+ ssim_values = []
+
+ for f in range(F_min):
+ orig_frame = orig_f32[:, f, :, :] # [C, H, W]
+ recon_frame = recon_f32[:, f, :, :] # [C, H, W]
+
+ psnr = calculate_psnr(orig_frame, recon_frame)
+ mse = calculate_mse(orig_frame, recon_frame)
+ ssim = calculate_ssim(orig_frame, recon_frame)
+
+ psnr_values.append(psnr)
+ mse_values.append(mse)
+ ssim_values.append(ssim)
+
+ # Overall metrics
+ avg_psnr = np.mean(psnr_values)
+ avg_mse = np.mean(mse_values)
+ avg_ssim = np.mean(ssim_values)
+
+ print(f"\n=== Metrics ===")
+ print(f"PSNR: {avg_psnr:.2f} dB (per frame: {psnr_values})")
+ print(f"MSE: {avg_mse:.6f} (per frame: {mse_values})")
+ print(f"SSIM: {avg_ssim:.4f} (per frame: {ssim_values})")
+
+ # Save metrics to file
+ metrics_file = os.path.join(args.output_dir, f"metrics_video_{args.video_index}.txt")
+ with open(metrics_file, 'w') as f:
+ f.write(f"Video Index: {args.video_index}\n")
+ f.write(f"Video Path: {video_path}\n")
+ f.write(f"Caption: {caption}\n")
+ f.write(f"Model Config: {args.config}\n")
+ f.write(f"Model Checkpoint: {args.ckpt}\n")
+ f.write(f"Use Continuous Latent: {args.use_continuous}\n")
+ f.write(f"\n=== Metrics ===\n")
+ f.write(f"Average PSNR: {avg_psnr:.2f} dB\n")
+ f.write(f"Average MSE: {avg_mse:.6f}\n")
+ f.write(f"Average SSIM: {avg_ssim:.4f}\n")
+ f.write(f"\nPer-frame PSNR: {psnr_values}\n")
+ f.write(f"Per-frame MSE: {mse_values}\n")
+ f.write(f"Per-frame SSIM: {ssim_values}\n")
+
+ print(f"Saved metrics to: {metrics_file}")
+
+ # Create side-by-side video
+ print("\nCreating side-by-side comparison video...")
+ video_output_path = os.path.join(args.output_dir, f"comparison_video_{args.video_index}.mp4")
+ create_side_by_side_video(original_video, reconstructed_video, video_output_path, fps=8)
+
+ # Create comparison grid
+ print("Creating comparison grid...")
+ grid_output_path = os.path.join(args.output_dir, f"comparison_grid_video_{args.video_index}.png")
+ create_comparison_grid(original_video, reconstructed_video, grid_output_path, nrow=4)
+
+ print(f"\n=== Test Complete ===")
+ print(f"Results saved to: {args.output_dir}")
+ print(f" - Metrics: {metrics_file}")
+ print(f" - Side-by-side video: {video_output_path}")
+ print(f" - Comparison grid: {grid_output_path}")
+
+
+if __name__ == "__main__":
+ main()
+
diff --git a/Meissonic/VidTok/test_vidtok.sh b/Meissonic/VidTok/test_vidtok.sh
new file mode 100644
index 0000000000000000000000000000000000000000..1f05c470bb98dd86e3bc24e83ab65183e4ac2edd
--- /dev/null
+++ b/Meissonic/VidTok/test_vidtok.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Test script for VidTok tokenizer performance
+# VidTok models will be automatically downloaded from HuggingFace
+
+# Option 1: Use HuggingFace model ID (recommended - simplest)
+python test_vidtok.py \
+ --ckpt "microsoft/VidTok" \
+ --model_name "vidtok_kl_causal_488_16chn_v1_1" \
+ --csv_path "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv" \
+ --video_index 3 \
+ --num_frames 17 \
+ --height 240 \
+ --width 424 \
+ --output_dir "./vidtok_test_output" \
+ --device cuda
+
+# Option 2: Use local paths (if you have VidTok repository cloned)
+# First, clone VidTok: git clone https://github.com/microsoft/VidTok
+# Then add to PYTHONPATH:
+# export PYTHONPATH="${PYTHONPATH}:$(pwd)/VidTok"
+#
+# python train/test_vidtok.py \
+# --config "VidTok/configs/vidtok_kl_causal_488_4chn.yaml" \
+# --ckpt "VidTok/checkpoints/vidtok_kl_causal_488_4chn.ckpt" \
+# --csv_path "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv" \
+# --video_index 0 \
+# --num_frames 17 \
+# --height 256 \
+# --width 256 \
+# --output_dir "./vidtok_test_output" \
+# --device cuda
+
+# Available model names:
+# - vidtok_kl_causal_488_4chn (KL divergence, causal, 4 channels)
+# - vidtok_kl_noncausal_488_4chn (KL divergence, non-causal, 4 channels)
+# - vidtok_fsq_causal_488_4096 (FSQ, causal, codebook size 4096)
+# - vidtok_fsq_causal_488_262144 (FSQ, causal, codebook size 262144)
+# - vidtok_kl_causal_41616_4chn (KL divergence, causal, larger model)
+# See https://huggingface.co/microsoft/VidTok/tree/main/checkpoints for all models
+
diff --git a/Meissonic/VidTok/vidtok/data/__pycache__/video_read.cpython-310.pyc b/Meissonic/VidTok/vidtok/data/__pycache__/video_read.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..765ac789bf7eb005946e803ad7e074cb463e0e64
Binary files /dev/null and b/Meissonic/VidTok/vidtok/data/__pycache__/video_read.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc b/Meissonic/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8d55b5c7b63ae35cd09bf4f00a149d7e28fecb69
Binary files /dev/null and b/Meissonic/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/data/datamodule.py b/Meissonic/VidTok/vidtok/data/datamodule.py
new file mode 100644
index 0000000000000000000000000000000000000000..c405b84703735b9a076b9ffb4e74de673470c15b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/data/datamodule.py
@@ -0,0 +1,150 @@
+import numpy as np
+from functools import partial
+
+import torch
+import lightning.pytorch as pl
+from torch.utils.data import DataLoader, Dataset, IterableDataset
+
+from vidtok.modules.util import instantiate_from_config
+
+
+class WrappedDataset(Dataset):
+ """Wraps an arbitrary object with __len__ and __getitem__ into a pytorch dataset"""
+
+ def __init__(self, dataset):
+ self.data = dataset
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, idx):
+ return self.data[idx]
+
+
+def worker_init_fn(_):
+ worker_info = torch.utils.data.get_worker_info()
+
+ dataset = worker_info.dataset
+ worker_id = worker_info.id
+
+ if isinstance(dataset, IterableDataset):
+ split_size = dataset.num_records // worker_info.num_workers
+ # reset num_records to the true number to retain reliable length information
+ dataset.sample_ids = dataset.valid_ids[
+ worker_id * split_size : (worker_id + 1) * split_size
+ ]
+ current_id = np.random.choice(len(np.random.get_state()[1]), 1)
+ return np.random.seed(np.random.get_state()[1][current_id] + worker_id)
+ else:
+ return np.random.seed(np.random.get_state()[1][0] + worker_id)
+
+
+class DataModuleFromConfig(pl.LightningDataModule):
+ def __init__(
+ self,
+ batch_size,
+ train=None,
+ validation=None,
+ test=None,
+ predict=None,
+ wrap=False,
+ num_workers=None,
+ pin_train_memory=True,
+ is_iterable_dataset=False,
+ shuffle_test_loader=False,
+ use_worker_init_fn=False,
+ shuffle_val_dataloader=False,
+ ):
+ super().__init__()
+ self.batch_size = batch_size
+ self.dataset_configs = dict()
+ self.num_workers = num_workers if num_workers is not None else batch_size * 2
+ self.pin_train_memory = pin_train_memory
+ self.is_iterable_dataset = is_iterable_dataset
+ self.use_worker_init_fn = use_worker_init_fn
+ if train is not None:
+ self.dataset_configs["train"] = train
+ self.train_dataloader = self._train_dataloader
+ if validation is not None:
+ self.dataset_configs["validation"] = validation
+ self.val_dataloader = partial(
+ self._val_dataloader, shuffle=shuffle_val_dataloader
+ )
+ if test is not None:
+ self.dataset_configs["test"] = test
+ self.test_dataloader = partial(
+ self._test_dataloader, shuffle=shuffle_test_loader
+ )
+ if predict is not None:
+ self.dataset_configs["predict"] = predict
+ self.predict_dataloader = self._predict_dataloader
+ self.wrap = wrap
+
+ def prepare_data(self):
+ for data_cfg in self.dataset_configs.values():
+ instantiate_from_config(data_cfg)
+
+ def setup(self, stage=None):
+ self.datasets = dict(
+ (k, instantiate_from_config(self.dataset_configs[k]))
+ for k in self.dataset_configs
+ )
+ if self.wrap:
+ for k in self.datasets:
+ self.datasets[k] = WrappedDataset(self.datasets[k])
+
+ def _train_dataloader(self):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["train"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ pin_memory=self.pin_train_memory,
+ shuffle=False if self.is_iterable_dataset else True,
+ worker_init_fn=init_fn,
+ )
+
+ def _val_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["validation"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _test_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+
+ # do not shuffle dataloader for iterable dataset
+ shuffle = shuffle and (not self.is_iterable_dataset)
+
+ return DataLoader(
+ self.datasets["test"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _predict_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["predict"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ )
diff --git a/Meissonic/VidTok/vidtok/data/video_read.py b/Meissonic/VidTok/vidtok/data/video_read.py
new file mode 100644
index 0000000000000000000000000000000000000000..357cd48305141ee70924fd6a79adbe5cb7f6ca5b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/data/video_read.py
@@ -0,0 +1,88 @@
+import os
+import random
+import decord
+import numpy as np
+import torch
+
+from vidtok.modules.util import print0
+
+decord.bridge.set_bridge("torch")
+
+
+def sample_frames_with_fps(
+ total_frames,
+ video_fps,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+):
+ """sample frames proportional to the length of the frames in one second
+ e.g., 1s video has 30 frames, when 'fps'=3, we sample frames with spacing of 30/3=10
+ return the frame indices
+
+ Parameters
+ ----------
+ total_frames : length of the video
+ video_fps : original fps of the video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frame indices
+ """
+ sample_num_frames = min(sample_num_frames, total_frames)
+ interval = round(video_fps / sample_fps)
+ frames_range = (sample_num_frames - 1) * interval + 1
+
+ if start_index is not None:
+ start = start_index
+ elif total_frames - frames_range - 1 < 0:
+ start = 0
+ else:
+ start = random.randint(0, total_frames - frames_range - 1)
+
+ frame_idxs = np.linspace(
+ start=start, stop=min(total_frames - 1, start + frames_range), num=sample_num_frames
+ ).astype(int)
+
+ return frame_idxs
+
+
+def read_frames_with_decord(
+ video_path,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+) -> tuple[torch.Tensor, list[int]]:
+ """read frames from video path using decord
+
+ Parameters
+ ----------
+ video_path : path to video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frames (tensor 0~1), frame indices
+ """
+ video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(video_reader)
+ video_fps = video_reader.get_avg_fps() # note that the fps here is float.
+ frame_idxs = sample_frames_with_fps(
+ total_frames=total_frames,
+ video_fps=video_fps,
+ sample_num_frames=sample_num_frames,
+ sample_fps=sample_fps,
+ start_index=start_index
+ )
+ frames = video_reader.get_batch(frame_idxs)
+ frames = frames.float() / 255
+ frames = frames.permute(0, 3, 1, 2)
+ if (frames.shape[0] != sample_num_frames) or (len(frame_idxs) != sample_num_frames):
+ print0(f"[bold yellow]\[vidtok.data.video_read][read_frames_with_decord][/bold yellow] Warning: need {sample_num_frames} frames, "
+ f"but got {frames.shape[0]} frames, {len(frame_idxs)} frame indices, video_path={video_path}.")
+ return frames, frame_idxs
diff --git a/Meissonic/VidTok/vidtok/data/vidtok.py b/Meissonic/VidTok/vidtok/data/vidtok.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b1898d4718de1450da96f25e9134dd4e1084cc1
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/data/vidtok.py
@@ -0,0 +1,333 @@
+import os
+import glob
+from typing import Union
+
+import decord
+import numpy as np
+import pandas as pd
+import torch
+from PIL import Image
+from torch.utils.data import Dataset
+from torchvision.transforms import v2
+from tqdm import trange
+
+from vidtok.modules.util import print0
+from .video_read import read_frames_with_decord
+
+
+class VidTokDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ meta_path: str,
+ video_params: dict,
+ data_frac: float = 1.0,
+ is_strict_loading: bool = False,
+ skip_missing_files: bool = True,
+ start_index: Union[None, int] = None
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use data dir: {self.data_dir}")
+
+ self.meta_path = meta_path
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use meta path: {self.meta_path}")
+
+ self.video_params = video_params
+
+ self.data_frac = data_frac
+ self.is_strict_loading = is_strict_loading
+ self.skip_missing_files = skip_missing_files
+ self.start_index = start_index
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",)
+
+ if self.data_frac < 1:
+ metadata = metadata.sample(frac=self.data_frac)
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk
+ """
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.metadata)
+
+ def __getitem__(self, item):
+ item = item % len(self.metadata)
+ sample = self.metadata.iloc[item]
+ video_fp, _ = self._get_video_path(sample)
+
+ try:
+ if os.path.isfile(video_fp):
+ imgs, idxs = read_frames_with_decord(
+ video_path=video_fp,
+ sample_num_frames=self.video_params["sample_num_frames"],
+ sample_fps=self.video_params["sample_fps"],
+ start_index=self.start_index
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video.")
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}.") from e
+ else:
+ print0("[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: using the pure black image as the frame sample")
+ imgs = Image.new("RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0))
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ # imgs: (T, C, H, W)
+ imgs = self.transforms(imgs)
+
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ imgs = torch.cat([imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)], dim=0)
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ 'jpg': imgs,
+ "path": video_fp
+ }
+
+
+class VidTokValDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ video_params: dict,
+ meta_path: Union[None, str] = None,
+ pre_load_frames: bool = True,
+ is_strict_loading: bool = True,
+ last_frames_handle: str = "repeat", # 'repeat', 'drop'
+ skip_missing_files: bool = False,
+ read_long_video: bool = False,
+ chunk_size: int = 16,
+ is_causal: bool = True,
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use data dir: {self.data_dir}"
+ )
+
+ self.meta_path = meta_path
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use meta path: {self.meta_path}"
+ )
+
+ self.video_params = video_params
+ self.read_long_video = read_long_video
+ self.chunk_size = chunk_size
+ self.is_causal = is_causal
+
+ self.is_strict_loading = is_strict_loading
+ self.last_frames_handle = last_frames_handle
+ self.skip_missing_files = skip_missing_files
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+ self._load_every_frame_from_meta()
+
+ if pre_load_frames:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Pre-loading all frames into CPU..."
+ )
+ self._pre_load_frames()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ if self.meta_path is not None:
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",
+ )
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+ else:
+ self.metadata = glob.glob(os.path.join(self.data_dir, '**', '*.mp4'), recursive=True)
+
+ def _load_every_frame_from_meta(self):
+ decord.bridge.set_bridge("torch")
+ self.frames_batch = []
+ for video_idx in range(len(self.metadata)):
+ try:
+ sample = self.metadata.iloc[video_idx]
+ video_fp, _ = self._get_video_path(sample)
+ except:
+ video_fp = self.metadata[video_idx]
+ if os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ total_frames = len(video_reader)
+ fps = video_reader.get_avg_fps() # float
+ interval = round(fps / self.video_params["sample_fps"])
+ frame_ids = list(range(0, total_frames, interval))
+
+ if self.read_long_video:
+ video_length = len(frame_ids)
+ if self.is_causal and video_length > self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * ((video_length - 1) // self.chunk_size) + 1]
+ elif not self.is_causal and video_length >= self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * (video_length // self.chunk_size)]
+ else:
+ continue
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ else:
+ for x in range(0, len(frame_ids), self.video_params["sample_num_frames"]):
+ num_frames_ids = frame_ids[x : x + self.video_params["sample_num_frames"]]
+ if len(num_frames_ids) < self.video_params["sample_num_frames"]:
+ if self.last_frames_handle == "repeat":
+ num_frames_ids += [num_frames_ids[-1]] * (
+ self.video_params["sample_num_frames"] - len(num_frames_ids)
+ )
+ elif self.last_frames_handle == "drop":
+ continue
+ else:
+ raise ValueError(f"Invalid last_frames_handle: {self.last_frames_handle}")
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Loaded all frames index from {len(self.metadata)} videos."
+ )
+
+ def _pre_load_frames(self):
+ last_video_fp = None
+ for idx in trange(len(self.frames_batch), desc="Pre-loading all frames"):
+ if self.frames_batch[idx]["video_fp"] != last_video_fp:
+ video_reader = decord.VideoReader(self.frames_batch[idx]["video_fp"], num_threads=0)
+ last_video_fp = self.frames_batch[idx]["video_fp"]
+ self.frames_batch[idx]["frames"] = (
+ video_reader.get_batch(self.frames_batch[idx]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk"""
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.frames_batch)
+
+ def __getitem__(self, item):
+ video_fp = self.frames_batch[item]["video_fp"]
+
+ try:
+ if "frames" in self.frames_batch[item]:
+ imgs = self.frames_batch[item]["frames"]
+ elif os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ imgs = (
+ video_reader.get_batch(self.frames_batch[item]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video."
+ )
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(
+ f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}."
+ ) from e
+ else:
+ print0(
+ "[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: using the pure black image as the frame sample"
+ )
+ imgs = Image.new(
+ "RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0)
+ )
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ imgs = self.transforms(imgs)
+
+ if not self.read_long_video:
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: video {video_fp} has less frames {imgs.shape[0]} than sample_num_frames {self.video_params['sample_num_frames']}."
+ )
+ imgs = torch.cat(
+ [imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)],
+ dim=0,
+ )
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ "jpg": imgs,
+ "path": video_fp,
+ }
diff --git a/Meissonic/VidTok/vidtok/models/__pycache__/autoencoder_v1_1.cpython-310.pyc b/Meissonic/VidTok/vidtok/models/__pycache__/autoencoder_v1_1.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5011d2f6bc8b7a1917c4520856a886063a277cc5
Binary files /dev/null and b/Meissonic/VidTok/vidtok/models/__pycache__/autoencoder_v1_1.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/models/autoencoder.py b/Meissonic/VidTok/vidtok/models/autoencoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..96da5e6c74621b4f82538453e850567245adbbb0
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/models/autoencoder.py
@@ -0,0 +1,517 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+ self.use_tiling = False
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ token_indices = rearrange(token_indices, "... -> ... 1")
+ token_indices, ps = pack_one(token_indices, "b * d")
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = rearrange(codes, "b d n c -> b n (c d)")
+ z = self.regularization.project_out(codes)
+ z = unpack_one(z, ps, "b * d")
+ z = rearrange(z, "b ... d -> b d ...")
+ return z
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ z = self.indices_to_latent(z)
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ tile = z[
+ :,
+ :,
+ start_frame : end_frame,
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+
+ dec = self.decode(z)
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
diff --git a/Meissonic/VidTok/vidtok/models/autoencoder_v1_1.py b/Meissonic/VidTok/vidtok/models/autoencoder_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c1182f573eee05b634daeb9a065b7adaacc600b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/models/autoencoder_v1_1.py
@@ -0,0 +1,588 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ use_tiling: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ self.use_tiling = kwargs.pop("use_tiling", False)
+ self.t_chunk_enc = kwargs.pop("t_chunk_enc", 16)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+
+ self.t_chunk_dec = self.t_chunk_enc // self.encoder.time_downsample_factor
+ self.use_overlap = False
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+
+ self.use_tiling = use_tiling
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def _empty_causal_cached(self, parent):
+ for name, module in parent.named_modules():
+ if hasattr(module, 'causal_cache'):
+ module.causal_cache = None
+
+ def _set_first_chunk(self, is_first_chunk=True):
+ for module in self.modules():
+ if hasattr(module, 'is_first_chunk'):
+ module.is_first_chunk = is_first_chunk
+
+ def _set_cache_offset(self, modules, cache_offset=0):
+ for module in modules:
+ for submodule in module.modules():
+ if hasattr(submodule, 'cache_offset'):
+ submodule.cache_offset = cache_offset
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def build_chunk_start_end(self, t, decoder_mode=False):
+ start_end = [[0, 1]]
+ start = 1
+ end = start
+ while True:
+ if start >= t:
+ break
+ end = min(t, end + (self.t_chunk_dec if decoder_mode else self.t_chunk_enc))
+ start_end.append([start, end])
+ start = end
+ return start_end
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ self._empty_causal_cached(self.encoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = self.build_chunk_start_end(num_frames)
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ assert token_indices.dim() == 4, "token_indices should be of shape (b, t, h, w)"
+ b, t, h, w = token_indices.shape
+ token_indices = token_indices.unsqueeze(-1).reshape(b, -1, 1)
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = codes.permute(0, 2, 3, 1).reshape(b, codes.shape[2], -1)
+ z = self.regularization.project_out(codes)
+ return z.reshape(b, t, h, w, -1).permute(0, 4, 1, 2, 3)
+
+ def tile_indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ num_frames = token_indices.shape[1]
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ result_z = []
+ for (start, end) in start_end:
+ chunk = token_indices[:, start:end, :, :]
+ chunk_z = self.indices_to_latent(chunk)
+ result_z.append(chunk_z.clone())
+ return torch.cat(result_z, dim=2)
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ if self.use_tiling:
+ z = self.tile_indices_to_latent(z)
+ else:
+ z = self.indices_to_latent(z)
+ self._empty_causal_cached(self.decoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ if self.is_causal:
+ assert self.encoder.time_downsample_factor in [2, 4, 8], "Only support 2x, 4x or 8x temporal downsampling now."
+ if self.encoder.time_downsample_factor == 4:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 4)
+ elif self.encoder.time_downsample_factor == 2:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1], self.decoder.up_temporal[0], self.decoder.conv_out], 2)
+ else:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[3].upsample, self.decoder.up_temporal[2]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 4)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 8)
+
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = z[
+ :,
+ :,
+ start_frame : (end_frame + 1 if self.is_causal and end_frame + 1 <= num_frames else end_frame),
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+ dec = self.decode(z)
+ if dec.shape[2] != x.shape[2]:
+ dec = dec[:, :, -x.shape[2]:, ...]
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
\ No newline at end of file
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/discriminator.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/discriminator.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6f06e0f1b5bfbcb1546a046217de1e26d4fc105a
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/discriminator.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/distributions.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/distributions.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0392cf08e73bca7b4dc09f7b34b26940f154872c
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/distributions.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/ema.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/ema.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f4da146078e3dc25acecdca337e11233bb1d6c25
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/ema.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/losses.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/losses.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cff63e976a587d14dfdea56824f1b435d1b35e2f
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/losses.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/lpips.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/lpips.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e05c31a332e050545c4c764ca231ecd6dc0d608e
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/lpips.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/model_3dcausal_v1_1.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/model_3dcausal_v1_1.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..69191a078f4d9a177c49c72d203408df55d824d4
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/model_3dcausal_v1_1.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/regularizers.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/regularizers.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fe3c22a6e0b59f3770a016344abfa2f7f0214e77
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/regularizers.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc b/Meissonic/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..901aa877898a6ff71557c21b252b206b280e9154
Binary files /dev/null and b/Meissonic/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok/modules/discriminator.py b/Meissonic/VidTok/vidtok/modules/discriminator.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9d94b21b22f5019f3cdcc4cbf2e98bd0ce0ee02
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/discriminator.py
@@ -0,0 +1,201 @@
+import functools
+
+import torch
+import torch.nn as nn
+
+
+def weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ nn.init.normal_(m.weight.data, 0.0, 0.02)
+ elif classname.find("BatchNorm") != -1:
+ nn.init.normal_(m.weight.data, 1.0, 0.02)
+ nn.init.constant_(m.bias.data, 0)
+
+
+class ActNorm(nn.Module):
+ def __init__(self, num_features, logdet=False, affine=True, allow_reverse_init=False):
+ assert affine
+ super().__init__()
+ self.logdet = logdet
+ self.loc = nn.Parameter(torch.zeros(1, num_features, 1, 1))
+ self.scale = nn.Parameter(torch.ones(1, num_features, 1, 1))
+ self.allow_reverse_init = allow_reverse_init
+
+ self.register_buffer("initialized", torch.tensor(0, dtype=torch.uint8))
+
+ def initialize(self, input):
+ with torch.no_grad():
+ flatten = input.permute(1, 0, 2, 3).contiguous().view(input.shape[1], -1)
+ mean = flatten.mean(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+ std = flatten.std(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+
+ self.loc.data.copy_(-mean)
+ self.scale.data.copy_(1 / (std + 1e-6))
+
+ def forward(self, input, reverse=False):
+ if reverse:
+ return self.reverse(input)
+ if len(input.shape) == 2:
+ input = input[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ _, _, height, width = input.shape
+
+ if self.training and self.initialized.item() == 0:
+ self.initialize(input)
+ self.initialized.fill_(1)
+
+ h = self.scale * (input + self.loc)
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+
+ if self.logdet:
+ log_abs = torch.log(torch.abs(self.scale))
+ logdet = height * width * torch.sum(log_abs)
+ logdet = logdet * torch.ones(input.shape[0]).to(input)
+ return h, logdet
+
+ return h
+
+ def reverse(self, output):
+ if self.training and self.initialized.item() == 0:
+ if not self.allow_reverse_init:
+ raise RuntimeError(
+ "Initializing ActNorm in reverse direction is "
+ "disabled by default. Use allow_reverse_init=True to enable."
+ )
+ else:
+ self.initialize(output)
+ self.initialized.fill_(1)
+
+ if len(output.shape) == 2:
+ output = output[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ h = output / self.scale - self.loc
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+ return h
+
+
+class NLayerDiscriminator(nn.Module):
+ """Defines a PatchGAN discriminator as in Pix2Pix."""
+ # https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py
+ def __init__(self, input_nc=3, ndf=64, n_layers=3, use_actnorm=False):
+ """Construct a PatchGAN discriminator
+ Parameters:
+ input_nc (int) -- the number of channels in input images
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ """
+ super(NLayerDiscriminator, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm2d
+ else:
+ norm_layer = ActNorm
+ if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters
+ use_bias = norm_layer.func != nn.BatchNorm2d
+ else:
+ use_bias = norm_layer != nn.BatchNorm2d
+
+ kw = 4
+ padw = 1
+ sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
+
+
+class NLayerDiscriminator3D(nn.Module):
+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/losses/discriminator.py
+ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
+ """
+ Construct a 3D PatchGAN discriminator
+
+ Parameters:
+ input_nc (int) -- the number of channels in input volumes
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm
+ """
+ super(NLayerDiscriminator3D, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm3d
+ else:
+ raise NotImplementedError("Not implemented.")
+ if type(norm_layer) == functools.partial:
+ use_bias = norm_layer.func != nn.BatchNorm3d
+ else:
+ use_bias = norm_layer != nn.BatchNorm3d
+
+ kw = 3
+ padw = 1
+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev,
+ ndf * nf_mult,
+ kernel_size=(kw, kw, kw),
+ stride=(2 if n == 1 else 1, 2, 2),
+ padding=padw,
+ bias=use_bias,
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
diff --git a/Meissonic/VidTok/vidtok/modules/distributions.py b/Meissonic/VidTok/vidtok/modules/distributions.py
new file mode 100644
index 0000000000000000000000000000000000000000..76e814475d4d32b9f5ead736cce3a234bb5a0e5f
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/distributions.py
@@ -0,0 +1,49 @@
+import numpy as np
+import torch
+
+
+class DiagonalGaussianDistribution(object):
+ def __init__(self, parameters, deterministic=False):
+ self.parameters = parameters
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
+ self.deterministic = deterministic
+ self.std = torch.exp(0.5 * self.logvar)
+ self.var = torch.exp(self.logvar)
+ if self.deterministic:
+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device)
+
+ def sample(self):
+ x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device)
+ return x
+
+ def kl(self, other=None):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ else:
+ if other is None:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar,
+ dim=[1, 2, 3],
+ )
+ else:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean - other.mean, 2) / other.var
+ + self.var / other.var
+ - 1.0
+ - self.logvar
+ + other.logvar,
+ dim=[1, 2, 3],
+ )
+
+ def nll(self, sample, dims=[1, 2, 3]):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ logtwopi = np.log(2.0 * np.pi)
+ return 0.5 * torch.sum(
+ logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
+ dim=dims,
+ )
+
+ def mode(self):
+ return self.mean
diff --git a/Meissonic/VidTok/vidtok/modules/ema.py b/Meissonic/VidTok/vidtok/modules/ema.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f1f7606c2c9b68ebd2302215a9e08f9f31ed8ab
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/ema.py
@@ -0,0 +1,82 @@
+import torch
+from torch import nn
+
+
+class LitEma(nn.Module):
+ def __init__(self, model, decay=0.9999, use_num_upates=True):
+ super().__init__()
+ if decay < 0.0 or decay > 1.0:
+ raise ValueError("Decay must be between 0 and 1")
+
+ self.m_name2s_name = {}
+ self.register_buffer("decay", torch.tensor(decay, dtype=torch.float32))
+ self.register_buffer(
+ "num_updates",
+ torch.tensor(0, dtype=torch.int) if use_num_upates else torch.tensor(-1, dtype=torch.int),
+ )
+
+ for name, p in model.named_parameters():
+ if p.requires_grad:
+ # remove as '.'-character is not allowed in buffers
+ s_name = name.replace(".", "")
+ self.m_name2s_name.update({name: s_name})
+ self.register_buffer(s_name, p.clone().detach().data)
+
+ self.collected_params = []
+
+ def reset_num_updates(self):
+ del self.num_updates
+ self.register_buffer("num_updates", torch.tensor(0, dtype=torch.int))
+
+ def forward(self, model):
+ decay = self.decay
+
+ if self.num_updates >= 0:
+ self.num_updates += 1
+ decay = min(self.decay, (1 + self.num_updates) / (10 + self.num_updates))
+
+ one_minus_decay = 1.0 - decay
+
+ with torch.no_grad():
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+
+ for key in m_param:
+ if m_param[key].requires_grad:
+ sname = self.m_name2s_name[key]
+ shadow_params[sname] = shadow_params[sname].type_as(m_param[key])
+ shadow_params[sname].sub_(one_minus_decay * (shadow_params[sname] - m_param[key]))
+ else:
+ assert not key in self.m_name2s_name
+
+ def copy_to(self, model):
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+ for key in m_param:
+ if m_param[key].requires_grad:
+ m_param[key].data.copy_(shadow_params[self.m_name2s_name[key]].data)
+ else:
+ assert not key in self.m_name2s_name
+
+ def store(self, parameters):
+ """
+ Save the current parameters for restoring later.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ temporarily stored.
+ """
+ self.collected_params = [param.clone() for param in parameters]
+
+ def restore(self, parameters):
+ """
+ Restore the parameters stored with the `store` method.
+ Useful to validate the model with EMA parameters without affecting the
+ original optimization process. Store the parameters before the
+ `copy_to` method. After validation (or model saving), use this to
+ restore the former parameters.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ updated with the stored parameters.
+ """
+ for c_param, param in zip(self.collected_params, parameters):
+ param.data.copy_(c_param.data)
diff --git a/Meissonic/VidTok/vidtok/modules/logger.py b/Meissonic/VidTok/vidtok/modules/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..23c7d066e6508433cb2141c59dbd80cb0030ab6d
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/logger.py
@@ -0,0 +1,289 @@
+import os
+import numpy as np
+import einops
+import imageio
+from typing import Union
+from matplotlib import pyplot as plt
+from PIL import Image, ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True # UnidentifiedImageError: https://github.com/python-pillow/Pillow/issues/5631
+from pathlib import Path
+
+import torch
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from .util import exists, isheatmap
+
+
+class ImageVideoLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ batch_frequency_val=None,
+ video_fps=8,
+ rescale=True,
+ disabled=False,
+ log_on_batch_idx=True, # log on batch_idx instead of global_step. global_step is fixed in validation. batch_idx restarts at each validation
+ log_first_step=True,
+ log_images_kwargs=None,
+ log_videos_kwargs=None,
+ log_before_first_step=True,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.batch_freq_val = batch_frequency_val if batch_frequency_val is not None else batch_frequency
+ self.video_fps = video_fps
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_videos_kwargs = log_videos_kwargs if log_videos_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_img_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ else:
+ if images[k].ndim == 5:
+ images[k] = einops.rearrange(images[k], "b c t h w -> (b t) c h w")
+ nrow = self.log_images_kwargs.get("n_rows", 8)
+ grid = torchvision.utils.make_grid(images[k], nrow=nrow)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_vid_local(
+ self,
+ save_dir,
+ split,
+ videos,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "videos", split)
+ for k in videos:
+ # if is video, we can add captions
+ if isinstance(videos[k], torch.Tensor) and videos[k].ndim == 5:
+ if self.rescale:
+ videos[k] = (videos[k] + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ frames = [videos[k][:, :, i] for i in range(videos[k].shape[2])]
+ frames = [torchvision.utils.make_grid(each, nrow=4) for each in frames]
+ frames = [einops.rearrange(each, "c h w -> 1 c h w") for each in frames]
+ frames = torch.clamp(torch.cat(frames, dim=0), min=0.0, max=1.0)
+ frames = (frames.numpy() * 255).astype(np.uint8)
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.gif".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ save_numpy_as_gif(frames, path, duration=1 / self.video_fps)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "log_videos only supports WandbLogger currently"
+ wandb.log({f"{split}/{k}": wandb.Video(frames, fps=self.video_fps)}) # k is str
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ images = pl_module.log_images(batch)
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_img_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ @rank_zero_only
+ def log_vid(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_videos") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_videos)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ videos = pl_module.log_videos(
+ batch, split=split, **self.log_videos_kwargs
+ )
+
+ for k in videos:
+ N = min(videos[k].shape[0], self.max_samples)
+ videos[k] = videos[k][:N]
+ if isinstance(videos[k], torch.Tensor):
+ videos[k] = videos[k].detach().float().cpu()
+ if self.clamp:
+ videos[k] = torch.clamp(videos[k], -1.0, 1.0)
+
+ self.log_vid_local(
+ pl_module.logger.save_dir,
+ split,
+ videos,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ pass
+ return True
+ return False
+
+ def check_frequency_val(self, check_idx, split):
+ if 'val' in split:
+ if ((check_idx % self.batch_freq_val) == 0) and (
+ check_idx > 0 or self.log_first_step):
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ self.log_vid(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+def save_numpy_as_gif(frames, path, duration=None):
+ """
+ save numpy array as gif file
+ """
+ image_list = []
+ for frame in frames:
+ image = frame.transpose(1, 2, 0)
+ image_list.append(image)
+ if duration:
+ imageio.mimsave(path, image_list, format="GIF", duration=duration, loop=0)
+ else:
+ imageio.mimsave(path, image_list, format="GIF", loop=0)
diff --git a/Meissonic/VidTok/vidtok/modules/losses.py b/Meissonic/VidTok/vidtok/modules/losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..c96f471c72124c56025b40a17f7d7bda81446ca5
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/losses.py
@@ -0,0 +1,262 @@
+from typing import Any, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .discriminator import (NLayerDiscriminator, NLayerDiscriminator3D,
+ weights_init)
+from .lpips import LPIPS
+from .util import default, print0
+
+
+def hinge_d_loss(logits_real, logits_fake):
+ loss_real = torch.mean(F.relu(1.0 - logits_real))
+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))
+ d_loss = 0.5 * (loss_real + loss_fake)
+ return d_loss
+
+
+def vanilla_d_loss(logits_real, logits_fake):
+ d_loss = 0.5 * (torch.mean(F.softplus(-logits_real)) + torch.mean(F.softplus(logits_fake)))
+ return d_loss
+
+
+def adopt_weight(weight, global_step, threshold=0, value=0.0):
+ if global_step < threshold:
+ weight = value
+ return weight
+
+
+def _sigmoid_cross_entropy_with_logits(labels, logits):
+ """
+ non-saturating loss
+ """
+ zeros = torch.zeros_like(logits, dtype=logits.dtype)
+ condition = logits >= zeros
+ relu_logits = torch.where(condition, logits, zeros)
+ neg_abs_logits = torch.where(condition, -logits, logits)
+ return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
+
+
+def non_saturate_gen_loss(logits_fake):
+ """
+ logits_fake: [B 1 H W]
+ """
+ B = logits_fake.shape[0]
+ logits_fake = logits_fake.reshape(B, -1)
+ logits_fake = torch.mean(logits_fake, dim=-1)
+ gen_loss = torch.mean(_sigmoid_cross_entropy_with_logits(labels=torch.ones_like(logits_fake), logits=logits_fake))
+ return gen_loss
+
+
+def lecam_reg(real_pred, fake_pred, lecam_ema):
+ reg = torch.mean(F.relu(real_pred - lecam_ema.logits_fake_ema).pow(2)) + torch.mean(
+ F.relu(lecam_ema.logits_real_ema - fake_pred).pow(2)
+ )
+ return reg
+
+
+class LeCAM_EMA(object):
+ # https://github.com/TencentARC/SEED-Voken/blob/main/src/Open_MAGVIT2/modules/losses/vqperceptual.py
+ def __init__(self, init=0.0, decay=0.999):
+ self.logits_real_ema = init
+ self.logits_fake_ema = init
+ self.decay = decay
+
+ def update(self, logits_real, logits_fake):
+ self.logits_real_ema = self.logits_real_ema * self.decay + torch.mean(logits_real).item() * (1 - self.decay)
+ self.logits_fake_ema = self.logits_fake_ema * self.decay + torch.mean(logits_fake).item() * (1 - self.decay)
+
+
+class GeneralLPIPSWithDiscriminator(nn.Module):
+ def __init__(
+ self,
+ disc_start: int,
+ logvar_init: float = 0.0,
+ pixelloss_weight=1.0,
+ disc_num_layers: int = 3,
+ disc_in_channels: int = 3,
+ disc_factor: float = 1.0,
+ disc_weight: float = 1.0,
+ disc_type: str = "3d",
+ perceptual_weight: float = 1.0,
+ lecam_loss_weight: float = 0.0,
+ disc_loss: str = "hinge",
+ scale_input_to_tgt_size: bool = False,
+ dims: int = 2,
+ learn_logvar: bool = False,
+ regularization_weights: Union[None, dict] = None,
+ gen_loss_cross_entropy: bool = False,
+ ):
+ super().__init__()
+ self.dims = dims
+ if self.dims > 2:
+ print0(
+ f"[bold cyan]\[vidtok.modules.losses][GeneralLPIPSWithDiscriminator][/bold cyan] running with dims={dims}. This means that for perceptual loss calculation, "
+ f"the LPIPS loss will be applied to each frame independently. "
+ )
+ self.scale_input_to_tgt_size = scale_input_to_tgt_size
+ assert disc_loss in ["hinge", "vanilla"]
+ self.pixel_weight = pixelloss_weight
+ self.perceptual_loss = LPIPS().eval()
+ self.perceptual_weight = perceptual_weight
+ # output log variance
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+ self.learn_logvar = learn_logvar
+ self.disc_type = disc_type
+ assert self.disc_type in ["2d", "3d"]
+
+ if self.disc_type == "2d":
+ self.discriminator = NLayerDiscriminator(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ else:
+ self.discriminator = NLayerDiscriminator3D(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ self.discriminator_iter_start = disc_start
+ self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss
+ self.disc_factor = disc_factor
+ self.discriminator_weight = disc_weight
+ self.regularization_weights = default(regularization_weights, {})
+ self.gen_loss_cross_entropy = gen_loss_cross_entropy
+ self.lecam_loss_weight = lecam_loss_weight
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema = LeCAM_EMA()
+
+ def get_trainable_parameters(self) -> Any:
+ return self.discriminator.parameters()
+
+ def get_trainable_autoencoder_parameters(self) -> Any:
+ if self.learn_logvar:
+ yield self.logvar
+ yield from ()
+
+ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):
+ if last_layer is not None:
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
+ else:
+ nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0]
+
+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
+ d_weight = d_weight * self.discriminator_weight
+ return d_weight
+
+ def forward(
+ self,
+ regularization_log,
+ inputs,
+ reconstructions,
+ optimizer_idx,
+ global_step,
+ last_layer=None,
+ split="train",
+ weights=None,
+ ):
+ if self.scale_input_to_tgt_size:
+ inputs = torch.nn.functional.interpolate(inputs, reconstructions.shape[2:], mode="bicubic", antialias=True)
+
+ if optimizer_idx == 0:
+ bs = inputs.shape[0]
+ t = inputs.shape[2]
+ if self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
+ if self.perceptual_weight > 0:
+ p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())
+ rec_loss = rec_loss + self.perceptual_weight * p_loss
+ else:
+ p_loss = torch.Tensor([0.0])
+
+ nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if weights is not None:
+ weighted_nll_loss = weights * nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+
+ # now the GAN part
+ if self.disc_type == "3d":
+ reconstructions = rearrange(reconstructions, "(b t) c h w -> b c t h w", t=t).contiguous()
+
+ # generator update
+ logits_fake = self.discriminator(reconstructions)
+
+ if not self.gen_loss_cross_entropy:
+ g_loss = -torch.mean(logits_fake)
+ else:
+ g_loss = non_saturate_gen_loss(logits_fake)
+
+ if self.disc_factor > 0.0:
+ try:
+ d_weight = self.calculate_adaptive_weight(nll_loss, g_loss, last_layer=last_layer)
+ except RuntimeError:
+ assert not self.training
+ d_weight = torch.tensor(0.0)
+ else:
+ d_weight = torch.tensor(0.0)
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+ loss = weighted_nll_loss + d_weight * disc_factor * g_loss
+ log = dict()
+ for k in regularization_log:
+ if k in self.regularization_weights:
+ loss = loss + self.regularization_weights[k] * regularization_log[k]
+ log[f"{split}/{k}"] = regularization_log[k].detach().mean()
+
+ log.update(
+ {
+ "{}/total_loss".format(split): loss.clone().detach().mean(),
+ "{}/logvar".format(split): self.logvar.detach(),
+ "{}/nll_loss".format(split): nll_loss.detach().mean(),
+ "{}/rec_loss".format(split): rec_loss.detach().mean(),
+ "{}/p_loss".format(split): p_loss.detach().mean(),
+ "{}/d_weight".format(split): d_weight.detach(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/g_loss".format(split): g_loss.detach().mean(),
+ }
+ )
+ return loss, log
+
+ if optimizer_idx == 1:
+ if self.disc_type == "2d" and self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ logits_real = self.discriminator(inputs.contiguous().detach())
+ logits_fake = self.discriminator(reconstructions.contiguous().detach())
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+
+ non_saturate_d_loss = self.disc_loss(logits_real, logits_fake)
+
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema.update(logits_real, logits_fake)
+ lecam_loss = lecam_reg(logits_real, logits_fake, self.lecam_ema)
+ d_loss = disc_factor * (lecam_loss * self.lecam_loss_weight + non_saturate_d_loss)
+ else:
+ d_loss = disc_factor * non_saturate_d_loss
+
+ log = {
+ "{}/disc_loss".format(split): d_loss.clone().detach().mean(),
+ "{}/logits_real".format(split): logits_real.detach().mean(),
+ "{}/logits_fake".format(split): logits_fake.detach().mean(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/non_saturated_d_loss".format(split): non_saturate_d_loss.detach(),
+ }
+
+ if self.lecam_loss_weight > 0:
+ log.update({"{}/lecam_loss".format(split): lecam_loss.detach()})
+
+ return d_loss, log
diff --git a/Meissonic/VidTok/vidtok/modules/lpips.py b/Meissonic/VidTok/vidtok/modules/lpips.py
new file mode 100644
index 0000000000000000000000000000000000000000..22bb5fa315618c0d1f0463d67ea57c29083fe302
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/lpips.py
@@ -0,0 +1,172 @@
+import hashlib
+import os
+from collections import namedtuple
+from tqdm import tqdm
+
+import requests
+import torch
+import torch.nn as nn
+from torchvision import models
+
+from .util import print0
+
+URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"}
+
+CKPT_MAP = {"vgg_lpips": "vgg.pth"}
+
+MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"}
+
+
+def download(url, local_path, chunk_size=1024):
+ os.makedirs(os.path.split(local_path)[0], exist_ok=True)
+ with requests.get(url, stream=True) as r:
+ total_size = int(r.headers.get("content-length", 0))
+ with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
+ with open(local_path, "wb") as f:
+ for data in r.iter_content(chunk_size=chunk_size):
+ if data:
+ f.write(data)
+ pbar.update(chunk_size)
+
+
+def md5_hash(path):
+ with open(path, "rb") as f:
+ content = f.read()
+ return hashlib.md5(content).hexdigest()
+
+
+def get_ckpt_path(name, root, check=False):
+ assert name in URL_MAP
+ path = os.path.join(root, CKPT_MAP[name])
+ if os.path.exists(path) and not (check and not md5_hash(path) == MD5_MAP[name]):
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Using existing path for {} model: {}".format(
+ name, path
+ )
+ )
+ return path
+
+ # if not, download the model
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Downloading {} model from {} to {}".format(
+ name, URL_MAP[name], path
+ )
+ )
+ download(URL_MAP[name], path)
+ md5 = md5_hash(path)
+ assert md5 == MD5_MAP[name], md5
+ return path
+
+
+class LPIPS(nn.Module):
+ # Learned perceptual metric
+ def __init__(self, use_dropout=True):
+ super().__init__()
+ self.scaling_layer = ScalingLayer()
+ self.chns = [64, 128, 256, 512, 512] # vg16 features
+ self.net = vgg16(pretrained=True, requires_grad=False)
+ self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
+ self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
+ self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
+ self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)
+ self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)
+ self.load_from_pretrained()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def load_from_pretrained(self, name="vgg_lpips"):
+ ckpt = get_ckpt_path(name, "checkpoints/lpips")
+ self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
+ print0("[bold cyan]\[vidtok.modules.lpips][LPIPS][/bold cyan] loaded pretrained LPIPS loss from {}".format(ckpt))
+
+ def forward(self, input, target):
+ in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))
+ outs0, outs1 = self.net(in0_input), self.net(in1_input)
+ feats0, feats1, diffs = {}, {}, {}
+ lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
+ for kk in range(len(self.chns)):
+ feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])
+ diffs[kk] = (feats0[kk] - feats1[kk]) ** 2
+
+ res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]
+ val = res[0]
+ for l in range(1, len(self.chns)):
+ val += res[l]
+ return val
+
+
+class ScalingLayer(nn.Module):
+ def __init__(self):
+ super(ScalingLayer, self).__init__()
+ self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None])
+ self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None])
+
+ def forward(self, inp):
+ return (inp - self.shift) / self.scale
+
+
+class NetLinLayer(nn.Module):
+ """A single linear layer which does a 1x1 conv"""
+
+ def __init__(self, chn_in, chn_out=1, use_dropout=False):
+ super(NetLinLayer, self).__init__()
+ layers = (
+ [
+ nn.Dropout(),
+ ]
+ if (use_dropout)
+ else []
+ )
+ layers += [
+ nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False),
+ ]
+ self.model = nn.Sequential(*layers)
+
+
+class vgg16(torch.nn.Module):
+ def __init__(self, requires_grad=False, pretrained=True):
+ super(vgg16, self).__init__()
+ vgg_pretrained_features = models.vgg16(pretrained=pretrained).features
+ self.slice1 = torch.nn.Sequential()
+ self.slice2 = torch.nn.Sequential()
+ self.slice3 = torch.nn.Sequential()
+ self.slice4 = torch.nn.Sequential()
+ self.slice5 = torch.nn.Sequential()
+ self.N_slices = 5
+ for x in range(4):
+ self.slice1.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(4, 9):
+ self.slice2.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(9, 16):
+ self.slice3.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(16, 23):
+ self.slice4.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(23, 30):
+ self.slice5.add_module(str(x), vgg_pretrained_features[x])
+ if not requires_grad:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, X):
+ h = self.slice1(X)
+ h_relu1_2 = h
+ h = self.slice2(h)
+ h_relu2_2 = h
+ h = self.slice3(h)
+ h_relu3_3 = h
+ h = self.slice4(h)
+ h_relu4_3 = h
+ h = self.slice5(h)
+ h_relu5_3 = h
+ vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"])
+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
+ return out
+
+
+def normalize_tensor(x, eps=1e-10):
+ norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True))
+ return x / (norm_factor + eps)
+
+
+def spatial_average(x, keepdim=True):
+ return x.mean([2, 3], keepdim=keepdim)
diff --git a/Meissonic/VidTok/vidtok/modules/model_3dcausal.py b/Meissonic/VidTok/vidtok/modules/model_3dcausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..c71a8b09e370451046eef9ba60315feab2459e35
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/model_3dcausal.py
@@ -0,0 +1,885 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+ self.time_causal_padding = (self.time_pad, 0)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (height_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ time_pad,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+ x1 = self.avg_pool(torch.nn.functional.pad(x, pad, mode="constant", value=0))
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest").to(
+ x.dtype
+ )
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ x = pad_at_dim(x, (self.time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in)
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x[:, :, self.time_padding :, :, :]
diff --git a/Meissonic/VidTok/vidtok/modules/model_3dcausal_v1_1.py b/Meissonic/VidTok/vidtok/modules/model_3dcausal_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..44397a135334a58cf5774fb899152f701d56a37d
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/model_3dcausal_v1_1.py
@@ -0,0 +1,959 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1].repeat(
+ (1, 1, self.time_pad)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (width_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.spatial_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ 0,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1, :, :].repeat(
+ (1, 1, self.time_pad, 1, 1)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ x = F.pad(x, self.spatial_padding, mode=self.pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+
+ if self.is_first_chunk:
+ x_pad = torch.nn.functional.pad(x, pad, mode="replicate")
+ else:
+ x_pad = torch.concatenate((self.causal_cache, x), dim=2)
+
+ self.causal_cache = x_pad[:,:,-1:].clone()
+
+ x1 = self.avg_pool(x_pad)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ interpolation_mode='nearest',
+ num_temp_upsample=1
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.interpolation_mode = interpolation_mode
+ self.num_temp_upsample = num_temp_upsample
+ self.enable_cached = (self.interpolation_mode == 'trilinear')
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ if not self.enable_cached:
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ elif not self.is_first_chunk:
+ x = torch.cat([self.causal_cache, x], dim=2)
+ self.causal_cache = x[:, :, -2*self.num_temp_upsample:-self.num_temp_upsample].clone()
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ x = x[:, :, 2*self.num_temp_upsample:]
+ else:
+ self.causal_cache = x[:, :, -self.num_temp_upsample:].clone()
+ x, _x = x[:, :, :self.num_temp_upsample], x[:, :, self.num_temp_upsample:]
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ if _x.shape[-3] > 0:
+ _x = F.interpolate(_x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(_x.dtype)
+ x = torch.concat([x, _x], dim=2)
+
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ time_padding = self.time_downsample_factor - video_len % self.time_downsample_factor
+ x = pad_at_dim(x, (time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.interpolation_mode = ignorekwargs.get("interpolation_mode", 'nearest')
+ assert self.interpolation_mode in ['nearest', 'trilinear']
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ num_temp_upsample = 1
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in, interpolation_mode=self.interpolation_mode, num_temp_upsample=num_temp_upsample)
+ num_temp_upsample *= 2
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x
diff --git a/Meissonic/VidTok/vidtok/modules/model_3dnoncausal.py b/Meissonic/VidTok/vidtok/modules/model_3dnoncausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..4223fb635eb88cd7e7292943c131965f2b814206
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/model_3dnoncausal.py
@@ -0,0 +1,652 @@
+from typing import Callable
+
+import einops
+import torch
+import torch.nn as nn
+from einops import rearrange
+
+from .model_3dcausal import (AttnBlock, Normalize, nonlinearity,
+ spatial_temporal_resblk)
+from .util import checkpoint
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, stride=(2, 1, 1), padding=(0, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x1 = self.avg_pool(x)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, padding=1)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ xlst = [
+ torch.nn.functional.interpolate(
+ sx.unsqueeze(0).to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest"
+ ).to(x.dtype)
+ for sx in x
+ ]
+ x = torch.cat(xlst, dim=0)
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ if zero_init:
+ self.conv2.weight.data.zero_()
+ self.conv2.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetNoncausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=1)
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class Encoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch=8,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3]
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) # add for spatial tiling
+ self.norm_type = norm_type
+ self.is_causal = False
+
+ # downsampling
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1, padding=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level != self.num_resolutions - 1:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleRes2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn(block_in, norm_type=self.norm_type)
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ )
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, _, _ = x.shape
+
+ # downsampling
+ if x.shape[1] == 4 and self.conv_in.in_channels == 3:
+ raise ValueError("Mismatched number of input channels")
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+ if i_level != self.num_resolutions - 1:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_ds:
+ # temporal downsample
+ htmp = self.down_temporal[i_level].downsample(htmp)
+ hs.append(htmp)
+ B, _, T, _, _ = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h)
+ return h
+
+
+class Decoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels=8,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.tempo_us = [1, 2]
+ self.norm_type = norm_type
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level != 0:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleRes2x(block_in, block_in)
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1, padding=1)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def get_last_layer(self, **kwargs):
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, _, _ = z.shape
+
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+ if i_level != 0:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_us:
+ # temporal upsample
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, _, _ = h.shape
+ # end
+ if self.give_pre_end:
+ return h
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+ if self.tanh_out:
+ h = torch.tanh(h)
+ return h
diff --git a/Meissonic/VidTok/vidtok/modules/regularizers.py b/Meissonic/VidTok/vidtok/modules/regularizers.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f4f1a4fccdb873f640e637ecbaaaef389e75b41
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/regularizers.py
@@ -0,0 +1,268 @@
+from abc import abstractmethod
+from functools import cache
+from typing import Any, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import pack, rearrange, reduce, unpack
+from torch import Tensor, int32
+from torch.cuda.amp import autocast
+
+from .distributions import DiagonalGaussianDistribution
+
+
+def exists(v):
+ return v is not None
+
+
+def default(*args):
+ for arg in args:
+ if exists(arg):
+ return arg
+ return None
+
+
+def pack_one(t, pattern):
+ return pack([t], pattern)
+
+
+def unpack_one(t, ps, pattern):
+ return unpack(t, ps, pattern)[0]
+
+
+def round_ste(z: Tensor) -> Tensor:
+ """Round with straight through gradients."""
+ zhat = z.round()
+ return z + (zhat - z).detach()
+
+
+def log(t, eps=1e-5):
+ return t.clamp(min=eps).log()
+
+
+def entropy(prob):
+ return (-prob * log(prob)).sum(dim=-1)
+
+
+def maybe_distributed_mean(t):
+ if not is_distributed():
+ return t
+ dist.all_reduce(t)
+ t = t / dist.get_world_size()
+ return t
+
+
+@cache
+def is_distributed():
+ return dist.is_initialized() and dist.get_world_size() > 1
+
+
+class AbstractRegularizer(nn.Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, z: torch.Tensor) -> Tuple[torch.Tensor, dict]:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_trainable_parameters(self) -> Any:
+ raise NotImplementedError()
+
+
+class DiagonalGaussianRegularizer(AbstractRegularizer):
+ def __init__(self, sample: bool = True):
+ super().__init__()
+ self.sample = sample
+
+ def get_trainable_parameters(self) -> Any:
+ yield from ()
+
+ def forward(self, z: torch.Tensor, n_steps=None) -> Tuple[torch.Tensor, dict]:
+ log = dict()
+ posterior = DiagonalGaussianDistribution(z)
+ if self.sample:
+ z = posterior.sample()
+ else:
+ z = posterior.mode()
+ kl_loss = posterior.kl()
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ log["kl_loss"] = kl_loss
+ return z, log
+
+
+class FSQRegularizer(AbstractRegularizer):
+ # https://github.com/lucidrains/vector-quantize-pytorch/blob/master/vector_quantize_pytorch/finite_scalar_quantization.py
+ def __init__(
+ self,
+ levels: List[int],
+ dim: Optional[int] = None,
+ num_codebooks=1,
+ keep_num_codebooks_dim: Optional[bool] = None,
+ scale: Optional[float] = None,
+ entropy_loss_weight: float = 0.0,
+ entropy_loss_annealing_steps: int = 0,
+ entropy_loss_annealing_factor: float = 1.0,
+ commitment_loss_weight: float = 0.0,
+ diversity_gamma: float = 1.0,
+ ):
+ super().__init__()
+ _levels = torch.tensor(levels, dtype=int32)
+ self.register_buffer("_levels", _levels, persistent=False)
+
+ _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=int32)
+ self.register_buffer("_basis", _basis, persistent=False)
+
+ self.scale = scale
+ self.entropy_loss_weight = entropy_loss_weight
+ self.entropy_loss_annealing_steps = entropy_loss_annealing_steps
+ self.entropy_loss_annealing_factor = entropy_loss_annealing_factor
+ self.commitment_loss_weight = commitment_loss_weight
+ self.diversity_gamma = diversity_gamma
+
+ codebook_dim = len(levels)
+ self.codebook_dim = codebook_dim
+
+ effective_codebook_dim = codebook_dim * num_codebooks
+ self.num_codebooks = num_codebooks
+ self.effective_codebook_dim = effective_codebook_dim
+
+ keep_num_codebooks_dim = default(keep_num_codebooks_dim, num_codebooks > 1)
+ assert not (num_codebooks > 1 and not keep_num_codebooks_dim)
+ self.keep_num_codebooks_dim = keep_num_codebooks_dim
+
+ self.dim = default(dim, len(_levels) * num_codebooks)
+
+ has_projections = self.dim != effective_codebook_dim
+ self.project_in = nn.Linear(self.dim, effective_codebook_dim) if has_projections else nn.Identity()
+ self.project_out = nn.Linear(effective_codebook_dim, self.dim) if has_projections else nn.Identity()
+ self.has_projections = has_projections
+
+ self.codebook_size = self._levels.prod().item()
+
+ implicit_codebook = self.indices_to_codes(torch.arange(self.codebook_size), project_out=False)
+ self.register_buffer("implicit_codebook", implicit_codebook, persistent=False)
+ self.register_buffer("zero", torch.tensor(0.0), persistent=False)
+
+ self.global_codebook_usage = torch.zeros([2**self.codebook_dim, self.num_codebooks], dtype=torch.long)
+
+ def get_trainable_parameters(self) -> Any:
+ return self.parameters()
+
+ def bound(self, z: Tensor, eps: float = 1e-3) -> Tensor:
+ """Bound `z`, an array of shape (..., d)."""
+ half_l = (self._levels - 1) * (1 + eps) / 2
+ offset = torch.where(self._levels % 2 == 0, 0.5, 0.0)
+ shift = (offset / half_l).atanh()
+ return (z + shift).tanh() * half_l - offset
+
+ def quantize(self, z: Tensor) -> Tensor:
+ """Quantizes z, returns quantized zhat, same shape as z."""
+ quantized = round_ste(self.bound(z))
+ half_width = self._levels // 2
+ return quantized / half_width
+
+ def _scale_and_shift(self, zhat_normalized: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat_normalized * half_width) + half_width
+
+ def _scale_and_shift_inverse(self, zhat: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat - half_width) / half_width
+
+ def codes_to_indices(self, zhat: Tensor) -> Tensor:
+ """Converts a `code` to an index in the codebook."""
+ assert zhat.shape[-1] == self.codebook_dim
+ zhat = self._scale_and_shift(zhat)
+ return (zhat * self._basis).sum(dim=-1).to(int32)
+
+ def indices_to_codes(self, indices: Tensor, project_out=True) -> Tensor:
+ """Inverse of `codes_to_indices`."""
+
+ is_img_or_video = indices.ndim >= (3 + int(self.keep_num_codebooks_dim))
+
+ indices = rearrange(indices, "... -> ... 1")
+ codes_non_centered = (indices // self._basis) % self._levels
+ codes = self._scale_and_shift_inverse(codes_non_centered)
+
+ if self.keep_num_codebooks_dim:
+ codes = rearrange(codes, "... c d -> ... (c d)")
+
+ if project_out:
+ codes = self.project_out(codes)
+
+ if is_img_or_video:
+ codes = rearrange(codes, "b ... d -> b d ...")
+
+ return codes
+
+ def calculate_entropy_loss_weight(self, n_steps):
+ if n_steps >= self.entropy_loss_annealing_steps:
+ return self.entropy_loss_weight
+ start = self.entropy_loss_annealing_factor * self.entropy_loss_weight
+ return start - (n_steps / self.entropy_loss_annealing_steps) * (start - self.entropy_loss_weight)
+
+ @autocast(enabled=False)
+ def forward(self, z: Tensor, inv_temperature: float = 100.0, n_steps: int = 0) -> Tensor:
+ """
+ einstein notation
+ b - batch
+ n - sequence (or flattened spatial dimensions)
+ d - feature dimension
+ c - number of codebook dim
+ """
+ is_img_or_video = z.ndim >= 4
+ if is_img_or_video:
+ z = rearrange(z, "b d ... -> b ... d")
+ z, ps = pack_one(z, "b * d")
+
+ assert z.shape[-1] == self.dim, f"expected dimension of {self.dim} but found dimension of {z.shape[-1]}"
+
+ z = self.project_in(z)
+ z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks)
+
+ with torch.autocast("cuda", enabled=False):
+ orig_dtype = z.dtype
+ z = z.float()
+ original_input = z
+ codes = self.quantize(z)
+ indices = self.codes_to_indices(codes)
+
+ if self.entropy_loss_weight > 0 or self.commitment_loss_weight > 0:
+ # the same as euclidean distance up to a constant
+ distance = -2 * torch.einsum("... i d, j d -> ... i j", original_input, self.implicit_codebook)
+ prob = (-distance * inv_temperature).softmax(dim=-1)
+ per_sample_probs = rearrange(prob, "b n ... -> (b n) ...")
+ per_sample_entropy = entropy(per_sample_probs).mean()
+ # distribution over all available tokens in the batch
+ avg_prob = reduce(per_sample_probs, "... c d -> c d", "mean")
+ avg_prob = maybe_distributed_mean(avg_prob)
+ codebook_entropy = entropy(avg_prob).mean()
+ entropy_aux_loss = per_sample_entropy - self.diversity_gamma * codebook_entropy
+ # commit loss
+ commit_loss = F.mse_loss(original_input, codes.detach(), reduction="none")
+ commit_loss = commit_loss.mean()
+ else:
+ entropy_aux_loss = per_sample_entropy = codebook_entropy = commit_loss = self.zero
+
+ codes = codes.type(orig_dtype)
+
+ codes = rearrange(codes, "b n c d -> b n (c d)")
+ out = self.project_out(codes)
+
+ # reconstitute image or video dimensions
+ if is_img_or_video:
+ out = unpack_one(out, ps, "b * d")
+ out = rearrange(out, "b ... d -> b d ...")
+
+ indices = unpack_one(indices, ps, "b * c")
+
+ if not self.keep_num_codebooks_dim:
+ indices = rearrange(indices, "... 1 -> ...")
+
+ aux_loss = (
+ entropy_aux_loss * self.calculate_entropy_loss_weight(n_steps) + commit_loss * self.commitment_loss_weight
+ )
+
+ return out, dict(indices=indices, aux_loss=aux_loss)
diff --git a/Meissonic/VidTok/vidtok/modules/util.py b/Meissonic/VidTok/vidtok/modules/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..9570016221e47b50d536d30352b5718a29bb0009
--- /dev/null
+++ b/Meissonic/VidTok/vidtok/modules/util.py
@@ -0,0 +1,324 @@
+import importlib
+import random
+import os
+import einops
+import numpy as np
+from inspect import isfunction
+from rich import print
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+
+def get_valid_dirs(dir1: str, dir2: str, dir3: Union[None, str] = None) -> Union[None, str]:
+ if (dir1 is not None) and os.path.isdir(dir1):
+ return dir1
+ elif (dir2 is not None) and os.path.isdir(dir2):
+ return dir2
+ elif (dir3 is not None) and os.path.isdir(dir3):
+ return dir3
+ else:
+ return None
+
+
+def get_valid_paths(path1: str, path2: str, path3: Union[None, str] = None) -> Union[None, str]:
+ if (path1 is not None) and os.path.isfile(path1):
+ return path1
+ elif (path2 is not None) and os.path.isfile(path2):
+ return path2
+ elif (path3 is not None) and os.path.isfile(path3):
+ return path3
+ else:
+ return None
+
+
+@rank_zero_only
+def print0(*args, **kwargs):
+ print(*args, **kwargs)
+
+
+def seed_anything(seed: int):
+ os.environ['PYTHONHASHSEED'] = str(seed)
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+
+
+def isheatmap(x):
+ if not isinstance(x, torch.Tensor):
+ return False
+
+ return x.ndim == 2
+
+
+def exists(x):
+ return x is not None
+
+
+def default(val, d):
+ if exists(val):
+ return val
+ return d() if isfunction(d) else d
+
+
+def instantiate_from_config(config):
+ if not "target" in config:
+ if config == "__is_first_stage__":
+ return None
+ elif config == "__is_unconditional__":
+ return None
+ raise KeyError("Expected key `target` to instantiate.")
+ return get_obj_from_str(config["target"])(**config.get("params", dict()))
+
+
+def get_obj_from_str(string, reload=False, invalidate_cache=True):
+ module, cls = string.rsplit(".", 1)
+ if invalidate_cache:
+ importlib.invalidate_caches()
+ if reload:
+ module_imp = importlib.import_module(module)
+ importlib.reload(module_imp)
+ return getattr(importlib.import_module(module, package=None), cls)
+
+
+def checkpoint(func, inputs, params, flag):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ """
+ Evaluate a function without caching intermediate activations, allowing for
+ reduced memory at the expense of extra compute in the backward pass.
+ :param func: the function to evaluate.
+ :param inputs: the argument sequence to pass to `func`.
+ :param params: a sequence of parameters `func` depends on but does not
+ explicitly take as arguments.
+ :param flag: if False, disable gradient checkpointing.
+ """
+ if flag:
+ args = tuple(inputs) + tuple(params)
+ return CheckpointFunction.apply(func, len(inputs), *args)
+ else:
+ return func(*inputs)
+
+
+class CheckpointFunction(torch.autograd.Function):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ @staticmethod
+ def forward(ctx, run_function, length, *args):
+ ctx.run_function = run_function
+ ctx.input_tensors = list(args[:length])
+ ctx.input_params = list(args[length:])
+ ctx.gpu_autocast_kwargs = {
+ "enabled": torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad():
+ output_tensors = ctx.run_function(*ctx.input_tensors)
+ return output_tensors
+
+ @staticmethod
+ def backward(ctx, *output_grads):
+ ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
+ # Ensure all tensors have requires_grad set to True
+ ctx.input_params = [p.requires_grad_(True) for p in ctx.input_params]
+ with torch.enable_grad(), torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs):
+ # Fixes a bug where the first op in run_function modifies the
+ # Tensor storage in place, which is not allowed for detach()'d
+ # Tensors.
+ shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
+ output_tensors = ctx.run_function(*shallow_copies)
+ input_grads = torch.autograd.grad(
+ output_tensors,
+ ctx.input_tensors + ctx.input_params,
+ output_grads,
+ allow_unused=True,
+ )
+ del ctx.input_tensors
+ del ctx.input_params
+ del output_tensors
+ return (None, None) + input_grads
+
+
+def compute_psnr(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ EPS = 1e-8
+ mse = torch.mean((x - y) ** 2, dim=[1, 2, 3])
+ psnr = -10 * torch.log10(mse + EPS)
+ return psnr.mean(dim=0)
+
+
+def compute_ssim(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ kernel_size = 11
+ kernel_sigma = 1.5
+ k1 = 0.01
+ k2 = 0.03
+
+ f = max(1, round(min(x.size()[-2:]) / 256))
+ if f > 1:
+ x = F.avg_pool2d(x, kernel_size=f)
+ y = F.avg_pool2d(y, kernel_size=f)
+
+ kernel = gaussian_filter(kernel_size, kernel_sigma, device=x.device, dtype=x.dtype).repeat(x.size(1), 1, 1, 1)
+
+ _compute_ssim_per_channel = _ssim_per_channel_complex if x.dim() == 5 else _ssim_per_channel
+ ssim_map, cs_map = _compute_ssim_per_channel(x=x, y=y, kernel=kernel, data_range=1, k1=k1, k2=k2)
+ ssim_val = ssim_map.mean(1)
+
+ return ssim_val.mean(dim=0)
+
+
+def _ssim_per_channel(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W)`.
+ y: A target tensor. Shape :math:`(N, C, H, W)`.
+ kernel: 2D Gaussian kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Structural Similarity (SSIM) index.
+ """
+ if x.size(-1) < kernel.size(-1) or x.size(-2) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. "
+ f"Input size: {x.size()}. Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+ n_channels = x.size(1)
+ mu_x = F.conv2d(x, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu_y = F.conv2d(y, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu_xx = mu_x**2
+ mu_yy = mu_y**2
+ mu_xy = mu_x * mu_y
+
+ sigma_xx = F.conv2d(x**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xx
+ sigma_yy = F.conv2d(y**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_yy
+ sigma_xy = F.conv2d(x * y, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xy
+
+ # Contrast sensitivity (CS) with alpha = beta = gamma = 1.
+ cs = (2.0 * sigma_xy + c2) / (sigma_xx + sigma_yy + c2)
+
+ # Structural similarity (SSIM)
+ ss = (2.0 * mu_xy + c1) / (mu_xx + mu_yy + c1) * cs
+
+ ssim_val = ss.mean(dim=(-1, -2))
+ cs = cs.mean(dim=(-1, -2))
+ return ssim_val, cs
+
+
+def _ssim_per_channel_complex(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for Complex X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W, 2)`.
+ y: A target tensor. Shape :math:`(N, C, H, W, 2)`.
+ kernel: 2-D gauss kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Complex Structural Similarity (SSIM) index.
+ """
+ n_channels = x.size(1)
+ if x.size(-2) < kernel.size(-1) or x.size(-3) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. Input size: {x.size()}. "
+ f"Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+
+ x_real = x[..., 0]
+ x_imag = x[..., 1]
+ y_real = y[..., 0]
+ y_imag = y[..., 1]
+
+ mu1_real = F.conv2d(x_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu1_imag = F.conv2d(x_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_real = F.conv2d(y_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_imag = F.conv2d(y_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu1_sq = mu1_real.pow(2) + mu1_imag.pow(2)
+ mu2_sq = mu2_real.pow(2) + mu2_imag.pow(2)
+ mu1_mu2_real = mu1_real * mu2_real - mu1_imag * mu2_imag
+ mu1_mu2_imag = mu1_real * mu2_imag + mu1_imag * mu2_real
+
+ compensation = 1.0
+
+ x_sq = x_real.pow(2) + x_imag.pow(2)
+ y_sq = y_real.pow(2) + y_imag.pow(2)
+ x_y_real = x_real * y_real - x_imag * y_imag
+ x_y_imag = x_real * y_imag + x_imag * y_real
+
+ sigma1_sq = F.conv2d(x_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_sq
+ sigma2_sq = F.conv2d(y_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu2_sq
+ sigma12_real = F.conv2d(x_y_real, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_real
+ sigma12_imag = F.conv2d(x_y_imag, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_imag
+ sigma12 = torch.stack((sigma12_imag, sigma12_real), dim=-1)
+ mu1_mu2 = torch.stack((mu1_mu2_real, mu1_mu2_imag), dim=-1)
+ # Set alpha = beta = gamma = 1.
+ cs_map = (sigma12 * 2 + c2 * compensation) / (sigma1_sq.unsqueeze(-1) + sigma2_sq.unsqueeze(-1) + c2 * compensation)
+ ssim_map = (mu1_mu2 * 2 + c1 * compensation) / (mu1_sq.unsqueeze(-1) + mu2_sq.unsqueeze(-1) + c1 * compensation)
+ ssim_map = ssim_map * cs_map
+
+ ssim_val = ssim_map.mean(dim=(-2, -3))
+ cs = cs_map.mean(dim=(-2, -3))
+
+ return ssim_val, cs
+
+
+def gaussian_filter(
+ kernel_size: int, sigma: float, device: Optional[str] = None, dtype: Optional[type] = None
+) -> torch.Tensor:
+ r"""Returns 2D Gaussian kernel N(0,`sigma`^2)
+ Args:
+ kernel_size: Size of the kernel
+ sigma: Std of the distribution
+ device: target device for kernel generation
+ dtype: target data type for kernel generation
+ Returns:
+ gaussian_kernel: Tensor with shape (1, kernel_size, kernel_size)
+ """
+ coords = torch.arange(kernel_size, dtype=dtype, device=device)
+ coords -= (kernel_size - 1) / 2.0
+
+ g = coords**2
+ g = (-(g.unsqueeze(0) + g.unsqueeze(1)) / (2 * sigma**2)).exp()
+
+ g /= g.sum()
+ return g.unsqueeze(0)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/.gitignore b/Meissonic/VidTok/vidtok_cache/VidTok/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..81849eac5c147c73d4202463658bd4b49571d163
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/.gitignore
@@ -0,0 +1,167 @@
+amlt
+.amltconfig
+checkpoints
+logs
+wandb
+tmp
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
\ No newline at end of file
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/CODE_OF_CONDUCT.md b/Meissonic/VidTok/vidtok_cache/VidTok/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000000000000000000000000000000000000..f9ba8cf65f3e3104dd061c178066ec8247811f33
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/CODE_OF_CONDUCT.md
@@ -0,0 +1,9 @@
+# Microsoft Open Source Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+
+Resources:
+
+- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
+- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
+- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/LICENSE b/Meissonic/VidTok/vidtok_cache/VidTok/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..9e841e7a26e4eb057b24511e7b92d42b257a80e5
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/LICENSE
@@ -0,0 +1,21 @@
+ MIT License
+
+ Copyright (c) Microsoft Corporation.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/README.md b/Meissonic/VidTok/vidtok_cache/VidTok/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..edd9df5e2319e6dab35604d0d67192381dfacabf
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/README.md
@@ -0,0 +1,472 @@
+
+
+# VidTok
A Family of Versatile and State-Of-The-Art Video Tokenizers
+
+[](https://arxiv.org/pdf/2412.13061) [](https://github.com/microsoft/VidTok) [](https://huggingface.co/microsoft/VidTok)
+
+
+---
+
+
+
+ We introduce VidTok, a cutting-edge family of video tokenizers that excels in both continuous and discrete tokenizations. VidTok incorporates several key advancements over existing approaches:
+ * ⚡️ **Efficient Architecture**. Separate spatial and temporal sampling reduces computational complexity without sacrificing quality.
+ * 🔥 **Advanced Quantization**. Finite Scalar Quantization (FSQ) addresses training instability and codebook collapse in discrete tokenization.
+ * 💥 **Enhanced Training**. A two-stage strategy—pre-training on low-res videos and fine-tuning on high-res—boosts efficiency. Reduced frame rates improve motion dynamics representation.
+
+VidTok, trained on a large-scale video dataset, outperforms previous models across all metrics, including PSNR, SSIM, LPIPS, and FVD.
+
+https://github.com/user-attachments/assets/a3341037-130d-4a83-aba6-c3daeaf66932
+
+## 🔥 News
+- August, 2025: 🚀 Introduced spatial tiling for large resolutions (>256), reducing GPU memory usage to ~6 GB when encoding and decoding a 17 × 768 × 768 video.
+* March, 2025: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) has been accepted by CVPR 2025, and the [checkpoint](https://huggingface.co/microsoft/vidtwin) was released!
+* March, 2025: 🚀 [VidTok v1.1](#-updates-in-vidtok-v11) was released! We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. [Relevant checkpoints](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints/vidtok_v1_1) are continuously updating.
+* December, 2024: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) was released!
+* December, 2024: 🚀 [VidTok](https://github.com/microsoft/vidtok) was released!
+
+
+## 💥 Updates in VidTok v1.1
+> VidTok v1.1 is an update for causal models. We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. See performance [here](#v11-performance).
+
+### v1.1: Long Video Reconstruction
+Run the following inference script to [reconstruct an input video](#reconstruct-an-input-video):
+```bash
+python scripts/inference_reconstruct.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --output_video_dir OUTPUT_DIR --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+and run the following inference script to [evaluate the reconstruction performance](#performance-evaluation):
+```bash
+python scripts/inference_evaluate.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+
+For an easy usage of VidTok v1.1 models, refer to [this script](#easy-usage) and make the following revision:
+```python
+# Use VidTok v1.1 models
+cfg_path = "configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml"
+ckpt_path = "checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt"
+
+...
+
+model.to('cuda').eval()
+# Using tiling inference to save memory usage
+model.use_tiling = True
+model.t_chunk_enc = 16
+model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+model.use_overlap = True
+# random input: long video
+x_input = (torch.rand(1, 3, 129, 256, 256) * 2 - 1).to('cuda')
+
+...
+
+if x_recon.shape[2] != x_input.shape[2]:
+ x_recon = x_recon[:, :, -x_input.shape[2]:, ...]
+```
+
+### v1.1: Long Video Fine-tuning
+Follow this [training guidance](#fine-tune-on-custom-data) to fine-tune on your custom long video data and note that:
+- Compared to VidTok v1.0, we tend to use longer sequences to fine-tune the model (for example, setting `NUM_FRAMES_1` to 33, 49, or larger).
+- The resolution and the sequence length of training data should be adjusted according to GPU memory.
+
+### v1.1: Performance
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.13 | 0.941 | 0.049 | 87.4 |
+| [vidtok_kl_causal_41616_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x16x16 | 29.61 | 0.854 | 0.113 | 162.7 |
+| [vidtok_kl_causal_288_8chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt) | KL-8chn | ✔️ | 2x8x8 | 34.59 | 0.935 | 0.051 | 78.2 |
+| [vidtok_fsq_causal_488_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.39 | 0.856 | 0.114 | 168.5 |
+| [vidtok_fsq_causal_888_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 8x8x8 | 27.95 | 0.817 | 0.142 | 293.2 |
+- This is the evaluation result of long video reconstruction conducted on each complete video in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30 and a resolution of `256x256`.
+
+
+## 🔧 Setup
+1. Clone this repository and navigate to VidTok folder:
+```bash
+git clone https://github.com/microsoft/VidTok
+cd VidTok
+```
+2. We provide an `environment.yaml` file for setting up a Conda environment. Conda's installation instructions are available [here](https://docs.anaconda.com/miniconda/index.html).
+```bash
+# 1. Prepare conda environment
+conda env create -f environment.yaml
+# 2. Activate the environment
+conda activate vidtok
+```
+
+We recommend using 1+ high-end GPU for training and inference. We have done all testing and development using A100 and MI300X GPUs. For convenience, we also provide prebuilt [Docker](https://hub.docker.com/) images with required dependencies. You can use it as follows:
+
+```bash
+# NVIDIA GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-cuda12.1-python3.10-pytorch2.5:orig-vidtok bash
+# AMD GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-rocm6.2.4-python3.10-pytorch2.5:orig-vidtok bash
+```
+
+## 🎈 Checkpoints
+Download pre-trained models [here](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints), and put them in `checkpoints` folder, like:
+```
+└── checkpoints
+ ├── vidtok_v1_1
+ │ ├── vidtok_kl_causal_488_16chn_v1_1.ckpt
+ │ └── ...
+ ├── vidtok_fsq_causal_41616_262144.ckpt
+ ├── vidtok_fsq_causal_488_262144.ckpt
+ ├── vidtok_fsq_causal_488_32768.ckpt
+ ├── vidtok_fsq_causal_488_4096.ckpt
+ ├── vidtok_fsq_noncausal_41616_262144.ckpt
+ ├── vidtok_fsq_noncausal_488_262144.ckpt
+ ├── vidtok_kl_causal_288_8chn.ckpt
+ ├── vidtok_kl_causal_41616_4chn.ckpt
+ ├── vidtok_kl_causal_444_4chn.ckpt
+ ├── vidtok_kl_causal_488_16chn.ckpt
+ ├── vidtok_kl_causal_488_4chn.ckpt
+ ├── vidtok_kl_causal_488_8chn.ckpt
+ ├── vidtok_kl_noncausal_41616_16chn.ckpt
+ ├── vidtok_kl_noncausal_41616_4chn.ckpt
+ ├── vidtok_kl_noncausal_488_16chn.ckpt
+ └── vidtok_kl_noncausal_488_4chn.ckpt
+```
+Each checkpoint has a corresponding config file with the same name in `configs` folder.
+
+
+## 🔆 Performance
+
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_4chn.ckpt) | KL-4chn | ✔️ | 4x8x8 | 29.64 | 0.852| 0.114| 194.2|
+| [vidtok_kl_causal_488_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_8chn.ckpt) | KL-8chn | ✔️ |4x8x8 | 31.83 | 0.897| 0.083| 109.3|
+| [vidtok_kl_causal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_16chn.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.04 |0.942 |0.047 | 78.9|
+| [vidtok_kl_causal_288_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_288_8chn.ckpt) | KL-8chn | ✔️ | 2x8x8 | 33.86 | 0.928 |0.057 | 80.7 |
+| [vidtok_kl_causal_444_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_444_4chn.ckpt) | KL-4chn | ✔️ | 4x4x4 | 34.78 | 0.941 | 0.051| 87.2|
+| [vidtok_kl_causal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_41616_4chn.ckpt) | KL-4chn | ✔️ | 4x16x16 | 25.05 | 0.711| 0.228| 549.1|
+| [vidtok_kl_noncausal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_4chn.ckpt) | KL-4chn | ✖️ | 4x8x8 | 30.60 | 0.876 | 0.098| 157.9|
+| [vidtok_kl_noncausal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_16chn.ckpt) | KL-16chn | ✖️ | 4x8x8 | 36.13 | 0.950 | 0.044| 60.5|
+| [vidtok_kl_noncausal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt) | KL-4chn | ✖️ | 4x16x16 | 26.06 | 0.751 | 0.190|423.2 |
+| [vidtok_kl_noncausal_41616_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt) | KL-16chn | ✖️ | 4x16x16 | 30.69 | 0.878 | 0.095| 147.1|
+| [vidtok_fsq_causal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_262144.ckpt) | FSQ-262,144 | ✔️ | 4x8x8 | 29.82 | 0.867 |0.106 | 160.1|
+| [vidtok_fsq_causal_488_32768](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_32768.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.16 | 0.854 | 0.117| 196.9|
+| [vidtok_fsq_causal_488_4096](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_4096.ckpt) | FSQ-4096 | ✔️ | 4x8x8 | 28.36 | 0.832 | 0.133| 218.1|
+| [vidtok_fsq_causal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_41616_262144.ckpt) | FSQ-262,144 | ✔️ | 4x16x16 | 25.38 | 0.738 |0.206 | 430.1|
+| [vidtok_fsq_noncausal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_488_262144.ckpt) | FSQ-262,144 | ✖️ | 4x8x8 | 30.78 | 0.889| 0.091| 132.1|
+| [vidtok_fsq_noncausal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt) | FSQ-262,144 | ✖️ | 4x16x16 | 26.37 | 0.772| 0.171| 357.0|
+
+- `VCR` indicates the video compression ratio `TxHxW`.
+- The above table shows model performance evaluated on 30 test videos in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30. The input size is `17x256x256` for causal models and `16x256x256` for non-causal models.
+
+## 🔛 Training
+
+### Data Preparation
+1. Put all training videos under `DATA_DIR`:
+```
+└── DATA_DIR
+ ├── subset1
+ │ ├── videoname11.mp4
+ │ └── videoname12.mp4
+ ├── subset2
+ │ ├── videoname21.mp4
+ │ ├── videoname22.mp4
+ │ └── subsubset1
+ │ ├── videoname211.mp4
+ │ └── videoname212.mp4
+ └── ...
+```
+2. Prepare a `.csv` meta file to record the relative paths of these videos with respect to `DATA_DIR`, like:
+```
+videos
+subset1/videoname11.mp4
+subset2/videoname21.mp4
+subset2/subsubset1/videoname211.mp4
+```
+
+> Validation data is also prepared following the above steps.
+
+### Fine-tune on Custom Data
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and modify the following parameters:
+ - Specify the `ckpt_path` parameter to initialize the model with pre-trained checkpoint parameters:
+ ```yaml
+ model:
+ params:
+ ckpt_path: PATH_TO_CHECKPOINT # train from existing checkpoint
+ ```
+ - Specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+ - Set `fix_encoder` and `fix_decoder` to be `False` to enable full model fine-tuning:
+ ```yaml
+ model:
+ params:
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+ ```
+ - Other hyperparameters according to your needs.
+
+3. Run the following command to start training:
+```bash
+python main.py -b CONFIG --logdir LOGDIR
+
+# You can also use `torchrun` to start the training code.
+```
+Training logs and checkpoints are saved in `LOGDIR`.
+
+It is recommended to use [Weights & Biases](https://wandb.ai/site) as the data visualization tool ([TensorBoard](https://www.tensorflow.org/tensorboard) by default). Use `wandb login` to log in first, and then run:
+```bash
+python main.py -b CONFIG --logdir LOGDIR --wandb --wandb_entity ENTITY --wandb_project PROJECT
+```
+
+
+### Train from Scratch
+
+Two-stage Training
+We adopt a two-stage training strategy to improve training efficiency: initially pre-training the full model on low-resolution videos, followed by fine-tuning only the decoder on high-resolution videos.
+
+| First Stage | Second Stage | Fix encoder | PSNR | SSIM | LPIPS | GPU Hours|
+|------|------|------|------|------|------|------|
+| 256 x 256 | - | - | 29.19 | 0.843 | 0.127| 3,072|
+| 128 x 128 | 256 x 256 | ✔️ | 29.21 | 0.843 | 0.125| 1,536|
+
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1 # vary in different training stages
+ input_width: INPUT_WIDTH_1 # vary in different training stages
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+
+3. Start the first stage of training. First, revise the `CONFIG` file to enable full model training with low-resolution data:
+```yaml
+model:
+ params:
+ # ckpt_path: # disable this parameter so as to train from scratch
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 128
+ input_width: 128
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 50,000 steps with batch size 16 in this stage.
+
+4. Start the second stage of training. First, revise the `CONFIG` file to enable the fine-tuning of the decoder with high-resolution data:
+```yaml
+model:
+ params:
+ ckpt_path: CKPT_PATH # path to the saved checkpoint after the first stage of training
+ encoder_config:
+ params:
+ fix_encoder: true
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 256
+ input_width: 256
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 30,000 steps with batch size 8 in this stage.
+
+
+
+## 🚀 Inference
+
+### Easy Usage
+We provide the following example for a quick usage of our models. It works for both continuous and discrete tokenization and both causal and non-causal models.
+Just provide the path to the configuration file `cfg_path` and checkpoint file `ckpt_path`.
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+# model forward
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+assert x_input.shape == x_recon.shape
+```
+If you want to directly infer from latent tokens, run the following code:
+```python
+z, reg_log = model.encode(x_input, return_reg_log=True)
+# infer from continuous latent space
+x_recon = model.decode(z)
+# infer from discrete latent tokens
+x_recon = model.decode(reg_log['indices'], decode_from_indices=True)
+```
+
+### Use Torch Compile to Speed Up Inference
+Use compiled components in VidTok can speed up inference by as much as 2X. The following code snippet demonstrates how to compile our models.
+
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+torch._inductor.config.cpp.weight_prepack=True
+torch._inductor.config.freezing=True
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+model.encoder = torch.compile(model.encoder)
+model.decoder = torch.compile(model.decoder)
+
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+
+# Warm Up
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+
+torch.cuda.synchronize()
+import time
+start = time.time()
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ for i in range(10):
+ _, x_recon, _ = model(x_input)
+torch.cuda.synchronize()
+print(f"Average inference time: {(time.time() - start)/10 :.4f} seconds")
+```
+
+### Reconstruct an Input Video
+```bash
+python scripts/inference_reconstruct.py --config CONFIG --ckpt CKPT --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --output_video_dir OUTPUT_DIR
+```
+- Specify `VIDEO_PATH` to the path of your test video. We provide an example video in `assets/example.mp4`.
+- The reconstructed video is saved in `OUTPUT_DIR`.
+- For causal models, you can choose to add `--pad_gen_frames` to the command line, which may improve the smoothness of the reconstructed video.
+
+### Performance Evaluation
+We also provide a manuscript `scripts/inference_evaluate.py` to evaluate the video reconstruction performance in PSNR, SSIM and LPIPS.
+
+1. Put all of your test videos under `DATA_DIR`.
+2. Run the following command, and all `.mp4` videos under `DATA_DIR` will be tested:
+```bash
+python scripts/inference_evaluate.py --config CONFIG --ckpt CKPT --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30
+```
+(Optional) If you only want to test certain videos under `DATA_DIR`, you need to prepare a `.csv` meta file
+to indicate the video files to be tested (refer to [Data Preparation](#data-preparation)). And add `--meta_path META_PATH` to the above command to specify the path to the `.csv` meta file.
+
+## 💡 Intended Uses
+
+We are sharing our model with the research community to foster further research in this area:
+* Training your own video tokenizers for research purpose.
+* Video tokenization with various compression rates.
+
+
+## 🪧 Out-of-scope Uses
+
+Our models are not specifically designed or evaluated for all downstream purposes. Developers should consider common limitations of video tokenizers (e.g., performance degradation on out-of-domain data) as they select use cases, and evaluate and mitigate for privacy, safety, and fairness before using within a specific downstream use case, particularly for high-risk scenarios.
+
+Developers should be aware of and adhere to applicable laws or regulations (including privacy, trade compliance laws, etc.) that are relevant to their use case.
+
+
+## 🤖️ Risks and Limitations
+
+Some of the limitations of this model to be aware of include:
+* VidTok may lose detailed information on the reconstructed content.
+* VidTok inherits any biases, errors, or omissions characteristic of its training data.
+* VidTok was developed for research and experimental purposes. Further testing and validation are needed before considering its application in commercial or real-world scenarios.
+
+
+## 🤗 Acknowledgments
+
+This codebase borrows code from [generative-models](https://github.com/Stability-AI/generative-models). We thank Stability AI for its efforts and innovations, which have made the development process more efficient and convenient.
+
+Thank you to everyone who contributed their wisdom and efforts to this project.
+
+## ✏️ BibTeX
+
+```bibtex
+@article{tang2024vidtok,
+ title={VidTok: A Versatile and Open-Source Video Tokenizer},
+ author={Tang, Anni and He, Tianyu and Guo, Junliang and Cheng, Xinle and Song, Li and Bian, Jiang},
+ year={2024},
+ journal={arXiv preprint arXiv:2412.13061},
+}
+```
+
+## ☎️ Contact
+
+We welcome feedback and collaboration from our audience. If you have suggestions, questions, or observe unexpected/offensive behavior in our technology, please contact us at tianyuhe@microsoft.com.
+
+## 📄 Contributing
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+
+## 📍 Trademarks
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
+trademarks or logos is subject to and must follow
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/SECURITY.md b/Meissonic/VidTok/vidtok_cache/VidTok/SECURITY.md
new file mode 100644
index 0000000000000000000000000000000000000000..b3c89efc852e22f71eabf5dfbc6ac62493425eb6
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/SECURITY.md
@@ -0,0 +1,41 @@
+
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+ * Full paths of source file(s) related to the manifestation of the issue
+ * The location of the affected source code (tag/branch/commit or direct URL)
+ * Any special configuration required to reproduce the issue
+ * Step-by-step instructions to reproduce the issue
+ * Proof-of-concept or exploit code (if possible)
+ * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd).
+
+
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/SUPPORT.md b/Meissonic/VidTok/vidtok_cache/VidTok/SUPPORT.md
new file mode 100644
index 0000000000000000000000000000000000000000..291d4d43733f4c15a81ff598ec1c99fd6c18f64c
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/SUPPORT.md
@@ -0,0 +1,25 @@
+# TODO: The maintainer of this repo has not yet edited this file
+
+**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
+
+- **No CSS support:** Fill out this template with information about how to file issues and get help.
+- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
+- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
+
+*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
+
+# Support
+
+## How to file issues and get help
+
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates. For new issues, file your bug or
+feature request as a new Issue.
+
+For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
+FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
+CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
+
+## Microsoft Support Policy
+
+Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/assets/example.mp4 b/Meissonic/VidTok/vidtok_cache/VidTok/assets/example.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..21be26c8a8f5dbb76de5a225e5b284fdbb024904
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/assets/example.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:588ca89fae7320a079d4f77cf963f88075959f06310594ab35d3e04b844c4d50
+size 540937
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/assets/gemini.png b/Meissonic/VidTok/vidtok_cache/VidTok/assets/gemini.png
new file mode 100644
index 0000000000000000000000000000000000000000..fe326dc4b4b4b6db82575b15e9b4bf2f3b63e63d
Binary files /dev/null and b/Meissonic/VidTok/vidtok_cache/VidTok/assets/gemini.png differ
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/assets/radar.png b/Meissonic/VidTok/vidtok_cache/VidTok/assets/radar.png
new file mode 100644
index 0000000000000000000000000000000000000000..6eff9338928faf73c0e39e859237bc6c588ec183
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/assets/radar.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cfdef783e26264ff671f81845b54471b237f5cd7df0dbd63642fde0c20f935e
+size 424581
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin.png b/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin.png
new file mode 100644
index 0000000000000000000000000000000000000000..a7ae41105384d077a7d128d1d687e3ee8e17fbd8
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e58210f7ca7784a4df737f8a2ece2b5e9f85fff3bf423b24aa6aeeb0b196cef
+size 549933
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin_demo.png b/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin_demo.png
new file mode 100644
index 0000000000000000000000000000000000000000..ca79126104a7cbfb67c0acb70f03c6e66546675c
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/assets/vidtwin_demo.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af8e15c46050cc7957bc5d334bd1902d58b9be9aa7669df82f2ae9ba08c90585
+size 6371846
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..80b2edbebf97fd747933000d32ccbdfe0e135702
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f9c3e94ed8b64a681b2ed3887929109b6ccc53
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e661c533b367c7be4eea8de629e074b395e1684
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_32768.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9bf654bb21c1bd67ddde3f4878497c6ad780503a
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_4096.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8] # codebook size: 8*8*8*8=4096
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e400ff097155465e2a477bfbc3ca32f346a0ca12
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6c8731af435aae0efbde9716ee696ce3efd30d25
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4c20bf1aaf2aa97f67d19d0de1dbac47cf2d55ac
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_288_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1db41cd612f5e6b1390bebda33e559e347b60907
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bcbbd11cba27e60bd2f44866655adcfff91f91a6
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_444_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ spatial_ds: [1, 2]
+ spatial_us: [1, 2]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a7527679fe0766616df45223fe1e9595101fadc5
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d14b0d75435b24affad90095e4d6e42c6525a8d
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..336cbac8319dc3f9c232c02889e598fdf098a777
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b32064f1589eb3b21cb978f808a95fd6ade31ff9
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bdd33a5ebc635cbf9a04fb94006d14a473d87a3c
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..47a1090f42b07655626f0d522d756157fdeb1d6a
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05bb8c54df1374c0f67c79d256195dd6451acfbd
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..74364fa3209abbda65b0c23311d92ad0975570dd
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..334749091ff2c561208f169029eae4704d4213e3
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2966072ff2a073d404d0cb438674acfc92319033
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
@@ -0,0 +1,122 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [0, 1, 2]
+ tempo_us: [1, 2, 3]
+ time_downsample_factor: 8
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..deefcc29ecd8212e8bd3edd9d9870d8c64079db7
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
@@ -0,0 +1,116 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..965b243859e513244d3d3fd9cc68aa27aee887da
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f68342b48dd5fcae8e5bfcd2fcb5cc5bd1ab1c
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b4badafed742c03fd7850f197cdb8207c59b992e
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8bcd59650c229ac57560602854015262969d658b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
@@ -0,0 +1,154 @@
+model:
+ base_learning_rate: 1.6e-4
+ target: vidtwin.models.vidtwin_ae.VidAutoEncoderQformerCompactSymVidVAE
+ params:
+ input_key: jpg
+ monitor: val/rec_loss
+ ckpt_path: PATH_TO_CHECKPOINT
+ ignore_keys: []
+ expect_ch: 8
+ cont_num_blocks: 1
+ downsample_motion: True
+ motion_num_blocks: 1
+ d_dim: 8
+
+ temporal_qformer_config:
+ target: vidtwin.modules.qformer.MyQformerInterface
+ params:
+ num_query_tokens: 16
+ query_hidden_size: 64
+ encoder_hidden_size: 768
+
+ encoder_config:
+ target: vidtwin.modules.st_transformer.STTEncoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ decoder_config:
+ target: vidtwin.modules.st_transformer.STTDecoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ perceptual_weight: 0.05
+ disc_start: 20001
+ disc_weight: 0.05
+ learn_logvar: True
+ dims: 3
+ disc_type: 2d
+ regularization_weights:
+ kl_loss: 0.001
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+ params:
+ sample: True
+
+
+ lr_scheduler_config_d:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 1.5e-05
+ lr_start: 1.0e-05
+ warmup_steps: 5000
+ lr_scheduler_config_g:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 3.0e-05
+ lr_start: 0
+ warmup_steps: 5000
+ optimizer_config:
+ target: torch.optim.AdamW
+ params:
+ betas:
+ - 0
+ - 0.9
+ weight_decay: 0.0001
+ lr_scheduler_config:
+ target: inverse_sqrt
+ params:
+ num_warmup_steps: 2000
+ frequency: 1
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: True
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 2
+
+
+
+ trainer:
+ # precision: bf16-mixed # 16-mixed
+ benchmark: True
+ devices: 4
+ num_sanity_val_steps: 10
+ val_check_interval: 5000
+ accumulate_grad_batches: 1
+ max_epochs: 10
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/environment.yaml b/Meissonic/VidTok/vidtok_cache/VidTok/environment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6abf1dcc7cefdab27b272681c0395791b3d432f
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/environment.yaml
@@ -0,0 +1,114 @@
+name: vidtok
+channels:
+ - defaults
+dependencies:
+ - _libgcc_mutex=0.1=main
+ - _openmp_mutex=5.1=1_gnu
+ - bzip2=1.0.8=h5eee18b_6
+ - ca-certificates=2024.11.26=h06a4308_0
+ - ld_impl_linux-64=2.40=h12ee557_0
+ - libffi=3.4.4=h6a678d5_1
+ - libgcc-ng=11.2.0=h1234567_1
+ - libgomp=11.2.0=h1234567_1
+ - libstdcxx-ng=11.2.0=h1234567_1
+ - libuuid=1.41.5=h5eee18b_0
+ - ncurses=6.4=h6a678d5_0
+ - openssl=3.0.15=h5eee18b_0
+ - pip=24.2=py310h06a4308_0
+ - python=3.10.15=he870216_1
+ - readline=8.2=h5eee18b_0
+ - setuptools=75.1.0=py310h06a4308_0
+ - sqlite=3.45.3=h5eee18b_0
+ - tk=8.6.14=h39e8969_0
+ - wheel=0.44.0=py310h06a4308_0
+ - xz=5.4.6=h5eee18b_1
+ - zlib=1.2.13=h5eee18b_1
+ - pip:
+ - absl-py==2.1.0
+ - aiohappyeyeballs==2.4.4
+ - aiohttp==3.11.9
+ - aiosignal==1.3.1
+ - antlr4-python3-runtime==4.9.3
+ - appdirs==1.4.4
+ - async-timeout==5.0.1
+ - attrs==24.2.0
+ - av==12.0.0
+ - beartype==0.18.2
+ - certifi==2024.8.30
+ - charset-normalizer==3.4.0
+ - click==8.1.7
+ - contourpy==1.3.1
+ - cycler==0.12.1
+ - decord==0.6.0
+ - docker-pycreds==0.4.0
+ - einops==0.8.0
+ - filelock==3.16.1
+ - fonttools==4.55.1
+ - frozenlist==1.5.0
+ - fsspec==2024.10.0
+ - gitdb==4.0.11
+ - gitpython==3.1.43
+ - grpcio==1.68.1
+ - idna==3.10
+ - imageio==2.34.0
+ - jinja2==3.1.4
+ - kiwisolver==1.4.7
+ - lightning==2.2.4
+ - lightning-utilities==0.11.9
+ - markdown==3.7
+ - markdown-it-py==3.0.0
+ - markupsafe==3.0.2
+ - matplotlib==3.8.4
+ - mdurl==0.1.2
+ - mpmath==1.3.0
+ - multidict==6.1.0
+ - natsort==8.4.0
+ - networkx==3.4.2
+ - numpy==1.26.4
+ - nvidia-cublas-cu12==12.1.3.1
+ - nvidia-cuda-cupti-cu12==12.1.105
+ - nvidia-cuda-nvrtc-cu12==12.1.105
+ - nvidia-cuda-runtime-cu12==12.1.105
+ - nvidia-cudnn-cu12==8.9.2.26
+ - nvidia-cufft-cu12==11.0.2.54
+ - nvidia-curand-cu12==10.3.2.106
+ - nvidia-cusolver-cu12==11.4.5.107
+ - nvidia-cusparse-cu12==12.1.0.106
+ - nvidia-nccl-cu12==2.19.3
+ - nvidia-nvjitlink-cu12==12.6.85
+ - nvidia-nvtx-cu12==12.1.105
+ - omegaconf==2.3.0
+ - opencv-python==4.6.0.66
+ - packaging==24.2
+ - pandas==2.1.4
+ - pillow==11.0.0
+ - propcache==0.2.1
+ - protobuf==4.25.5
+ - psutil==6.1.0
+ - pygments==2.18.0
+ - pyparsing==3.2.0
+ - python-dateutil==2.9.0.post0
+ - pytorch-lightning==2.2.4
+ - pytz==2024.2
+ - pyyaml==6.0.2
+ - requests==2.32.3
+ - rich==13.5.3
+ - safetensors==0.4.2
+ - sentry-sdk==2.19.0
+ - setproctitle==1.3.4
+ - six==1.17.0
+ - smmap==5.0.1
+ - sympy==1.13.3
+ - tensorboard==2.16.2
+ - tensorboard-data-server==0.7.2
+ - torch==2.2.2
+ - torchmetrics==1.6.0
+ - torchvision==0.17.2
+ - tqdm==4.67.1
+ - triton==2.2.0
+ - typing-extensions==4.12.2
+ - tzdata==2024.2
+ - urllib3==2.2.3
+ - wandb==0.16.6
+ - werkzeug==3.1.3
+ - yarl==1.18.3
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/main.py b/Meissonic/VidTok/vidtok_cache/VidTok/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..e03f0de8a1c683fbdde7177083c87a0a3df85f83
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/main.py
@@ -0,0 +1,1124 @@
+import argparse
+import datetime
+import pytz
+import glob
+import inspect
+import os
+import re
+import sys
+import numpy as np
+import warnings
+warnings.filterwarnings("ignore")
+from rich import print
+from inspect import Parameter
+from typing import Union
+from matplotlib import pyplot as plt
+from natsort import natsorted
+from omegaconf import OmegaConf
+from packaging import version
+from PIL import Image
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch import seed_everything
+from lightning.pytorch.trainer import Trainer
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from vidtok.modules.util import (exists, instantiate_from_config, isheatmap,
+ print0, seed_anything)
+
+MULTINODE_HACKS = True
+
+
+def default_trainer_args():
+ argspec = dict(inspect.signature(Trainer.__init__).parameters)
+ argspec.pop("self")
+ default_args = {
+ param: argspec[param].default
+ for param in argspec
+ if argspec[param] != Parameter.empty
+ }
+ return default_args
+
+
+def get_step_value(folder_name):
+ match = re.search(r"step=(\d+)", folder_name)
+ if match:
+ return int(match.group(1))
+ return 0
+
+
+def get_parser(**parser_kwargs):
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser(**parser_kwargs)
+ parser.add_argument(
+ "-n",
+ "--name",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="postfix for logdir",
+ )
+ parser.add_argument(
+ "--no_date",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="if True, skip date generation for logdir and only use naming via opt.base or opt.name (+ opt.postfix, optionally)",
+ )
+ parser.add_argument(
+ "-r",
+ "--resume",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="resume from logdir or checkpoint in logdir",
+ )
+ parser.add_argument(
+ "-b",
+ "--base",
+ nargs="*",
+ metavar="base_config.yaml",
+ help="paths to base configs. Loaded from left-to-right. "
+ "Parameters can be overwritten or added with command-line options of the form `--key value`.",
+ default=list(),
+ )
+ parser.add_argument(
+ "-t",
+ "--train",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="train",
+ )
+ parser.add_argument(
+ "--no-test",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="disable test",
+ )
+ parser.add_argument(
+ "-p", "--project", help="name of new or path to existing project"
+ )
+ parser.add_argument(
+ "-d",
+ "--debug",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="enable post-mortem debugging",
+ )
+ parser.add_argument(
+ "-s",
+ "--seed",
+ type=int,
+ default=23,
+ help="seed for seed_everything",
+ )
+ parser.add_argument(
+ "--seed_rank",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="reset seed every rank on fit start",
+ )
+ parser.add_argument(
+ "-f",
+ "--postfix",
+ type=str,
+ default="",
+ help="post-postfix for default name",
+ )
+ parser.add_argument(
+ "-l",
+ "--logdir",
+ type=str,
+ default="logs",
+ help="directory for logging dat shit",
+ )
+ parser.add_argument(
+ "--scale_lr",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="scale base-lr by ngpu * batch_size * n_accumulate",
+ )
+ parser.add_argument(
+ "--legacy_naming",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="name run based on config file name if true, else by whole path",
+ )
+ parser.add_argument(
+ "--enable_tf32",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="enables the TensorFloat32 format both for matmuls and cuDNN for pytorch 1.12",
+ )
+ parser.add_argument(
+ "--startup",
+ type=str,
+ default=None,
+ help="Startuptime from distributed script",
+ )
+ parser.add_argument(
+ "--wandb",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="log to wandb",
+ )
+ parser.add_argument(
+ "--wandb_entity",
+ type=str,
+ default="",
+ help="Wandb entity name string",
+ )
+ parser.add_argument(
+ "--wandb_key",
+ type=str,
+ default="",
+ help="Wandb key",
+ )
+ parser.add_argument(
+ "--wandb_project",
+ type=str,
+ default="vidtok",
+ )
+ parser.add_argument(
+ "--wandb_id",
+ type=str,
+ default=None,
+ help="automatically resume from the same wandb id"
+ "must be used in combination with --wandb_auto_resume False",
+ )
+ parser.add_argument(
+ "--wandb_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest run id in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--checkpoint_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest checkpoint in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--no_base_name",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False, # TODO: later default to True
+ help="log to wandb",
+ )
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ parser.add_argument(
+ "--resume_from_checkpoint",
+ type=str,
+ default=None,
+ help="single checkpoint file to resume from",
+ )
+ default_args = default_trainer_args()
+ for key in default_args:
+ # parameters in the pl.Trainer are passed as --key value
+ parser.add_argument("--" + key, default=default_args[key])
+ return parser
+
+
+def get_checkpoint_name(logdir):
+ ckpt = os.path.join(logdir, "checkpoints", "last**.ckpt")
+ ckpt = natsorted(glob.glob(ckpt))
+ print0('available "last" checkpoints:')
+ print0(ckpt)
+ if len(ckpt) > 1:
+ print0("got most recent checkpoint")
+ ckpt = sorted(ckpt, key=lambda x: os.path.getmtime(x))[-1]
+ print0(f"Most recent ckpt is {ckpt}")
+ with open(os.path.join(logdir, "most_recent_ckpt.txt"), "w") as f:
+ f.write(ckpt + "\n")
+ try:
+ version = int(ckpt.split("/")[-1].split("-v")[-1].split(".")[0])
+ except Exception as e:
+ print0("version confusion but not bad")
+ print0(e)
+ version = 1
+ # version = last_version + 1
+ else:
+ # in this case, we only have one "last.ckpt"
+ ckpt = ckpt[0]
+ version = 1
+ melk_ckpt_name = f"last-v{version}.ckpt"
+ print0(f"Current melk ckpt name: {melk_ckpt_name}")
+ return ckpt, melk_ckpt_name
+
+
+class SetupCallback(Callback):
+ def __init__(
+ self,
+ resume,
+ now,
+ logdir,
+ ckptdir,
+ cfgdir,
+ config,
+ lightning_config,
+ debug,
+ save_ckpt_on_exception=False,
+ ckpt_name=None,
+ seed=None,
+ seed_rank=False,
+ ):
+ super().__init__()
+ self.resume = resume
+ self.now = now
+ self.logdir = logdir
+ self.ckptdir = ckptdir
+ self.cfgdir = cfgdir
+ self.config = config
+ self.lightning_config = lightning_config
+ self.debug = debug
+ self.save_ckpt_on_exception = save_ckpt_on_exception
+ self.ckpt_name = ckpt_name
+ self.seed = seed
+ self.seed_rank = seed_rank
+
+ def on_exception(self, trainer: pl.Trainer, pl_module, exception):
+ if self.save_ckpt_on_exception and (not self.debug) and (trainer.global_rank == 0):
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Saving checkpoint to {self.ckptdir}")
+ if self.ckpt_name is None:
+ ckpt_path = os.path.join(self.ckptdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(self.ckptdir, self.ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def on_fit_start(self, trainer, pl_module):
+ if self.seed_rank:
+ # current_seed = torch.initial_seed()
+ seed_anything(self.seed + trainer.global_rank)
+ print(f"[bold red]\[main][SetupCallback][/bold red] Rank {trainer.global_rank}: Reset GLOBAL seed to {self.seed + trainer.global_rank}")
+ elif hasattr(pl_module, "set_seed") and callable(pl_module.set_seed):
+ pl_module.set_seed(self.seed)
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Set pl_module seed to {self.seed} with pl_module.set_seed")
+ if trainer.global_rank == 0:
+ # Create logdirs and save configs
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Creating logdir: {self.logdir}, ckptdir: {self.ckptdir}, cfgdir: {self.cfgdir}")
+ os.makedirs(self.logdir, exist_ok=True)
+ os.makedirs(self.ckptdir, exist_ok=True)
+ os.makedirs(self.cfgdir, exist_ok=True)
+
+ if "callbacks" in self.lightning_config:
+ if (
+ "metrics_over_trainsteps_checkpoint"
+ in self.lightning_config["callbacks"]
+ ):
+ os.makedirs(
+ os.path.join(self.ckptdir, "trainstep_checkpoints"),
+ exist_ok=True,
+ )
+ print0("[bold red]\[main][SetupCallback][/bold red] Project config")
+ print0(OmegaConf.to_yaml(self.config))
+ if MULTINODE_HACKS and not self.debug:
+ import time
+ time.sleep(5)
+ OmegaConf.save(
+ self.config,
+ os.path.join(self.cfgdir, "{}-project.yaml".format(self.now)),
+ )
+
+ print0("[bold red]\[main][SetupCallback][/bold red] Lightning config")
+ print0(OmegaConf.to_yaml(self.lightning_config))
+ OmegaConf.save(
+ OmegaConf.create({"lightning": self.lightning_config}),
+ os.path.join(self.cfgdir, "{}-lightning.yaml".format(self.now)),
+ )
+
+ else:
+ # ModelCheckpoint callback created log directory --- remove it
+ if not MULTINODE_HACKS and not self.resume and os.path.exists(self.logdir):
+ dst, name = os.path.split(self.logdir)
+ dst = os.path.join(dst, "child_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ try:
+ os.rename(self.logdir, dst)
+ except FileNotFoundError:
+ pass
+
+
+class ImageLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ rescale=True,
+ disabled=True,
+ log_on_batch_idx=False,
+ log_first_step=False,
+ log_images_kwargs=None,
+ log_before_first_step=False,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ # TODO: support wandb
+ else:
+ grid = torchvision.utils.make_grid(images[k], nrow=4)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ self.check_frequency(check_idx)
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ gpu_autocast_kwargs = {
+ "enabled": self.enable_autocast, # torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad(), torch.cuda.amp.autocast(**gpu_autocast_kwargs):
+ images = pl_module.log_images(
+ batch, split=split, **self.log_images_kwargs
+ )
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ print0("[bold red]\[main][ImageLogger][/bold red]", e)
+ pass
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ print0(f"[bold red]\[main][ImageLogger][/bold red] {self.__class__.__name__}: logging before training")
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+@rank_zero_only
+def init_wandb(save_dir, opt, config, group_name, name_str):
+ print0(f"[bold red]\[main][init_wandb][/bold red] Creating WANDB_DIR: {save_dir}")
+ os.makedirs(save_dir, exist_ok=True)
+
+ # os.environ["WANDB_DIR"] = save_dir
+ gitcmd = f'git config --global --add safe.directory {os.path.dirname(os.path.abspath(__file__))}'
+ os.system(gitcmd)
+ print0(f"[bold red]\[main][init_wandb][/bold red] wandb_id is set to {opt.wandb_id}")
+ wandb_id = opt.wandb_id if opt.wandb_id is not None else name_str
+
+ if not wandb.api.api_key:
+ wandb.login(key=opt.wandb_key)
+ if opt.debug:
+ wandb.init(project=opt.wandb_project, mode="offline", group=group_name)
+ else:
+ wandb.init(
+ project=opt.wandb_project,
+ entity=opt.wandb_entity,
+ config=dict(config),
+ group=group_name,
+ name=name_str,
+ resume='auto',
+ id=wandb_id,
+ )
+
+
+if __name__ == "__main__":
+ # custom parser to specify config files, train, test and debug mode,
+ # postfix, resume.
+ # `--key value` arguments are interpreted as arguments to the trainer.
+ # `nested.key=value` arguments are interpreted as config parameters.
+ # configs are merged from left-to-right followed by command line parameters.
+
+ # model:
+ # base_learning_rate: float
+ # target: path to lightning module
+ # params:
+ # key: value
+ # data:
+ # target: main.DataModuleFromConfig
+ # params:
+ # batch_size: int
+ # wrap: bool
+ # train:
+ # target: path to train dataset
+ # params:
+ # key: value
+ # validation:
+ # target: path to validation dataset
+ # params:
+ # key: value
+ # test:
+ # target: path to test dataset
+ # params:
+ # key: value
+ # lightning: (optional, has sane defaults and can be specified on cmdline)
+ # trainer:
+ # additional arguments to trainer
+ # logger:
+ # logger to instantiate
+ # modelcheckpoint:
+ # modelcheckpoint to instantiate
+ # callbacks:
+ # callback1:
+ # target: importpath
+ # params:
+ # key: value
+
+ now = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+
+ # add cwd for convenience and to make classes in this file available when
+ # running as `python main.py`
+ # (in particular `main.DataModuleFromConfig`)
+ sys.path.append(os.getcwd())
+
+ parser = get_parser()
+
+ opt, unknown = parser.parse_known_args()
+
+ if opt.name and opt.resume:
+ raise ValueError(
+ "-n/--name and -r/--resume cannot be specified both."
+ "If you want to resume training in a new log folder, "
+ "use -n/--name in combination with --resume_from_checkpoint"
+ )
+ melk_ckpt_name = None
+ name = None
+ if opt.resume:
+ if not os.path.exists(opt.resume):
+ raise ValueError("Cannot find {}".format(opt.resume))
+ if os.path.isfile(opt.resume):
+ paths = opt.resume.split("/")
+ # idx = len(paths)-paths[::-1].index("logs")+1
+ # logdir = "/".join(paths[:idx])
+ logdir = "/".join(paths[:-2])
+ ckpt = opt.resume
+ _, melk_ckpt_name = get_checkpoint_name(logdir)
+ else:
+ assert os.path.isdir(opt.resume), opt.resume
+ logdir = opt.resume.rstrip("/")
+ ckpt, melk_ckpt_name = get_checkpoint_name(logdir)
+
+ print0("-" * 80)
+ print0(f'[bold red][main][/bold red] Resuming from checkpoint "{ckpt}"')
+
+ opt.resume_from_checkpoint = ckpt
+ base_configs = sorted(glob.glob(os.path.join(logdir, "configs/*.yaml")))
+ opt.base = base_configs + opt.base
+ _tmp = logdir.split("/")
+ nowname = _tmp[-1]
+ else:
+ if opt.name:
+ name = "_" + opt.name
+ elif opt.base:
+ if opt.no_base_name:
+ name = ""
+ else:
+ if opt.legacy_naming:
+ cfg_fname = os.path.split(opt.base[0])[-1]
+ cfg_name = os.path.splitext(cfg_fname)[0]
+ else:
+ assert "configs" in os.path.split(opt.base[0])[0], os.path.split(
+ opt.base[0]
+ )[0]
+ cfg_path = os.path.split(opt.base[0])[0].split(os.sep)[
+ os.path.split(opt.base[0])[0].split(os.sep).index("configs")
+ + 1 :
+ ] # cut away the first one (we assert all configs are in "configs")
+ cfg_name = os.path.splitext(os.path.split(opt.base[0])[-1])[0]
+ cfg_name = "-".join(cfg_path) + f"-{cfg_name}"
+ name = "_" + cfg_name
+ else:
+ name = ""
+ # automatic resume last checkpoint if available
+ if os.path.exists(opt.logdir):
+ auto_resumed = False
+ for sub_dir in sorted(os.listdir(opt.logdir)):
+ if sub_dir.endswith(name + opt.postfix):
+ ## checkpoint resume
+ if opt.checkpoint_auto_resume and not opt.debug:
+ checkpoint_dir = os.path.join(opt.logdir, sub_dir, "checkpoints")
+ # Use the max step checkpoint file
+ ckpt_files1 = glob.glob(os.path.join(checkpoint_dir, "*/*.ckpt"))
+ ckpt_files2 = glob.glob(os.path.join(checkpoint_dir, "*.ckpt"))
+ ckpt_files = ckpt_files1 + ckpt_files2
+ ckpt_files.sort(key=get_step_value, reverse=True)
+ if ckpt_files:
+ ckpt = ckpt_files[0]
+ else:
+ # If no checkpoint files found, use a random initialized model
+ ckpt = None
+ if ckpt is not None and os.path.isfile(ckpt):
+ opt.resume_from_checkpoint = ckpt
+ auto_resumed = True
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous log dir and checkpoint: {ckpt}")
+ ## wandb resume
+ if opt.wandb_auto_resume:
+ wandb_dir = Path(os.path.join(opt.logdir, sub_dir)) / "wandb"
+ if wandb_dir.exists() and any((wandb_dir / "latest-run").iterdir()):
+ # Parse unique `run_id` from the `.wandb.` file...
+ wandb_fns = [f.name for f in (wandb_dir / "latest-run").iterdir() if f.name.endswith(".wandb")]
+ assert len(wandb_fns) == 1, f"There should only be 1 `.wandb.` file... found {len(wandb_fns)}!"
+ # Regex Match on `run-{id}.wandb`
+ opt.wandb_id = re.search("run-(.+?).wandb", wandb_fns[0]).group(1)
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous wandb run id: {opt.wandb_id}")
+ if auto_resumed:
+ print0(f"[bold red]\[main][/bold red] Auto-resuming from checkpoint: {opt.resume_from_checkpoint} and wandb id: {opt.wandb_id}")
+ ckpt_basename = os.path.basename(opt.resume_from_checkpoint)
+ seed_str = ''.join(re.findall(r'\d+', ckpt_basename))
+ if len(seed_str) > 0:
+ opt.seed = int(seed_str)
+ print0(f"[bold red]\[main][/bold red] Auto-reseting seed to {opt.seed} from checkpoint name")
+
+ if not opt.no_date:
+ nowname = now + name + opt.postfix
+ else:
+ nowname = name + opt.postfix
+ if nowname.startswith("_"):
+ nowname = nowname[1:]
+ logdir = os.path.join(opt.logdir, nowname)
+ print0(f"[bold red]\[main][/bold red] LOGDIR: {logdir}")
+
+ ckptdir = os.path.join(logdir, "checkpoints")
+ cfgdir = os.path.join(logdir, "configs")
+ if not opt.seed_rank:
+ seed_everything(opt.seed, workers=True) # torch.initial_seed()
+
+ # move before model init, in case a torch.compile(...) is called somewhere
+ if opt.enable_tf32:
+ # pt_version = version.parse(torch.__version__)
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.backends.cudnn.allow_tf32 = True
+ print0(f"[bold red]\[main][/bold red] Enabling TF32 for PyTorch {torch.__version__}")
+ else:
+ print0(f"[bold red]\[main][/bold red] Using default TF32 settings for PyTorch {torch.__version__}:")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cuda.matmul.allow_tf32={torch.backends.cuda.matmul.allow_tf32}")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cudnn.allow_tf32={torch.backends.cudnn.allow_tf32}")
+
+ try:
+ # init and save configs
+ configs = [OmegaConf.load(cfg) for cfg in opt.base]
+ # deal with the unknown args, e.g., --model.base_learning_rate=1.0e-4
+ for i, u in enumerate(unknown):
+ if u.startswith("--"):
+ unknown[i] = u[2:]
+ # merge all configs and cli args
+ cli = OmegaConf.from_dotlist(unknown)
+ config = OmegaConf.merge(*configs, cli)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged input config: {config}")
+ lightning_config = config.pop("lightning", OmegaConf.create())
+ # merge trainer cli with config
+ trainer_config = lightning_config.get("trainer", OmegaConf.create())
+
+ # debug: default to one node
+ if opt.debug:
+ trainer_config["num_nodes"] = 1
+
+ # default profiler
+ trainer_config["profiler"] = None if not opt.debug else "simple"
+
+ # default to gpu
+ trainer_config["accelerator"] = "gpu"
+ #
+ standard_args = default_trainer_args()
+ for k in standard_args:
+ if getattr(opt, k) != standard_args[k]:
+ trainer_config[k] = getattr(opt, k)
+
+ if not "devices" in trainer_config and trainer_config["accelerator"] != "gpu":
+ del trainer_config["accelerator"]
+ cpu = True
+ else:
+ gpuinfo = trainer_config["devices"]
+ print0(f"[bold red]\[main][/bold red] Running on {gpuinfo} GPUs")
+ cpu = False
+ trainer_opt = argparse.Namespace(**trainer_config)
+ lightning_config.trainer = trainer_config
+
+ # model
+ model = instantiate_from_config(config.model)
+
+ # trainer and callbacks
+ trainer_kwargs = dict()
+
+ # default logger configs
+ default_logger_cfgs = {
+ "wandb": {
+ "target": "lightning.pytorch.loggers.WandbLogger",
+ "params": {
+ "name": nowname,
+ "save_dir": logdir,
+ "offline": opt.debug,
+ "id": nowname,
+ "project": opt.wandb_project,
+ "log_model": False,
+ "entity": opt.wandb_entity,
+ },
+ },
+ "csv": {
+ "target": "lightning.pytorch.loggers.CSVLogger",
+ "params": {
+ "name": "testtube", # hack for sbord fanatics
+ "save_dir": logdir,
+ },
+ },
+ "tensorboard": {
+ "target": "lightning.pytorch.loggers.TensorBoardLogger",
+ "params": {
+ "save_dir": logdir,
+ "name": 'tensorboard',
+ "version": nowname,
+ }
+ },
+ }
+ default_logger_cfg = default_logger_cfgs["wandb" if opt.wandb else "tensorboard"]
+ if opt.wandb:
+ # change once leaving "swiffer" config directory
+ try:
+ group_name = nowname.split(now)[-1].split("-")[1]
+ except:
+ group_name = nowname
+ default_logger_cfg["params"]["group"] = group_name
+
+ wandb_save_dir = os.path.join(os.getcwd(), logdir)
+ os.environ["WANDB_DIR"] = wandb_save_dir
+
+ init_wandb(
+ wandb_save_dir,
+ opt=opt,
+ group_name=group_name,
+ config=config,
+ name_str=nowname,
+ )
+ if "logger" in lightning_config:
+ logger_cfg = lightning_config.logger
+ else:
+ logger_cfg = OmegaConf.create()
+ logger_cfg = OmegaConf.merge(default_logger_cfg, logger_cfg)
+ trainer_kwargs["logger"] = instantiate_from_config(logger_cfg)
+
+ ckpt_resume_path = opt.resume_from_checkpoint
+
+ # modelcheckpoint - use TrainResult/EvalResult(checkpoint_on=metric) to
+ # specify which metric is used to determine best models
+ default_modelckpt_cfg = {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": ckptdir,
+ "filename": "{epoch:04}-{step:08}", # "epoch={epoch:06}-step={step:07}"
+ "verbose": True,
+ "save_last": True,
+ "auto_insert_metric_name": True,
+ },
+ }
+ if hasattr(model, "monitor"):
+ print0(f"[bold red]\[main][/bold red] Monitoring {model.monitor} as checkpoint metric.")
+ default_modelckpt_cfg["params"]["monitor"] = model.monitor
+ default_modelckpt_cfg["params"]["save_top_k"] = 3
+
+ if "modelcheckpoint" in lightning_config:
+ modelckpt_cfg = lightning_config.modelcheckpoint
+ else:
+ modelckpt_cfg = OmegaConf.create()
+ modelckpt_cfg = OmegaConf.merge(default_modelckpt_cfg, modelckpt_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged modelckpt-cfg: {modelckpt_cfg}")
+
+ # https://pytorch-lightning.readthedocs.io/en/stable/extensions/strategy.html
+ # default to ddp if not further specified
+ default_strategy_config = {"target": "lightning.pytorch.strategies.DDPStrategy"}
+
+ if "strategy" in lightning_config:
+ strategy_cfg = lightning_config.strategy
+ else:
+ strategy_cfg = OmegaConf.create()
+ default_strategy_config["params"] = {
+ "find_unused_parameters": False,
+ # "static_graph": True,
+ # "ddp_comm_hook": default.fp16_compress_hook # experiment with this, also for DDPSharded
+ }
+ strategy_cfg = OmegaConf.merge(default_strategy_config, strategy_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] strategy config: {strategy_cfg}")
+ trainer_kwargs["strategy"] = instantiate_from_config(strategy_cfg)
+ if hasattr(trainer_kwargs["strategy"], "_timeout"):
+ trainer_kwargs["strategy"]._timeout = datetime.timedelta(seconds=5400) # 3600s = 1h
+
+ # add callback which sets up log directory
+ default_callbacks_cfg = {
+ "setup_callback": {
+ "target": "main.SetupCallback",
+ "params": {
+ "resume": opt.resume,
+ "now": now,
+ "logdir": logdir,
+ "ckptdir": ckptdir,
+ "cfgdir": cfgdir,
+ "config": config,
+ "lightning_config": lightning_config,
+ "debug": opt.debug,
+ "ckpt_name": melk_ckpt_name,
+ "seed": opt.seed,
+ "seed_rank": opt.seed_rank
+ },
+ },
+ "image_logger": {
+ "target": "main.ImageLogger",
+ "params": {"batch_frequency": 1000, "max_samples": 4, "clamp": True},
+ },
+ "learning_rate_logger": {
+ "target": "lightning.pytorch.callbacks.LearningRateMonitor",
+ "params": {
+ "logging_interval": "step",
+ # "log_momentum": True
+ },
+ },
+ }
+ if version.parse(pl.__version__) >= version.parse("1.4.0"):
+ default_callbacks_cfg.update({"checkpoint_callback": modelckpt_cfg})
+
+ if "callbacks" in lightning_config:
+ callbacks_cfg = lightning_config.callbacks
+ else:
+ callbacks_cfg = OmegaConf.create()
+
+ if "metrics_over_trainsteps_checkpoint" in callbacks_cfg:
+ print0(
+ "[bold red]\[main][/bold red] Caution: Saving checkpoints every n train steps without deleting. This might require some free space."
+ )
+ default_metrics_over_trainsteps_ckpt_dict = {
+ "metrics_over_trainsteps_checkpoint": {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": os.path.join(ckptdir, "trainstep_checkpoints"),
+ "filename": "{epoch:04}-{step:08}", # "{epoch:06}-{step:09}"
+ "verbose": True,
+ "save_top_k": -1,
+ "every_n_train_steps": 10000,
+ "save_weights_only": True,
+ },
+ }
+ }
+ default_callbacks_cfg.update(default_metrics_over_trainsteps_ckpt_dict)
+
+ callbacks_cfg = OmegaConf.merge(default_callbacks_cfg, callbacks_cfg)
+ if "ignore_keys_callback" in callbacks_cfg and ckpt_resume_path is not None:
+ callbacks_cfg.ignore_keys_callback.params["ckpt_path"] = ckpt_resume_path
+ elif "ignore_keys_callback" in callbacks_cfg:
+ del callbacks_cfg["ignore_keys_callback"]
+
+ trainer_kwargs["callbacks"] = [
+ instantiate_from_config(callbacks_cfg[k]) for k in callbacks_cfg
+ ]
+ if not "plugins" in trainer_kwargs:
+ trainer_kwargs["plugins"] = list()
+
+ # cmd line trainer args (which are in trainer_opt) have always priority over config-trainer-args (which are in trainer_kwargs)
+ trainer_opt = vars(trainer_opt)
+ trainer_kwargs = {
+ key: val for key, val in trainer_kwargs.items() if key not in trainer_opt
+ }
+ trainer = Trainer(**trainer_opt, **trainer_kwargs)
+
+ trainer.logdir = logdir
+
+ # data
+ if ((not opt.train) or opt.debug) and hasattr(config.data.params, "validation"):
+ config.data.params.train = config.data.params.validation
+ print0("[bold red]\[main][/bold red] Using validation data as training data for fast loading.")
+ data = instantiate_from_config(config.data)
+ # NOTE according to https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html
+ # calling these ourselves should not be necessary but it is.
+ # lightning still takes care of proper multiprocessing though
+ data.prepare_data()
+ # data.setup()
+ try:
+ for k in data.datasets:
+ print0(
+ f"[bold red]\[main][/bold red] {k}, {data.datasets[k].__class__.__name__}, {len(data.datasets[k])}"
+ )
+ except:
+ print0("[bold red]\[main][/bold red] datasets not yet initialized.")
+
+ # configure learning rate
+ if "batch_size" in config.data.params:
+ bs, base_lr = config.data.params.batch_size, config.model.base_learning_rate
+ else:
+ bs, base_lr = (
+ config.data.params.train.loader.batch_size,
+ config.model.base_learning_rate,
+ )
+ if not cpu:
+ # add for different device input type
+ if isinstance(lightning_config.trainer.devices, int):
+ ngpu = lightning_config.trainer.devices
+ elif isinstance(lightning_config.trainer.devices, list):
+ ngpu = len(lightning_config.trainer.devices)
+ elif isinstance(lightning_config.trainer.devices, str):
+ ngpu = len(lightning_config.trainer.devices.strip(",").split(","))
+ else:
+ ngpu = 1
+ if "accumulate_grad_batches" in lightning_config.trainer:
+ accumulate_grad_batches = lightning_config.trainer.accumulate_grad_batches
+ else:
+ accumulate_grad_batches = 1
+ print0(f"[bold red]\[main][/bold red] accumulate_grad_batches = {accumulate_grad_batches}")
+ lightning_config.trainer.accumulate_grad_batches = accumulate_grad_batches
+
+ if opt.scale_lr:
+ model.learning_rate = accumulate_grad_batches * ngpu * bs * base_lr
+ print0(
+ "[bold red]\[main][/bold red] Setting learning rate to {:.2e} = {} (accumulate_grad_batches) * {} (num_gpus) * {} (batchsize) * {:.2e} (base_lr)".format(
+ model.learning_rate, accumulate_grad_batches, ngpu, bs, base_lr
+ )
+ )
+ else:
+ model.learning_rate = base_lr
+ print0("[bold red]\[main][/bold red] NOT using learning rate scaling")
+ print0(f"[bold red]\[main][/bold red] Setting learning rate to {model.learning_rate:.2e}")
+
+ # allow checkpointing via USR1
+ def melk(*args, **kwargs):
+ # run all checkpoint hooks
+ if trainer.global_rank == 0:
+ melkdir = os.path.join(logdir, "melk")
+ os.makedirs(melkdir, exist_ok=True)
+ print0(f"[bold red]\[main][/bold red] Saving checkpoint to {melkdir}")
+ if melk_ckpt_name is None:
+ ckpt_path = os.path.join(melkdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(melkdir, melk_ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def divein(*args, **kwargs):
+ if trainer.global_rank == 0:
+ import pudb
+ pudb.set_trace()
+
+ import signal
+ signal.signal(signal.SIGUSR1, melk)
+ signal.signal(signal.SIGUSR2, divein)
+
+ # run
+ if opt.train:
+ try:
+ trainer.fit(model, data, ckpt_path=ckpt_resume_path)
+ print0(f"[bold red]\[main][/bold red] Finish training with logdir: {logdir}")
+ except Exception as e:
+ print(f"")
+ print(f"[bold red]\[main][/bold red] Exception: {e}")
+ print(f"[bold red]\[main][/bold red] Beijing Time {datetime.datetime.now(tz=pytz.timezone('Asia/Shanghai'))}")
+ if not opt.debug:
+ melk()
+ raise
+ else:
+ trainer.validate(model, data, ckpt_path=ckpt_resume_path)
+ exit()
+ if not opt.no_test and not trainer.interrupted:
+ trainer.test(model, data)
+ except RuntimeError as err:
+ if MULTINODE_HACKS:
+ import datetime
+ import os
+ import socket
+ import requests
+
+ device = os.environ.get("CUDA_VISIBLE_DEVICES", "?")
+ hostname = socket.gethostname()
+ ts = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+ resp = requests.get("http://169.254.169.254/latest/meta-data/instance-id")
+ print(
+ f"[bold red]\[main][/bold red] ERROR at {ts} on {hostname}/{resp.text} (CUDA_VISIBLE_DEVICES={device}): {type(err).__name__}: {err}",
+ flush=True,
+ )
+ raise err
+ except Exception:
+ if opt.debug and trainer.global_rank == 0:
+ try:
+ import pudb as debugger
+ except ImportError:
+ import pdb as debugger
+ # debugger.post_mortem()
+ raise
+ finally:
+ # move newly created debug project to debug_runs
+ if opt.debug and not opt.resume and trainer.global_rank == 0:
+ dst, name = os.path.split(logdir)
+ dst = os.path.join(dst, "debug_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ os.rename(logdir, dst)
+
+ if opt.wandb:
+ wandb.finish()
+
+ # clean up
+ # dist.barrier()
+ # torch.cuda.empty_cache()
+ dist.destroy_process_group()
+
+ if trainer.global_rank == 0 and opt.debug:
+ print0(f"[bold red]\[main][/bold red] Current logdir: {logdir}")
+ # print0(f"[bold red]\[main][/bold red] Profiler summary:")
+ # print(trainer.profiler.summary())
+ print0(f"[bold red]\[main][/bold red] Memory summary:")
+ num_params = sum([p.numel() for p in model.parameters()])
+ print0(f"[bold red]\[main][/bold red] Expected bf16 memory usage from params: {num_params * 2 / 1e9:.2f} GB")
+ print0(f"[bold red]\[main][/bold red] Current memory usage with model on device {torch.cuda.max_memory_allocated() / 1e9:.2f} GB")
+ # trainer.print(torch.cuda.memory_summary())
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c26fbccfd6bb7ce7052966d21ee38c2b0bc24ba7
Binary files /dev/null and b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_evaluate.py b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4152046727305af7b32e0a3e5728072b7085ca4
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_evaluate.py
@@ -0,0 +1,198 @@
+import argparse
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from torch import autocast
+from tqdm import tqdm
+
+import numpy as np
+import torch
+from einops import rearrange
+from lightning.pytorch import seed_everything
+
+from vidtok.data.vidtok import VidTokValDataset
+from vidtok.modules.lpips import LPIPS
+from vidtok.modules.util import (compute_psnr, compute_ssim,
+ instantiate_from_config, print0)
+
+
+def load_model_from_config(config, ckpt, ignore_keys=[], verbose=False):
+ config = OmegaConf.load(config)
+ config.model.params.ckpt_path = ckpt
+ config.model.params.ignore_keys = ignore_keys
+ config.model.params.verbose = verbose
+ model = instantiate_from_config(config.model)
+ return model
+
+
+class MultiVideoDataset(VidTokValDataset):
+ def __init__(
+ self,
+ data_dir,
+ meta_path=None,
+ input_height=256,
+ input_width=256,
+ sample_fps=30,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ super().__init__(
+ data_dir=data_dir,
+ meta_path=meta_path,
+ video_params={
+ "input_height": input_height,
+ "input_width": input_width,
+ "sample_num_frames": chunk_size + 1 if is_causal else chunk_size,
+ "sample_fps": sample_fps,
+ },
+ pre_load_frames=True,
+ last_frames_handle="repeat",
+ read_long_video=read_long_video,
+ chunk_size=chunk_size,
+ is_causal=is_causal,
+ )
+
+ def __getitem__(self, idx):
+ frames = super().__getitem__(idx)["jpg"]
+ return frames
+
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--data_dir",
+ type=str,
+ default="./",
+ help="root folder",
+ )
+ parser.add_argument(
+ "--meta_path",
+ type=str,
+ default=None,
+ help="path to the .csv meta file",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.enable_tiling()
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+
+ if args.input_width > 256:
+ model.enable_tiling()
+
+ dataset = MultiVideoDataset(
+ data_dir=args.data_dir,
+ meta_path=args.meta_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ perceptual_loss = LPIPS().eval()
+ perceptual_loss = perceptual_loss.to(device)
+
+ psnrs, ssims, lpipss = [], [], []
+
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+ _, output, reg_log = model(input)
+ output = output.clamp(-1, 1)
+ input, output = map(lambda x: (x + 1) / 2, (input, output))
+
+ if input.dim() == 5:
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ assert output.dim() == 5
+ output = rearrange(output, "b c t h w -> (b t) c h w")
+
+ for inp, out in zip(torch.split(input, 16), torch.split(output, 16)):
+ psnrs += [compute_psnr(inp, out).item()] * inp.shape[0]
+ ssims += [compute_ssim(inp, out).item()] * inp.shape[0]
+ lpipss += [perceptual_loss(inp * 2 - 1, out * 2 - 1).mean().item()] * inp.shape[0]
+
+ toc = time.time()
+ print0(
+ f"[bold red]\[scripts.inference_evaluate][/bold red] PSNR: {np.mean(psnrs):.4f}, SSIM: {np.mean(ssims):.4f}, LPIPS: {np.mean(lpipss):.4f}"
+ )
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_reconstruct.py b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_reconstruct.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a26b9475339f6675e01fd052637c8465ca37caf
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/scripts/inference_reconstruct.py
@@ -0,0 +1,246 @@
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import argparse
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from pathlib import Path
+from tqdm import tqdm
+
+import numpy as np
+import torch
+import decord
+from einops import rearrange
+from lightning.pytorch import seed_everything
+from torch import autocast
+from torchvision import transforms
+from torchvision.io import write_video
+
+from vidtok.modules.util import print0
+from scripts.inference_evaluate import load_model_from_config
+
+
+class SingleVideoDataset(torch.utils.data.Dataset):
+ def __init__(
+ self,
+ video_path,
+ input_height=128,
+ input_width=128,
+ sample_fps=8,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ decord.bridge.set_bridge("torch")
+ self.video_path = video_path
+ self.transform = transforms.Compose(
+ [
+ transforms.Resize(input_height, antialias=True),
+ transforms.CenterCrop((input_height, input_width)),
+ transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
+ ]
+ )
+
+ self.video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(self.video_reader)
+ fps = self.video_reader.get_avg_fps() # float
+
+ interval = round(fps / sample_fps)
+ frame_ids = list(range(0, total_frames, interval))
+ self.frame_ids_batch = []
+ if read_long_video:
+ video_length = len(frame_ids)
+ if is_causal and video_length > chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * ((video_length - 1) // chunk_size) + 1])
+ elif not is_causal and video_length >= chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * (video_length // chunk_size)])
+ else:
+ num_frames_per_batch = chunk_size + 1 if is_causal else chunk_size
+ for x in range(0, len(frame_ids), num_frames_per_batch):
+ if len(frame_ids[x : x + num_frames_per_batch]) == num_frames_per_batch:
+ self.frame_ids_batch.append(frame_ids[x : x + num_frames_per_batch])
+
+ def __len__(self):
+ return len(self.frame_ids_batch)
+
+ def __getitem__(self, idx):
+ frame_ids = self.frame_ids_batch[idx]
+ frames = self.video_reader.get_batch(frame_ids).permute(0, 3, 1, 2).float() / 255.0
+ frames = self.transform(frames).permute(1, 0, 2, 3)
+ return frames
+
+
+def tensor_to_uint8(tensor):
+ tensor = torch.clamp(tensor, -1.0, 1.0)
+ tensor = (tensor + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ tensor = (tensor.cpu().numpy() * 255).astype(np.uint8)
+ return tensor
+
+
+def main():
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--output_video_dir",
+ type=str,
+ default="tmp",
+ help="path to save the outputs",
+ )
+ parser.add_argument(
+ "--input_video_path",
+ type=str,
+ default="assets/example.mp4",
+ help="path to the input video",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+ parser.add_argument(
+ "--pad_gen_frames",
+ action="store_true",
+ help="Used only in causal mode. If True, pad frames generated in the last batch, else replicate the first frame instead",
+ )
+ parser.add_argument(
+ "--concate_input",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="",
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+ config = OmegaConf.load(args.config)
+
+ os.makedirs(args.output_video_dir, exist_ok=True)
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.use_tiling = True
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+ model.use_overlap = True
+
+ dataset = SingleVideoDataset(
+ video_path=args.input_video_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ inputs = []
+ outputs = []
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+
+ if model.is_causal and not args.read_long_video and args.pad_gen_frames:
+ if i == 0:
+ _, xrec, _ = model(input)
+ else:
+ _, xrec, _ = model(torch.cat([last_gen_frames, input], dim=2))
+ xrec = xrec[:, :, -input.shape[2]:].clamp(-1, 1)
+ last_gen_frames = xrec[:, :, (1 - model.encoder.time_downsample_factor):, :, :]
+ else:
+ _, xrec, _ = model(input)
+
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ inputs.append(input)
+ xrec = rearrange(xrec.clamp(-1, 1), "b c t h w -> (b t) c h w")
+ outputs.append(xrec)
+
+ toc = time.time()
+
+ # save the outputs as videos
+ inputs = tensor_to_uint8(torch.cat(inputs, dim=0))
+ inputs = rearrange(inputs, "t c h w -> t h w c")
+ outputs = tensor_to_uint8(torch.cat(outputs, dim=0))
+ outputs = rearrange(outputs, "t c h w -> t h w c")
+ min_len = min(inputs.shape[0], outputs.shape[0])
+ final = np.concatenate([inputs[:min_len], outputs[:min_len]], axis=2) if args.concate_input else outputs[:min_len]
+
+ output_video_path = os.path.join(args.output_video_dir, f"{Path(args.input_video_path).stem}_reconstructed.mp4")
+ write_video(output_video_path, final, args.sample_fps)
+
+ print0(f"[bold red]Results saved in: {output_video_path}[/bold red]")
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1f33d1eb5f9eb0a67085dce41389212b2f3e07c5
Binary files /dev/null and b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/datamodule.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/datamodule.py
new file mode 100644
index 0000000000000000000000000000000000000000..c405b84703735b9a076b9ffb4e74de673470c15b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/datamodule.py
@@ -0,0 +1,150 @@
+import numpy as np
+from functools import partial
+
+import torch
+import lightning.pytorch as pl
+from torch.utils.data import DataLoader, Dataset, IterableDataset
+
+from vidtok.modules.util import instantiate_from_config
+
+
+class WrappedDataset(Dataset):
+ """Wraps an arbitrary object with __len__ and __getitem__ into a pytorch dataset"""
+
+ def __init__(self, dataset):
+ self.data = dataset
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, idx):
+ return self.data[idx]
+
+
+def worker_init_fn(_):
+ worker_info = torch.utils.data.get_worker_info()
+
+ dataset = worker_info.dataset
+ worker_id = worker_info.id
+
+ if isinstance(dataset, IterableDataset):
+ split_size = dataset.num_records // worker_info.num_workers
+ # reset num_records to the true number to retain reliable length information
+ dataset.sample_ids = dataset.valid_ids[
+ worker_id * split_size : (worker_id + 1) * split_size
+ ]
+ current_id = np.random.choice(len(np.random.get_state()[1]), 1)
+ return np.random.seed(np.random.get_state()[1][current_id] + worker_id)
+ else:
+ return np.random.seed(np.random.get_state()[1][0] + worker_id)
+
+
+class DataModuleFromConfig(pl.LightningDataModule):
+ def __init__(
+ self,
+ batch_size,
+ train=None,
+ validation=None,
+ test=None,
+ predict=None,
+ wrap=False,
+ num_workers=None,
+ pin_train_memory=True,
+ is_iterable_dataset=False,
+ shuffle_test_loader=False,
+ use_worker_init_fn=False,
+ shuffle_val_dataloader=False,
+ ):
+ super().__init__()
+ self.batch_size = batch_size
+ self.dataset_configs = dict()
+ self.num_workers = num_workers if num_workers is not None else batch_size * 2
+ self.pin_train_memory = pin_train_memory
+ self.is_iterable_dataset = is_iterable_dataset
+ self.use_worker_init_fn = use_worker_init_fn
+ if train is not None:
+ self.dataset_configs["train"] = train
+ self.train_dataloader = self._train_dataloader
+ if validation is not None:
+ self.dataset_configs["validation"] = validation
+ self.val_dataloader = partial(
+ self._val_dataloader, shuffle=shuffle_val_dataloader
+ )
+ if test is not None:
+ self.dataset_configs["test"] = test
+ self.test_dataloader = partial(
+ self._test_dataloader, shuffle=shuffle_test_loader
+ )
+ if predict is not None:
+ self.dataset_configs["predict"] = predict
+ self.predict_dataloader = self._predict_dataloader
+ self.wrap = wrap
+
+ def prepare_data(self):
+ for data_cfg in self.dataset_configs.values():
+ instantiate_from_config(data_cfg)
+
+ def setup(self, stage=None):
+ self.datasets = dict(
+ (k, instantiate_from_config(self.dataset_configs[k]))
+ for k in self.dataset_configs
+ )
+ if self.wrap:
+ for k in self.datasets:
+ self.datasets[k] = WrappedDataset(self.datasets[k])
+
+ def _train_dataloader(self):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["train"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ pin_memory=self.pin_train_memory,
+ shuffle=False if self.is_iterable_dataset else True,
+ worker_init_fn=init_fn,
+ )
+
+ def _val_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["validation"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _test_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+
+ # do not shuffle dataloader for iterable dataset
+ shuffle = shuffle and (not self.is_iterable_dataset)
+
+ return DataLoader(
+ self.datasets["test"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _predict_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["predict"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ )
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/video_read.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/video_read.py
new file mode 100644
index 0000000000000000000000000000000000000000..357cd48305141ee70924fd6a79adbe5cb7f6ca5b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/video_read.py
@@ -0,0 +1,88 @@
+import os
+import random
+import decord
+import numpy as np
+import torch
+
+from vidtok.modules.util import print0
+
+decord.bridge.set_bridge("torch")
+
+
+def sample_frames_with_fps(
+ total_frames,
+ video_fps,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+):
+ """sample frames proportional to the length of the frames in one second
+ e.g., 1s video has 30 frames, when 'fps'=3, we sample frames with spacing of 30/3=10
+ return the frame indices
+
+ Parameters
+ ----------
+ total_frames : length of the video
+ video_fps : original fps of the video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frame indices
+ """
+ sample_num_frames = min(sample_num_frames, total_frames)
+ interval = round(video_fps / sample_fps)
+ frames_range = (sample_num_frames - 1) * interval + 1
+
+ if start_index is not None:
+ start = start_index
+ elif total_frames - frames_range - 1 < 0:
+ start = 0
+ else:
+ start = random.randint(0, total_frames - frames_range - 1)
+
+ frame_idxs = np.linspace(
+ start=start, stop=min(total_frames - 1, start + frames_range), num=sample_num_frames
+ ).astype(int)
+
+ return frame_idxs
+
+
+def read_frames_with_decord(
+ video_path,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+) -> tuple[torch.Tensor, list[int]]:
+ """read frames from video path using decord
+
+ Parameters
+ ----------
+ video_path : path to video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frames (tensor 0~1), frame indices
+ """
+ video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(video_reader)
+ video_fps = video_reader.get_avg_fps() # note that the fps here is float.
+ frame_idxs = sample_frames_with_fps(
+ total_frames=total_frames,
+ video_fps=video_fps,
+ sample_num_frames=sample_num_frames,
+ sample_fps=sample_fps,
+ start_index=start_index
+ )
+ frames = video_reader.get_batch(frame_idxs)
+ frames = frames.float() / 255
+ frames = frames.permute(0, 3, 1, 2)
+ if (frames.shape[0] != sample_num_frames) or (len(frame_idxs) != sample_num_frames):
+ print0(f"[bold yellow]\[vidtok.data.video_read][read_frames_with_decord][/bold yellow] Warning: need {sample_num_frames} frames, "
+ f"but got {frames.shape[0]} frames, {len(frame_idxs)} frame indices, video_path={video_path}.")
+ return frames, frame_idxs
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/vidtok.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/vidtok.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b1898d4718de1450da96f25e9134dd4e1084cc1
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/data/vidtok.py
@@ -0,0 +1,333 @@
+import os
+import glob
+from typing import Union
+
+import decord
+import numpy as np
+import pandas as pd
+import torch
+from PIL import Image
+from torch.utils.data import Dataset
+from torchvision.transforms import v2
+from tqdm import trange
+
+from vidtok.modules.util import print0
+from .video_read import read_frames_with_decord
+
+
+class VidTokDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ meta_path: str,
+ video_params: dict,
+ data_frac: float = 1.0,
+ is_strict_loading: bool = False,
+ skip_missing_files: bool = True,
+ start_index: Union[None, int] = None
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use data dir: {self.data_dir}")
+
+ self.meta_path = meta_path
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use meta path: {self.meta_path}")
+
+ self.video_params = video_params
+
+ self.data_frac = data_frac
+ self.is_strict_loading = is_strict_loading
+ self.skip_missing_files = skip_missing_files
+ self.start_index = start_index
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",)
+
+ if self.data_frac < 1:
+ metadata = metadata.sample(frac=self.data_frac)
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk
+ """
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.metadata)
+
+ def __getitem__(self, item):
+ item = item % len(self.metadata)
+ sample = self.metadata.iloc[item]
+ video_fp, _ = self._get_video_path(sample)
+
+ try:
+ if os.path.isfile(video_fp):
+ imgs, idxs = read_frames_with_decord(
+ video_path=video_fp,
+ sample_num_frames=self.video_params["sample_num_frames"],
+ sample_fps=self.video_params["sample_fps"],
+ start_index=self.start_index
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video.")
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}.") from e
+ else:
+ print0("[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: using the pure black image as the frame sample")
+ imgs = Image.new("RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0))
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ # imgs: (T, C, H, W)
+ imgs = self.transforms(imgs)
+
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ imgs = torch.cat([imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)], dim=0)
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ 'jpg': imgs,
+ "path": video_fp
+ }
+
+
+class VidTokValDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ video_params: dict,
+ meta_path: Union[None, str] = None,
+ pre_load_frames: bool = True,
+ is_strict_loading: bool = True,
+ last_frames_handle: str = "repeat", # 'repeat', 'drop'
+ skip_missing_files: bool = False,
+ read_long_video: bool = False,
+ chunk_size: int = 16,
+ is_causal: bool = True,
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use data dir: {self.data_dir}"
+ )
+
+ self.meta_path = meta_path
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use meta path: {self.meta_path}"
+ )
+
+ self.video_params = video_params
+ self.read_long_video = read_long_video
+ self.chunk_size = chunk_size
+ self.is_causal = is_causal
+
+ self.is_strict_loading = is_strict_loading
+ self.last_frames_handle = last_frames_handle
+ self.skip_missing_files = skip_missing_files
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+ self._load_every_frame_from_meta()
+
+ if pre_load_frames:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Pre-loading all frames into CPU..."
+ )
+ self._pre_load_frames()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ if self.meta_path is not None:
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",
+ )
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+ else:
+ self.metadata = glob.glob(os.path.join(self.data_dir, '**', '*.mp4'), recursive=True)
+
+ def _load_every_frame_from_meta(self):
+ decord.bridge.set_bridge("torch")
+ self.frames_batch = []
+ for video_idx in range(len(self.metadata)):
+ try:
+ sample = self.metadata.iloc[video_idx]
+ video_fp, _ = self._get_video_path(sample)
+ except:
+ video_fp = self.metadata[video_idx]
+ if os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ total_frames = len(video_reader)
+ fps = video_reader.get_avg_fps() # float
+ interval = round(fps / self.video_params["sample_fps"])
+ frame_ids = list(range(0, total_frames, interval))
+
+ if self.read_long_video:
+ video_length = len(frame_ids)
+ if self.is_causal and video_length > self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * ((video_length - 1) // self.chunk_size) + 1]
+ elif not self.is_causal and video_length >= self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * (video_length // self.chunk_size)]
+ else:
+ continue
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ else:
+ for x in range(0, len(frame_ids), self.video_params["sample_num_frames"]):
+ num_frames_ids = frame_ids[x : x + self.video_params["sample_num_frames"]]
+ if len(num_frames_ids) < self.video_params["sample_num_frames"]:
+ if self.last_frames_handle == "repeat":
+ num_frames_ids += [num_frames_ids[-1]] * (
+ self.video_params["sample_num_frames"] - len(num_frames_ids)
+ )
+ elif self.last_frames_handle == "drop":
+ continue
+ else:
+ raise ValueError(f"Invalid last_frames_handle: {self.last_frames_handle}")
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Loaded all frames index from {len(self.metadata)} videos."
+ )
+
+ def _pre_load_frames(self):
+ last_video_fp = None
+ for idx in trange(len(self.frames_batch), desc="Pre-loading all frames"):
+ if self.frames_batch[idx]["video_fp"] != last_video_fp:
+ video_reader = decord.VideoReader(self.frames_batch[idx]["video_fp"], num_threads=0)
+ last_video_fp = self.frames_batch[idx]["video_fp"]
+ self.frames_batch[idx]["frames"] = (
+ video_reader.get_batch(self.frames_batch[idx]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk"""
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.frames_batch)
+
+ def __getitem__(self, item):
+ video_fp = self.frames_batch[item]["video_fp"]
+
+ try:
+ if "frames" in self.frames_batch[item]:
+ imgs = self.frames_batch[item]["frames"]
+ elif os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ imgs = (
+ video_reader.get_batch(self.frames_batch[item]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video."
+ )
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(
+ f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}."
+ ) from e
+ else:
+ print0(
+ "[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: using the pure black image as the frame sample"
+ )
+ imgs = Image.new(
+ "RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0)
+ )
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ imgs = self.transforms(imgs)
+
+ if not self.read_long_video:
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: video {video_fp} has less frames {imgs.shape[0]} than sample_num_frames {self.video_params['sample_num_frames']}."
+ )
+ imgs = torch.cat(
+ [imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)],
+ dim=0,
+ )
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ "jpg": imgs,
+ "path": video_fp,
+ }
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..96da5e6c74621b4f82538453e850567245adbbb0
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder.py
@@ -0,0 +1,517 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+ self.use_tiling = False
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ token_indices = rearrange(token_indices, "... -> ... 1")
+ token_indices, ps = pack_one(token_indices, "b * d")
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = rearrange(codes, "b d n c -> b n (c d)")
+ z = self.regularization.project_out(codes)
+ z = unpack_one(z, ps, "b * d")
+ z = rearrange(z, "b ... d -> b d ...")
+ return z
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ z = self.indices_to_latent(z)
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ tile = z[
+ :,
+ :,
+ start_frame : end_frame,
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+
+ dec = self.decode(z)
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c1182f573eee05b634daeb9a065b7adaacc600b
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py
@@ -0,0 +1,588 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ use_tiling: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ self.use_tiling = kwargs.pop("use_tiling", False)
+ self.t_chunk_enc = kwargs.pop("t_chunk_enc", 16)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+
+ self.t_chunk_dec = self.t_chunk_enc // self.encoder.time_downsample_factor
+ self.use_overlap = False
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+
+ self.use_tiling = use_tiling
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def _empty_causal_cached(self, parent):
+ for name, module in parent.named_modules():
+ if hasattr(module, 'causal_cache'):
+ module.causal_cache = None
+
+ def _set_first_chunk(self, is_first_chunk=True):
+ for module in self.modules():
+ if hasattr(module, 'is_first_chunk'):
+ module.is_first_chunk = is_first_chunk
+
+ def _set_cache_offset(self, modules, cache_offset=0):
+ for module in modules:
+ for submodule in module.modules():
+ if hasattr(submodule, 'cache_offset'):
+ submodule.cache_offset = cache_offset
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def build_chunk_start_end(self, t, decoder_mode=False):
+ start_end = [[0, 1]]
+ start = 1
+ end = start
+ while True:
+ if start >= t:
+ break
+ end = min(t, end + (self.t_chunk_dec if decoder_mode else self.t_chunk_enc))
+ start_end.append([start, end])
+ start = end
+ return start_end
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ self._empty_causal_cached(self.encoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = self.build_chunk_start_end(num_frames)
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ assert token_indices.dim() == 4, "token_indices should be of shape (b, t, h, w)"
+ b, t, h, w = token_indices.shape
+ token_indices = token_indices.unsqueeze(-1).reshape(b, -1, 1)
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = codes.permute(0, 2, 3, 1).reshape(b, codes.shape[2], -1)
+ z = self.regularization.project_out(codes)
+ return z.reshape(b, t, h, w, -1).permute(0, 4, 1, 2, 3)
+
+ def tile_indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ num_frames = token_indices.shape[1]
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ result_z = []
+ for (start, end) in start_end:
+ chunk = token_indices[:, start:end, :, :]
+ chunk_z = self.indices_to_latent(chunk)
+ result_z.append(chunk_z.clone())
+ return torch.cat(result_z, dim=2)
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ if self.use_tiling:
+ z = self.tile_indices_to_latent(z)
+ else:
+ z = self.indices_to_latent(z)
+ self._empty_causal_cached(self.decoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ if self.is_causal:
+ assert self.encoder.time_downsample_factor in [2, 4, 8], "Only support 2x, 4x or 8x temporal downsampling now."
+ if self.encoder.time_downsample_factor == 4:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 4)
+ elif self.encoder.time_downsample_factor == 2:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1], self.decoder.up_temporal[0], self.decoder.conv_out], 2)
+ else:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[3].upsample, self.decoder.up_temporal[2]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 4)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 8)
+
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = z[
+ :,
+ :,
+ start_frame : (end_frame + 1 if self.is_causal and end_frame + 1 <= num_frames else end_frame),
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+ dec = self.decode(z)
+ if dec.shape[2] != x.shape[2]:
+ dec = dec[:, :, -x.shape[2]:, ...]
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
\ No newline at end of file
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2441aebdb3844a7ef2103c501213cd4537abea6b
Binary files /dev/null and b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc differ
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/discriminator.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/discriminator.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9d94b21b22f5019f3cdcc4cbf2e98bd0ce0ee02
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/discriminator.py
@@ -0,0 +1,201 @@
+import functools
+
+import torch
+import torch.nn as nn
+
+
+def weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ nn.init.normal_(m.weight.data, 0.0, 0.02)
+ elif classname.find("BatchNorm") != -1:
+ nn.init.normal_(m.weight.data, 1.0, 0.02)
+ nn.init.constant_(m.bias.data, 0)
+
+
+class ActNorm(nn.Module):
+ def __init__(self, num_features, logdet=False, affine=True, allow_reverse_init=False):
+ assert affine
+ super().__init__()
+ self.logdet = logdet
+ self.loc = nn.Parameter(torch.zeros(1, num_features, 1, 1))
+ self.scale = nn.Parameter(torch.ones(1, num_features, 1, 1))
+ self.allow_reverse_init = allow_reverse_init
+
+ self.register_buffer("initialized", torch.tensor(0, dtype=torch.uint8))
+
+ def initialize(self, input):
+ with torch.no_grad():
+ flatten = input.permute(1, 0, 2, 3).contiguous().view(input.shape[1], -1)
+ mean = flatten.mean(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+ std = flatten.std(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+
+ self.loc.data.copy_(-mean)
+ self.scale.data.copy_(1 / (std + 1e-6))
+
+ def forward(self, input, reverse=False):
+ if reverse:
+ return self.reverse(input)
+ if len(input.shape) == 2:
+ input = input[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ _, _, height, width = input.shape
+
+ if self.training and self.initialized.item() == 0:
+ self.initialize(input)
+ self.initialized.fill_(1)
+
+ h = self.scale * (input + self.loc)
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+
+ if self.logdet:
+ log_abs = torch.log(torch.abs(self.scale))
+ logdet = height * width * torch.sum(log_abs)
+ logdet = logdet * torch.ones(input.shape[0]).to(input)
+ return h, logdet
+
+ return h
+
+ def reverse(self, output):
+ if self.training and self.initialized.item() == 0:
+ if not self.allow_reverse_init:
+ raise RuntimeError(
+ "Initializing ActNorm in reverse direction is "
+ "disabled by default. Use allow_reverse_init=True to enable."
+ )
+ else:
+ self.initialize(output)
+ self.initialized.fill_(1)
+
+ if len(output.shape) == 2:
+ output = output[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ h = output / self.scale - self.loc
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+ return h
+
+
+class NLayerDiscriminator(nn.Module):
+ """Defines a PatchGAN discriminator as in Pix2Pix."""
+ # https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py
+ def __init__(self, input_nc=3, ndf=64, n_layers=3, use_actnorm=False):
+ """Construct a PatchGAN discriminator
+ Parameters:
+ input_nc (int) -- the number of channels in input images
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ """
+ super(NLayerDiscriminator, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm2d
+ else:
+ norm_layer = ActNorm
+ if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters
+ use_bias = norm_layer.func != nn.BatchNorm2d
+ else:
+ use_bias = norm_layer != nn.BatchNorm2d
+
+ kw = 4
+ padw = 1
+ sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
+
+
+class NLayerDiscriminator3D(nn.Module):
+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/losses/discriminator.py
+ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
+ """
+ Construct a 3D PatchGAN discriminator
+
+ Parameters:
+ input_nc (int) -- the number of channels in input volumes
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm
+ """
+ super(NLayerDiscriminator3D, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm3d
+ else:
+ raise NotImplementedError("Not implemented.")
+ if type(norm_layer) == functools.partial:
+ use_bias = norm_layer.func != nn.BatchNorm3d
+ else:
+ use_bias = norm_layer != nn.BatchNorm3d
+
+ kw = 3
+ padw = 1
+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev,
+ ndf * nf_mult,
+ kernel_size=(kw, kw, kw),
+ stride=(2 if n == 1 else 1, 2, 2),
+ padding=padw,
+ bias=use_bias,
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/distributions.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/distributions.py
new file mode 100644
index 0000000000000000000000000000000000000000..76e814475d4d32b9f5ead736cce3a234bb5a0e5f
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/distributions.py
@@ -0,0 +1,49 @@
+import numpy as np
+import torch
+
+
+class DiagonalGaussianDistribution(object):
+ def __init__(self, parameters, deterministic=False):
+ self.parameters = parameters
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
+ self.deterministic = deterministic
+ self.std = torch.exp(0.5 * self.logvar)
+ self.var = torch.exp(self.logvar)
+ if self.deterministic:
+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device)
+
+ def sample(self):
+ x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device)
+ return x
+
+ def kl(self, other=None):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ else:
+ if other is None:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar,
+ dim=[1, 2, 3],
+ )
+ else:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean - other.mean, 2) / other.var
+ + self.var / other.var
+ - 1.0
+ - self.logvar
+ + other.logvar,
+ dim=[1, 2, 3],
+ )
+
+ def nll(self, sample, dims=[1, 2, 3]):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ logtwopi = np.log(2.0 * np.pi)
+ return 0.5 * torch.sum(
+ logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
+ dim=dims,
+ )
+
+ def mode(self):
+ return self.mean
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/ema.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/ema.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f1f7606c2c9b68ebd2302215a9e08f9f31ed8ab
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/ema.py
@@ -0,0 +1,82 @@
+import torch
+from torch import nn
+
+
+class LitEma(nn.Module):
+ def __init__(self, model, decay=0.9999, use_num_upates=True):
+ super().__init__()
+ if decay < 0.0 or decay > 1.0:
+ raise ValueError("Decay must be between 0 and 1")
+
+ self.m_name2s_name = {}
+ self.register_buffer("decay", torch.tensor(decay, dtype=torch.float32))
+ self.register_buffer(
+ "num_updates",
+ torch.tensor(0, dtype=torch.int) if use_num_upates else torch.tensor(-1, dtype=torch.int),
+ )
+
+ for name, p in model.named_parameters():
+ if p.requires_grad:
+ # remove as '.'-character is not allowed in buffers
+ s_name = name.replace(".", "")
+ self.m_name2s_name.update({name: s_name})
+ self.register_buffer(s_name, p.clone().detach().data)
+
+ self.collected_params = []
+
+ def reset_num_updates(self):
+ del self.num_updates
+ self.register_buffer("num_updates", torch.tensor(0, dtype=torch.int))
+
+ def forward(self, model):
+ decay = self.decay
+
+ if self.num_updates >= 0:
+ self.num_updates += 1
+ decay = min(self.decay, (1 + self.num_updates) / (10 + self.num_updates))
+
+ one_minus_decay = 1.0 - decay
+
+ with torch.no_grad():
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+
+ for key in m_param:
+ if m_param[key].requires_grad:
+ sname = self.m_name2s_name[key]
+ shadow_params[sname] = shadow_params[sname].type_as(m_param[key])
+ shadow_params[sname].sub_(one_minus_decay * (shadow_params[sname] - m_param[key]))
+ else:
+ assert not key in self.m_name2s_name
+
+ def copy_to(self, model):
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+ for key in m_param:
+ if m_param[key].requires_grad:
+ m_param[key].data.copy_(shadow_params[self.m_name2s_name[key]].data)
+ else:
+ assert not key in self.m_name2s_name
+
+ def store(self, parameters):
+ """
+ Save the current parameters for restoring later.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ temporarily stored.
+ """
+ self.collected_params = [param.clone() for param in parameters]
+
+ def restore(self, parameters):
+ """
+ Restore the parameters stored with the `store` method.
+ Useful to validate the model with EMA parameters without affecting the
+ original optimization process. Store the parameters before the
+ `copy_to` method. After validation (or model saving), use this to
+ restore the former parameters.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ updated with the stored parameters.
+ """
+ for c_param, param in zip(self.collected_params, parameters):
+ param.data.copy_(c_param.data)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/logger.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..23c7d066e6508433cb2141c59dbd80cb0030ab6d
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/logger.py
@@ -0,0 +1,289 @@
+import os
+import numpy as np
+import einops
+import imageio
+from typing import Union
+from matplotlib import pyplot as plt
+from PIL import Image, ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True # UnidentifiedImageError: https://github.com/python-pillow/Pillow/issues/5631
+from pathlib import Path
+
+import torch
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from .util import exists, isheatmap
+
+
+class ImageVideoLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ batch_frequency_val=None,
+ video_fps=8,
+ rescale=True,
+ disabled=False,
+ log_on_batch_idx=True, # log on batch_idx instead of global_step. global_step is fixed in validation. batch_idx restarts at each validation
+ log_first_step=True,
+ log_images_kwargs=None,
+ log_videos_kwargs=None,
+ log_before_first_step=True,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.batch_freq_val = batch_frequency_val if batch_frequency_val is not None else batch_frequency
+ self.video_fps = video_fps
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_videos_kwargs = log_videos_kwargs if log_videos_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_img_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ else:
+ if images[k].ndim == 5:
+ images[k] = einops.rearrange(images[k], "b c t h w -> (b t) c h w")
+ nrow = self.log_images_kwargs.get("n_rows", 8)
+ grid = torchvision.utils.make_grid(images[k], nrow=nrow)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_vid_local(
+ self,
+ save_dir,
+ split,
+ videos,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "videos", split)
+ for k in videos:
+ # if is video, we can add captions
+ if isinstance(videos[k], torch.Tensor) and videos[k].ndim == 5:
+ if self.rescale:
+ videos[k] = (videos[k] + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ frames = [videos[k][:, :, i] for i in range(videos[k].shape[2])]
+ frames = [torchvision.utils.make_grid(each, nrow=4) for each in frames]
+ frames = [einops.rearrange(each, "c h w -> 1 c h w") for each in frames]
+ frames = torch.clamp(torch.cat(frames, dim=0), min=0.0, max=1.0)
+ frames = (frames.numpy() * 255).astype(np.uint8)
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.gif".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ save_numpy_as_gif(frames, path, duration=1 / self.video_fps)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "log_videos only supports WandbLogger currently"
+ wandb.log({f"{split}/{k}": wandb.Video(frames, fps=self.video_fps)}) # k is str
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ images = pl_module.log_images(batch)
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_img_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ @rank_zero_only
+ def log_vid(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_videos") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_videos)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ videos = pl_module.log_videos(
+ batch, split=split, **self.log_videos_kwargs
+ )
+
+ for k in videos:
+ N = min(videos[k].shape[0], self.max_samples)
+ videos[k] = videos[k][:N]
+ if isinstance(videos[k], torch.Tensor):
+ videos[k] = videos[k].detach().float().cpu()
+ if self.clamp:
+ videos[k] = torch.clamp(videos[k], -1.0, 1.0)
+
+ self.log_vid_local(
+ pl_module.logger.save_dir,
+ split,
+ videos,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ pass
+ return True
+ return False
+
+ def check_frequency_val(self, check_idx, split):
+ if 'val' in split:
+ if ((check_idx % self.batch_freq_val) == 0) and (
+ check_idx > 0 or self.log_first_step):
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ self.log_vid(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+def save_numpy_as_gif(frames, path, duration=None):
+ """
+ save numpy array as gif file
+ """
+ image_list = []
+ for frame in frames:
+ image = frame.transpose(1, 2, 0)
+ image_list.append(image)
+ if duration:
+ imageio.mimsave(path, image_list, format="GIF", duration=duration, loop=0)
+ else:
+ imageio.mimsave(path, image_list, format="GIF", loop=0)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/losses.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..c96f471c72124c56025b40a17f7d7bda81446ca5
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/losses.py
@@ -0,0 +1,262 @@
+from typing import Any, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .discriminator import (NLayerDiscriminator, NLayerDiscriminator3D,
+ weights_init)
+from .lpips import LPIPS
+from .util import default, print0
+
+
+def hinge_d_loss(logits_real, logits_fake):
+ loss_real = torch.mean(F.relu(1.0 - logits_real))
+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))
+ d_loss = 0.5 * (loss_real + loss_fake)
+ return d_loss
+
+
+def vanilla_d_loss(logits_real, logits_fake):
+ d_loss = 0.5 * (torch.mean(F.softplus(-logits_real)) + torch.mean(F.softplus(logits_fake)))
+ return d_loss
+
+
+def adopt_weight(weight, global_step, threshold=0, value=0.0):
+ if global_step < threshold:
+ weight = value
+ return weight
+
+
+def _sigmoid_cross_entropy_with_logits(labels, logits):
+ """
+ non-saturating loss
+ """
+ zeros = torch.zeros_like(logits, dtype=logits.dtype)
+ condition = logits >= zeros
+ relu_logits = torch.where(condition, logits, zeros)
+ neg_abs_logits = torch.where(condition, -logits, logits)
+ return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
+
+
+def non_saturate_gen_loss(logits_fake):
+ """
+ logits_fake: [B 1 H W]
+ """
+ B = logits_fake.shape[0]
+ logits_fake = logits_fake.reshape(B, -1)
+ logits_fake = torch.mean(logits_fake, dim=-1)
+ gen_loss = torch.mean(_sigmoid_cross_entropy_with_logits(labels=torch.ones_like(logits_fake), logits=logits_fake))
+ return gen_loss
+
+
+def lecam_reg(real_pred, fake_pred, lecam_ema):
+ reg = torch.mean(F.relu(real_pred - lecam_ema.logits_fake_ema).pow(2)) + torch.mean(
+ F.relu(lecam_ema.logits_real_ema - fake_pred).pow(2)
+ )
+ return reg
+
+
+class LeCAM_EMA(object):
+ # https://github.com/TencentARC/SEED-Voken/blob/main/src/Open_MAGVIT2/modules/losses/vqperceptual.py
+ def __init__(self, init=0.0, decay=0.999):
+ self.logits_real_ema = init
+ self.logits_fake_ema = init
+ self.decay = decay
+
+ def update(self, logits_real, logits_fake):
+ self.logits_real_ema = self.logits_real_ema * self.decay + torch.mean(logits_real).item() * (1 - self.decay)
+ self.logits_fake_ema = self.logits_fake_ema * self.decay + torch.mean(logits_fake).item() * (1 - self.decay)
+
+
+class GeneralLPIPSWithDiscriminator(nn.Module):
+ def __init__(
+ self,
+ disc_start: int,
+ logvar_init: float = 0.0,
+ pixelloss_weight=1.0,
+ disc_num_layers: int = 3,
+ disc_in_channels: int = 3,
+ disc_factor: float = 1.0,
+ disc_weight: float = 1.0,
+ disc_type: str = "3d",
+ perceptual_weight: float = 1.0,
+ lecam_loss_weight: float = 0.0,
+ disc_loss: str = "hinge",
+ scale_input_to_tgt_size: bool = False,
+ dims: int = 2,
+ learn_logvar: bool = False,
+ regularization_weights: Union[None, dict] = None,
+ gen_loss_cross_entropy: bool = False,
+ ):
+ super().__init__()
+ self.dims = dims
+ if self.dims > 2:
+ print0(
+ f"[bold cyan]\[vidtok.modules.losses][GeneralLPIPSWithDiscriminator][/bold cyan] running with dims={dims}. This means that for perceptual loss calculation, "
+ f"the LPIPS loss will be applied to each frame independently. "
+ )
+ self.scale_input_to_tgt_size = scale_input_to_tgt_size
+ assert disc_loss in ["hinge", "vanilla"]
+ self.pixel_weight = pixelloss_weight
+ self.perceptual_loss = LPIPS().eval()
+ self.perceptual_weight = perceptual_weight
+ # output log variance
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+ self.learn_logvar = learn_logvar
+ self.disc_type = disc_type
+ assert self.disc_type in ["2d", "3d"]
+
+ if self.disc_type == "2d":
+ self.discriminator = NLayerDiscriminator(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ else:
+ self.discriminator = NLayerDiscriminator3D(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ self.discriminator_iter_start = disc_start
+ self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss
+ self.disc_factor = disc_factor
+ self.discriminator_weight = disc_weight
+ self.regularization_weights = default(regularization_weights, {})
+ self.gen_loss_cross_entropy = gen_loss_cross_entropy
+ self.lecam_loss_weight = lecam_loss_weight
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema = LeCAM_EMA()
+
+ def get_trainable_parameters(self) -> Any:
+ return self.discriminator.parameters()
+
+ def get_trainable_autoencoder_parameters(self) -> Any:
+ if self.learn_logvar:
+ yield self.logvar
+ yield from ()
+
+ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):
+ if last_layer is not None:
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
+ else:
+ nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0]
+
+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
+ d_weight = d_weight * self.discriminator_weight
+ return d_weight
+
+ def forward(
+ self,
+ regularization_log,
+ inputs,
+ reconstructions,
+ optimizer_idx,
+ global_step,
+ last_layer=None,
+ split="train",
+ weights=None,
+ ):
+ if self.scale_input_to_tgt_size:
+ inputs = torch.nn.functional.interpolate(inputs, reconstructions.shape[2:], mode="bicubic", antialias=True)
+
+ if optimizer_idx == 0:
+ bs = inputs.shape[0]
+ t = inputs.shape[2]
+ if self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
+ if self.perceptual_weight > 0:
+ p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())
+ rec_loss = rec_loss + self.perceptual_weight * p_loss
+ else:
+ p_loss = torch.Tensor([0.0])
+
+ nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if weights is not None:
+ weighted_nll_loss = weights * nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+
+ # now the GAN part
+ if self.disc_type == "3d":
+ reconstructions = rearrange(reconstructions, "(b t) c h w -> b c t h w", t=t).contiguous()
+
+ # generator update
+ logits_fake = self.discriminator(reconstructions)
+
+ if not self.gen_loss_cross_entropy:
+ g_loss = -torch.mean(logits_fake)
+ else:
+ g_loss = non_saturate_gen_loss(logits_fake)
+
+ if self.disc_factor > 0.0:
+ try:
+ d_weight = self.calculate_adaptive_weight(nll_loss, g_loss, last_layer=last_layer)
+ except RuntimeError:
+ assert not self.training
+ d_weight = torch.tensor(0.0)
+ else:
+ d_weight = torch.tensor(0.0)
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+ loss = weighted_nll_loss + d_weight * disc_factor * g_loss
+ log = dict()
+ for k in regularization_log:
+ if k in self.regularization_weights:
+ loss = loss + self.regularization_weights[k] * regularization_log[k]
+ log[f"{split}/{k}"] = regularization_log[k].detach().mean()
+
+ log.update(
+ {
+ "{}/total_loss".format(split): loss.clone().detach().mean(),
+ "{}/logvar".format(split): self.logvar.detach(),
+ "{}/nll_loss".format(split): nll_loss.detach().mean(),
+ "{}/rec_loss".format(split): rec_loss.detach().mean(),
+ "{}/p_loss".format(split): p_loss.detach().mean(),
+ "{}/d_weight".format(split): d_weight.detach(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/g_loss".format(split): g_loss.detach().mean(),
+ }
+ )
+ return loss, log
+
+ if optimizer_idx == 1:
+ if self.disc_type == "2d" and self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ logits_real = self.discriminator(inputs.contiguous().detach())
+ logits_fake = self.discriminator(reconstructions.contiguous().detach())
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+
+ non_saturate_d_loss = self.disc_loss(logits_real, logits_fake)
+
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema.update(logits_real, logits_fake)
+ lecam_loss = lecam_reg(logits_real, logits_fake, self.lecam_ema)
+ d_loss = disc_factor * (lecam_loss * self.lecam_loss_weight + non_saturate_d_loss)
+ else:
+ d_loss = disc_factor * non_saturate_d_loss
+
+ log = {
+ "{}/disc_loss".format(split): d_loss.clone().detach().mean(),
+ "{}/logits_real".format(split): logits_real.detach().mean(),
+ "{}/logits_fake".format(split): logits_fake.detach().mean(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/non_saturated_d_loss".format(split): non_saturate_d_loss.detach(),
+ }
+
+ if self.lecam_loss_weight > 0:
+ log.update({"{}/lecam_loss".format(split): lecam_loss.detach()})
+
+ return d_loss, log
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/lpips.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/lpips.py
new file mode 100644
index 0000000000000000000000000000000000000000..22bb5fa315618c0d1f0463d67ea57c29083fe302
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/lpips.py
@@ -0,0 +1,172 @@
+import hashlib
+import os
+from collections import namedtuple
+from tqdm import tqdm
+
+import requests
+import torch
+import torch.nn as nn
+from torchvision import models
+
+from .util import print0
+
+URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"}
+
+CKPT_MAP = {"vgg_lpips": "vgg.pth"}
+
+MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"}
+
+
+def download(url, local_path, chunk_size=1024):
+ os.makedirs(os.path.split(local_path)[0], exist_ok=True)
+ with requests.get(url, stream=True) as r:
+ total_size = int(r.headers.get("content-length", 0))
+ with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
+ with open(local_path, "wb") as f:
+ for data in r.iter_content(chunk_size=chunk_size):
+ if data:
+ f.write(data)
+ pbar.update(chunk_size)
+
+
+def md5_hash(path):
+ with open(path, "rb") as f:
+ content = f.read()
+ return hashlib.md5(content).hexdigest()
+
+
+def get_ckpt_path(name, root, check=False):
+ assert name in URL_MAP
+ path = os.path.join(root, CKPT_MAP[name])
+ if os.path.exists(path) and not (check and not md5_hash(path) == MD5_MAP[name]):
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Using existing path for {} model: {}".format(
+ name, path
+ )
+ )
+ return path
+
+ # if not, download the model
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Downloading {} model from {} to {}".format(
+ name, URL_MAP[name], path
+ )
+ )
+ download(URL_MAP[name], path)
+ md5 = md5_hash(path)
+ assert md5 == MD5_MAP[name], md5
+ return path
+
+
+class LPIPS(nn.Module):
+ # Learned perceptual metric
+ def __init__(self, use_dropout=True):
+ super().__init__()
+ self.scaling_layer = ScalingLayer()
+ self.chns = [64, 128, 256, 512, 512] # vg16 features
+ self.net = vgg16(pretrained=True, requires_grad=False)
+ self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
+ self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
+ self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
+ self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)
+ self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)
+ self.load_from_pretrained()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def load_from_pretrained(self, name="vgg_lpips"):
+ ckpt = get_ckpt_path(name, "checkpoints/lpips")
+ self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
+ print0("[bold cyan]\[vidtok.modules.lpips][LPIPS][/bold cyan] loaded pretrained LPIPS loss from {}".format(ckpt))
+
+ def forward(self, input, target):
+ in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))
+ outs0, outs1 = self.net(in0_input), self.net(in1_input)
+ feats0, feats1, diffs = {}, {}, {}
+ lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
+ for kk in range(len(self.chns)):
+ feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])
+ diffs[kk] = (feats0[kk] - feats1[kk]) ** 2
+
+ res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]
+ val = res[0]
+ for l in range(1, len(self.chns)):
+ val += res[l]
+ return val
+
+
+class ScalingLayer(nn.Module):
+ def __init__(self):
+ super(ScalingLayer, self).__init__()
+ self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None])
+ self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None])
+
+ def forward(self, inp):
+ return (inp - self.shift) / self.scale
+
+
+class NetLinLayer(nn.Module):
+ """A single linear layer which does a 1x1 conv"""
+
+ def __init__(self, chn_in, chn_out=1, use_dropout=False):
+ super(NetLinLayer, self).__init__()
+ layers = (
+ [
+ nn.Dropout(),
+ ]
+ if (use_dropout)
+ else []
+ )
+ layers += [
+ nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False),
+ ]
+ self.model = nn.Sequential(*layers)
+
+
+class vgg16(torch.nn.Module):
+ def __init__(self, requires_grad=False, pretrained=True):
+ super(vgg16, self).__init__()
+ vgg_pretrained_features = models.vgg16(pretrained=pretrained).features
+ self.slice1 = torch.nn.Sequential()
+ self.slice2 = torch.nn.Sequential()
+ self.slice3 = torch.nn.Sequential()
+ self.slice4 = torch.nn.Sequential()
+ self.slice5 = torch.nn.Sequential()
+ self.N_slices = 5
+ for x in range(4):
+ self.slice1.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(4, 9):
+ self.slice2.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(9, 16):
+ self.slice3.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(16, 23):
+ self.slice4.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(23, 30):
+ self.slice5.add_module(str(x), vgg_pretrained_features[x])
+ if not requires_grad:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, X):
+ h = self.slice1(X)
+ h_relu1_2 = h
+ h = self.slice2(h)
+ h_relu2_2 = h
+ h = self.slice3(h)
+ h_relu3_3 = h
+ h = self.slice4(h)
+ h_relu4_3 = h
+ h = self.slice5(h)
+ h_relu5_3 = h
+ vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"])
+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
+ return out
+
+
+def normalize_tensor(x, eps=1e-10):
+ norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True))
+ return x / (norm_factor + eps)
+
+
+def spatial_average(x, keepdim=True):
+ return x.mean([2, 3], keepdim=keepdim)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..c71a8b09e370451046eef9ba60315feab2459e35
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py
@@ -0,0 +1,885 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+ self.time_causal_padding = (self.time_pad, 0)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (height_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ time_pad,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+ x1 = self.avg_pool(torch.nn.functional.pad(x, pad, mode="constant", value=0))
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest").to(
+ x.dtype
+ )
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ x = pad_at_dim(x, (self.time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in)
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x[:, :, self.time_padding :, :, :]
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..44397a135334a58cf5774fb899152f701d56a37d
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py
@@ -0,0 +1,959 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1].repeat(
+ (1, 1, self.time_pad)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (width_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.spatial_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ 0,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1, :, :].repeat(
+ (1, 1, self.time_pad, 1, 1)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ x = F.pad(x, self.spatial_padding, mode=self.pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+
+ if self.is_first_chunk:
+ x_pad = torch.nn.functional.pad(x, pad, mode="replicate")
+ else:
+ x_pad = torch.concatenate((self.causal_cache, x), dim=2)
+
+ self.causal_cache = x_pad[:,:,-1:].clone()
+
+ x1 = self.avg_pool(x_pad)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ interpolation_mode='nearest',
+ num_temp_upsample=1
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.interpolation_mode = interpolation_mode
+ self.num_temp_upsample = num_temp_upsample
+ self.enable_cached = (self.interpolation_mode == 'trilinear')
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ if not self.enable_cached:
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ elif not self.is_first_chunk:
+ x = torch.cat([self.causal_cache, x], dim=2)
+ self.causal_cache = x[:, :, -2*self.num_temp_upsample:-self.num_temp_upsample].clone()
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ x = x[:, :, 2*self.num_temp_upsample:]
+ else:
+ self.causal_cache = x[:, :, -self.num_temp_upsample:].clone()
+ x, _x = x[:, :, :self.num_temp_upsample], x[:, :, self.num_temp_upsample:]
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ if _x.shape[-3] > 0:
+ _x = F.interpolate(_x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(_x.dtype)
+ x = torch.concat([x, _x], dim=2)
+
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ time_padding = self.time_downsample_factor - video_len % self.time_downsample_factor
+ x = pad_at_dim(x, (time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.interpolation_mode = ignorekwargs.get("interpolation_mode", 'nearest')
+ assert self.interpolation_mode in ['nearest', 'trilinear']
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ num_temp_upsample = 1
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in, interpolation_mode=self.interpolation_mode, num_temp_upsample=num_temp_upsample)
+ num_temp_upsample *= 2
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..4223fb635eb88cd7e7292943c131965f2b814206
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py
@@ -0,0 +1,652 @@
+from typing import Callable
+
+import einops
+import torch
+import torch.nn as nn
+from einops import rearrange
+
+from .model_3dcausal import (AttnBlock, Normalize, nonlinearity,
+ spatial_temporal_resblk)
+from .util import checkpoint
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, stride=(2, 1, 1), padding=(0, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x1 = self.avg_pool(x)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, padding=1)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ xlst = [
+ torch.nn.functional.interpolate(
+ sx.unsqueeze(0).to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest"
+ ).to(x.dtype)
+ for sx in x
+ ]
+ x = torch.cat(xlst, dim=0)
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ if zero_init:
+ self.conv2.weight.data.zero_()
+ self.conv2.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetNoncausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=1)
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class Encoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch=8,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3]
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) # add for spatial tiling
+ self.norm_type = norm_type
+ self.is_causal = False
+
+ # downsampling
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1, padding=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level != self.num_resolutions - 1:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleRes2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn(block_in, norm_type=self.norm_type)
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ )
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, _, _ = x.shape
+
+ # downsampling
+ if x.shape[1] == 4 and self.conv_in.in_channels == 3:
+ raise ValueError("Mismatched number of input channels")
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+ if i_level != self.num_resolutions - 1:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_ds:
+ # temporal downsample
+ htmp = self.down_temporal[i_level].downsample(htmp)
+ hs.append(htmp)
+ B, _, T, _, _ = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h)
+ return h
+
+
+class Decoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels=8,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.tempo_us = [1, 2]
+ self.norm_type = norm_type
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level != 0:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleRes2x(block_in, block_in)
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1, padding=1)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def get_last_layer(self, **kwargs):
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, _, _ = z.shape
+
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+ if i_level != 0:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_us:
+ # temporal upsample
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, _, _ = h.shape
+ # end
+ if self.give_pre_end:
+ return h
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+ if self.tanh_out:
+ h = torch.tanh(h)
+ return h
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/regularizers.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/regularizers.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f4f1a4fccdb873f640e637ecbaaaef389e75b41
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/regularizers.py
@@ -0,0 +1,268 @@
+from abc import abstractmethod
+from functools import cache
+from typing import Any, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import pack, rearrange, reduce, unpack
+from torch import Tensor, int32
+from torch.cuda.amp import autocast
+
+from .distributions import DiagonalGaussianDistribution
+
+
+def exists(v):
+ return v is not None
+
+
+def default(*args):
+ for arg in args:
+ if exists(arg):
+ return arg
+ return None
+
+
+def pack_one(t, pattern):
+ return pack([t], pattern)
+
+
+def unpack_one(t, ps, pattern):
+ return unpack(t, ps, pattern)[0]
+
+
+def round_ste(z: Tensor) -> Tensor:
+ """Round with straight through gradients."""
+ zhat = z.round()
+ return z + (zhat - z).detach()
+
+
+def log(t, eps=1e-5):
+ return t.clamp(min=eps).log()
+
+
+def entropy(prob):
+ return (-prob * log(prob)).sum(dim=-1)
+
+
+def maybe_distributed_mean(t):
+ if not is_distributed():
+ return t
+ dist.all_reduce(t)
+ t = t / dist.get_world_size()
+ return t
+
+
+@cache
+def is_distributed():
+ return dist.is_initialized() and dist.get_world_size() > 1
+
+
+class AbstractRegularizer(nn.Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, z: torch.Tensor) -> Tuple[torch.Tensor, dict]:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_trainable_parameters(self) -> Any:
+ raise NotImplementedError()
+
+
+class DiagonalGaussianRegularizer(AbstractRegularizer):
+ def __init__(self, sample: bool = True):
+ super().__init__()
+ self.sample = sample
+
+ def get_trainable_parameters(self) -> Any:
+ yield from ()
+
+ def forward(self, z: torch.Tensor, n_steps=None) -> Tuple[torch.Tensor, dict]:
+ log = dict()
+ posterior = DiagonalGaussianDistribution(z)
+ if self.sample:
+ z = posterior.sample()
+ else:
+ z = posterior.mode()
+ kl_loss = posterior.kl()
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ log["kl_loss"] = kl_loss
+ return z, log
+
+
+class FSQRegularizer(AbstractRegularizer):
+ # https://github.com/lucidrains/vector-quantize-pytorch/blob/master/vector_quantize_pytorch/finite_scalar_quantization.py
+ def __init__(
+ self,
+ levels: List[int],
+ dim: Optional[int] = None,
+ num_codebooks=1,
+ keep_num_codebooks_dim: Optional[bool] = None,
+ scale: Optional[float] = None,
+ entropy_loss_weight: float = 0.0,
+ entropy_loss_annealing_steps: int = 0,
+ entropy_loss_annealing_factor: float = 1.0,
+ commitment_loss_weight: float = 0.0,
+ diversity_gamma: float = 1.0,
+ ):
+ super().__init__()
+ _levels = torch.tensor(levels, dtype=int32)
+ self.register_buffer("_levels", _levels, persistent=False)
+
+ _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=int32)
+ self.register_buffer("_basis", _basis, persistent=False)
+
+ self.scale = scale
+ self.entropy_loss_weight = entropy_loss_weight
+ self.entropy_loss_annealing_steps = entropy_loss_annealing_steps
+ self.entropy_loss_annealing_factor = entropy_loss_annealing_factor
+ self.commitment_loss_weight = commitment_loss_weight
+ self.diversity_gamma = diversity_gamma
+
+ codebook_dim = len(levels)
+ self.codebook_dim = codebook_dim
+
+ effective_codebook_dim = codebook_dim * num_codebooks
+ self.num_codebooks = num_codebooks
+ self.effective_codebook_dim = effective_codebook_dim
+
+ keep_num_codebooks_dim = default(keep_num_codebooks_dim, num_codebooks > 1)
+ assert not (num_codebooks > 1 and not keep_num_codebooks_dim)
+ self.keep_num_codebooks_dim = keep_num_codebooks_dim
+
+ self.dim = default(dim, len(_levels) * num_codebooks)
+
+ has_projections = self.dim != effective_codebook_dim
+ self.project_in = nn.Linear(self.dim, effective_codebook_dim) if has_projections else nn.Identity()
+ self.project_out = nn.Linear(effective_codebook_dim, self.dim) if has_projections else nn.Identity()
+ self.has_projections = has_projections
+
+ self.codebook_size = self._levels.prod().item()
+
+ implicit_codebook = self.indices_to_codes(torch.arange(self.codebook_size), project_out=False)
+ self.register_buffer("implicit_codebook", implicit_codebook, persistent=False)
+ self.register_buffer("zero", torch.tensor(0.0), persistent=False)
+
+ self.global_codebook_usage = torch.zeros([2**self.codebook_dim, self.num_codebooks], dtype=torch.long)
+
+ def get_trainable_parameters(self) -> Any:
+ return self.parameters()
+
+ def bound(self, z: Tensor, eps: float = 1e-3) -> Tensor:
+ """Bound `z`, an array of shape (..., d)."""
+ half_l = (self._levels - 1) * (1 + eps) / 2
+ offset = torch.where(self._levels % 2 == 0, 0.5, 0.0)
+ shift = (offset / half_l).atanh()
+ return (z + shift).tanh() * half_l - offset
+
+ def quantize(self, z: Tensor) -> Tensor:
+ """Quantizes z, returns quantized zhat, same shape as z."""
+ quantized = round_ste(self.bound(z))
+ half_width = self._levels // 2
+ return quantized / half_width
+
+ def _scale_and_shift(self, zhat_normalized: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat_normalized * half_width) + half_width
+
+ def _scale_and_shift_inverse(self, zhat: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat - half_width) / half_width
+
+ def codes_to_indices(self, zhat: Tensor) -> Tensor:
+ """Converts a `code` to an index in the codebook."""
+ assert zhat.shape[-1] == self.codebook_dim
+ zhat = self._scale_and_shift(zhat)
+ return (zhat * self._basis).sum(dim=-1).to(int32)
+
+ def indices_to_codes(self, indices: Tensor, project_out=True) -> Tensor:
+ """Inverse of `codes_to_indices`."""
+
+ is_img_or_video = indices.ndim >= (3 + int(self.keep_num_codebooks_dim))
+
+ indices = rearrange(indices, "... -> ... 1")
+ codes_non_centered = (indices // self._basis) % self._levels
+ codes = self._scale_and_shift_inverse(codes_non_centered)
+
+ if self.keep_num_codebooks_dim:
+ codes = rearrange(codes, "... c d -> ... (c d)")
+
+ if project_out:
+ codes = self.project_out(codes)
+
+ if is_img_or_video:
+ codes = rearrange(codes, "b ... d -> b d ...")
+
+ return codes
+
+ def calculate_entropy_loss_weight(self, n_steps):
+ if n_steps >= self.entropy_loss_annealing_steps:
+ return self.entropy_loss_weight
+ start = self.entropy_loss_annealing_factor * self.entropy_loss_weight
+ return start - (n_steps / self.entropy_loss_annealing_steps) * (start - self.entropy_loss_weight)
+
+ @autocast(enabled=False)
+ def forward(self, z: Tensor, inv_temperature: float = 100.0, n_steps: int = 0) -> Tensor:
+ """
+ einstein notation
+ b - batch
+ n - sequence (or flattened spatial dimensions)
+ d - feature dimension
+ c - number of codebook dim
+ """
+ is_img_or_video = z.ndim >= 4
+ if is_img_or_video:
+ z = rearrange(z, "b d ... -> b ... d")
+ z, ps = pack_one(z, "b * d")
+
+ assert z.shape[-1] == self.dim, f"expected dimension of {self.dim} but found dimension of {z.shape[-1]}"
+
+ z = self.project_in(z)
+ z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks)
+
+ with torch.autocast("cuda", enabled=False):
+ orig_dtype = z.dtype
+ z = z.float()
+ original_input = z
+ codes = self.quantize(z)
+ indices = self.codes_to_indices(codes)
+
+ if self.entropy_loss_weight > 0 or self.commitment_loss_weight > 0:
+ # the same as euclidean distance up to a constant
+ distance = -2 * torch.einsum("... i d, j d -> ... i j", original_input, self.implicit_codebook)
+ prob = (-distance * inv_temperature).softmax(dim=-1)
+ per_sample_probs = rearrange(prob, "b n ... -> (b n) ...")
+ per_sample_entropy = entropy(per_sample_probs).mean()
+ # distribution over all available tokens in the batch
+ avg_prob = reduce(per_sample_probs, "... c d -> c d", "mean")
+ avg_prob = maybe_distributed_mean(avg_prob)
+ codebook_entropy = entropy(avg_prob).mean()
+ entropy_aux_loss = per_sample_entropy - self.diversity_gamma * codebook_entropy
+ # commit loss
+ commit_loss = F.mse_loss(original_input, codes.detach(), reduction="none")
+ commit_loss = commit_loss.mean()
+ else:
+ entropy_aux_loss = per_sample_entropy = codebook_entropy = commit_loss = self.zero
+
+ codes = codes.type(orig_dtype)
+
+ codes = rearrange(codes, "b n c d -> b n (c d)")
+ out = self.project_out(codes)
+
+ # reconstitute image or video dimensions
+ if is_img_or_video:
+ out = unpack_one(out, ps, "b * d")
+ out = rearrange(out, "b ... d -> b d ...")
+
+ indices = unpack_one(indices, ps, "b * c")
+
+ if not self.keep_num_codebooks_dim:
+ indices = rearrange(indices, "... 1 -> ...")
+
+ aux_loss = (
+ entropy_aux_loss * self.calculate_entropy_loss_weight(n_steps) + commit_loss * self.commitment_loss_weight
+ )
+
+ return out, dict(indices=indices, aux_loss=aux_loss)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/util.py b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..9570016221e47b50d536d30352b5718a29bb0009
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtok/modules/util.py
@@ -0,0 +1,324 @@
+import importlib
+import random
+import os
+import einops
+import numpy as np
+from inspect import isfunction
+from rich import print
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+
+def get_valid_dirs(dir1: str, dir2: str, dir3: Union[None, str] = None) -> Union[None, str]:
+ if (dir1 is not None) and os.path.isdir(dir1):
+ return dir1
+ elif (dir2 is not None) and os.path.isdir(dir2):
+ return dir2
+ elif (dir3 is not None) and os.path.isdir(dir3):
+ return dir3
+ else:
+ return None
+
+
+def get_valid_paths(path1: str, path2: str, path3: Union[None, str] = None) -> Union[None, str]:
+ if (path1 is not None) and os.path.isfile(path1):
+ return path1
+ elif (path2 is not None) and os.path.isfile(path2):
+ return path2
+ elif (path3 is not None) and os.path.isfile(path3):
+ return path3
+ else:
+ return None
+
+
+@rank_zero_only
+def print0(*args, **kwargs):
+ print(*args, **kwargs)
+
+
+def seed_anything(seed: int):
+ os.environ['PYTHONHASHSEED'] = str(seed)
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+
+
+def isheatmap(x):
+ if not isinstance(x, torch.Tensor):
+ return False
+
+ return x.ndim == 2
+
+
+def exists(x):
+ return x is not None
+
+
+def default(val, d):
+ if exists(val):
+ return val
+ return d() if isfunction(d) else d
+
+
+def instantiate_from_config(config):
+ if not "target" in config:
+ if config == "__is_first_stage__":
+ return None
+ elif config == "__is_unconditional__":
+ return None
+ raise KeyError("Expected key `target` to instantiate.")
+ return get_obj_from_str(config["target"])(**config.get("params", dict()))
+
+
+def get_obj_from_str(string, reload=False, invalidate_cache=True):
+ module, cls = string.rsplit(".", 1)
+ if invalidate_cache:
+ importlib.invalidate_caches()
+ if reload:
+ module_imp = importlib.import_module(module)
+ importlib.reload(module_imp)
+ return getattr(importlib.import_module(module, package=None), cls)
+
+
+def checkpoint(func, inputs, params, flag):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ """
+ Evaluate a function without caching intermediate activations, allowing for
+ reduced memory at the expense of extra compute in the backward pass.
+ :param func: the function to evaluate.
+ :param inputs: the argument sequence to pass to `func`.
+ :param params: a sequence of parameters `func` depends on but does not
+ explicitly take as arguments.
+ :param flag: if False, disable gradient checkpointing.
+ """
+ if flag:
+ args = tuple(inputs) + tuple(params)
+ return CheckpointFunction.apply(func, len(inputs), *args)
+ else:
+ return func(*inputs)
+
+
+class CheckpointFunction(torch.autograd.Function):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ @staticmethod
+ def forward(ctx, run_function, length, *args):
+ ctx.run_function = run_function
+ ctx.input_tensors = list(args[:length])
+ ctx.input_params = list(args[length:])
+ ctx.gpu_autocast_kwargs = {
+ "enabled": torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad():
+ output_tensors = ctx.run_function(*ctx.input_tensors)
+ return output_tensors
+
+ @staticmethod
+ def backward(ctx, *output_grads):
+ ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
+ # Ensure all tensors have requires_grad set to True
+ ctx.input_params = [p.requires_grad_(True) for p in ctx.input_params]
+ with torch.enable_grad(), torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs):
+ # Fixes a bug where the first op in run_function modifies the
+ # Tensor storage in place, which is not allowed for detach()'d
+ # Tensors.
+ shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
+ output_tensors = ctx.run_function(*shallow_copies)
+ input_grads = torch.autograd.grad(
+ output_tensors,
+ ctx.input_tensors + ctx.input_params,
+ output_grads,
+ allow_unused=True,
+ )
+ del ctx.input_tensors
+ del ctx.input_params
+ del output_tensors
+ return (None, None) + input_grads
+
+
+def compute_psnr(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ EPS = 1e-8
+ mse = torch.mean((x - y) ** 2, dim=[1, 2, 3])
+ psnr = -10 * torch.log10(mse + EPS)
+ return psnr.mean(dim=0)
+
+
+def compute_ssim(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ kernel_size = 11
+ kernel_sigma = 1.5
+ k1 = 0.01
+ k2 = 0.03
+
+ f = max(1, round(min(x.size()[-2:]) / 256))
+ if f > 1:
+ x = F.avg_pool2d(x, kernel_size=f)
+ y = F.avg_pool2d(y, kernel_size=f)
+
+ kernel = gaussian_filter(kernel_size, kernel_sigma, device=x.device, dtype=x.dtype).repeat(x.size(1), 1, 1, 1)
+
+ _compute_ssim_per_channel = _ssim_per_channel_complex if x.dim() == 5 else _ssim_per_channel
+ ssim_map, cs_map = _compute_ssim_per_channel(x=x, y=y, kernel=kernel, data_range=1, k1=k1, k2=k2)
+ ssim_val = ssim_map.mean(1)
+
+ return ssim_val.mean(dim=0)
+
+
+def _ssim_per_channel(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W)`.
+ y: A target tensor. Shape :math:`(N, C, H, W)`.
+ kernel: 2D Gaussian kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Structural Similarity (SSIM) index.
+ """
+ if x.size(-1) < kernel.size(-1) or x.size(-2) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. "
+ f"Input size: {x.size()}. Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+ n_channels = x.size(1)
+ mu_x = F.conv2d(x, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu_y = F.conv2d(y, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu_xx = mu_x**2
+ mu_yy = mu_y**2
+ mu_xy = mu_x * mu_y
+
+ sigma_xx = F.conv2d(x**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xx
+ sigma_yy = F.conv2d(y**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_yy
+ sigma_xy = F.conv2d(x * y, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xy
+
+ # Contrast sensitivity (CS) with alpha = beta = gamma = 1.
+ cs = (2.0 * sigma_xy + c2) / (sigma_xx + sigma_yy + c2)
+
+ # Structural similarity (SSIM)
+ ss = (2.0 * mu_xy + c1) / (mu_xx + mu_yy + c1) * cs
+
+ ssim_val = ss.mean(dim=(-1, -2))
+ cs = cs.mean(dim=(-1, -2))
+ return ssim_val, cs
+
+
+def _ssim_per_channel_complex(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for Complex X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W, 2)`.
+ y: A target tensor. Shape :math:`(N, C, H, W, 2)`.
+ kernel: 2-D gauss kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Complex Structural Similarity (SSIM) index.
+ """
+ n_channels = x.size(1)
+ if x.size(-2) < kernel.size(-1) or x.size(-3) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. Input size: {x.size()}. "
+ f"Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+
+ x_real = x[..., 0]
+ x_imag = x[..., 1]
+ y_real = y[..., 0]
+ y_imag = y[..., 1]
+
+ mu1_real = F.conv2d(x_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu1_imag = F.conv2d(x_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_real = F.conv2d(y_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_imag = F.conv2d(y_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu1_sq = mu1_real.pow(2) + mu1_imag.pow(2)
+ mu2_sq = mu2_real.pow(2) + mu2_imag.pow(2)
+ mu1_mu2_real = mu1_real * mu2_real - mu1_imag * mu2_imag
+ mu1_mu2_imag = mu1_real * mu2_imag + mu1_imag * mu2_real
+
+ compensation = 1.0
+
+ x_sq = x_real.pow(2) + x_imag.pow(2)
+ y_sq = y_real.pow(2) + y_imag.pow(2)
+ x_y_real = x_real * y_real - x_imag * y_imag
+ x_y_imag = x_real * y_imag + x_imag * y_real
+
+ sigma1_sq = F.conv2d(x_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_sq
+ sigma2_sq = F.conv2d(y_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu2_sq
+ sigma12_real = F.conv2d(x_y_real, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_real
+ sigma12_imag = F.conv2d(x_y_imag, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_imag
+ sigma12 = torch.stack((sigma12_imag, sigma12_real), dim=-1)
+ mu1_mu2 = torch.stack((mu1_mu2_real, mu1_mu2_imag), dim=-1)
+ # Set alpha = beta = gamma = 1.
+ cs_map = (sigma12 * 2 + c2 * compensation) / (sigma1_sq.unsqueeze(-1) + sigma2_sq.unsqueeze(-1) + c2 * compensation)
+ ssim_map = (mu1_mu2 * 2 + c1 * compensation) / (mu1_sq.unsqueeze(-1) + mu2_sq.unsqueeze(-1) + c1 * compensation)
+ ssim_map = ssim_map * cs_map
+
+ ssim_val = ssim_map.mean(dim=(-2, -3))
+ cs = cs_map.mean(dim=(-2, -3))
+
+ return ssim_val, cs
+
+
+def gaussian_filter(
+ kernel_size: int, sigma: float, device: Optional[str] = None, dtype: Optional[type] = None
+) -> torch.Tensor:
+ r"""Returns 2D Gaussian kernel N(0,`sigma`^2)
+ Args:
+ kernel_size: Size of the kernel
+ sigma: Std of the distribution
+ device: target device for kernel generation
+ dtype: target data type for kernel generation
+ Returns:
+ gaussian_kernel: Tensor with shape (1, kernel_size, kernel_size)
+ """
+ coords = torch.arange(kernel_size, dtype=dtype, device=device)
+ coords -= (kernel_size - 1) / 2.0
+
+ g = coords**2
+ g = (-(g.unsqueeze(0) + g.unsqueeze(1)) / (2 * sigma**2)).exp()
+
+ g /= g.sum()
+ return g.unsqueeze(0)
diff --git a/Meissonic/VidTok/vidtok_cache/VidTok/vidtwin/README.md b/Meissonic/VidTok/vidtok_cache/VidTok/vidtwin/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e475c6eade763f9cef3b67a905dd738e1fb43fef
--- /dev/null
+++ b/Meissonic/VidTok/vidtok_cache/VidTok/vidtwin/README.md
@@ -0,0 +1,211 @@
+
+
+
+
VidTwin: Video VAE with Decoupled Structure and Dynamics (CVPR 2025)
+
+
+
+
+
+
+
+[](https://arxiv.org/pdf/2412.17726) [](https://huggingface.co/microsoft/vidtwin) [](https://vidtwin.github.io/)
+
+
🔥 Check our Demo Page for enhanced visual experience.
+
+
+
+
+
+
+
+
+ VidTwin: Video VAE with Decoupled Structure and Dynamics (CVPR 2025)
+
+
+
+
+
+
+
+[](https://arxiv.org/pdf/2412.17726) [](https://huggingface.co/microsoft/vidtwin) [](https://vidtwin.github.io/)
+
+
🔥 Check our Demo Page for enhanced visual experience.
+
+
+
+
+
+
+
+
+
+# VidTok
A Family of Versatile and State-Of-The-Art Video Tokenizers
+
+[](https://arxiv.org/pdf/2412.13061) [](https://github.com/microsoft/VidTok) [](https://huggingface.co/microsoft/VidTok)
+
+
+---
+
+
+
+ We introduce VidTok, a cutting-edge family of video tokenizers that excels in both continuous and discrete tokenizations. VidTok incorporates several key advancements over existing approaches:
+ * ⚡️ **Efficient Architecture**. Separate spatial and temporal sampling reduces computational complexity without sacrificing quality.
+ * 🔥 **Advanced Quantization**. Finite Scalar Quantization (FSQ) addresses training instability and codebook collapse in discrete tokenization.
+ * 💥 **Enhanced Training**. A two-stage strategy—pre-training on low-res videos and fine-tuning on high-res—boosts efficiency. Reduced frame rates improve motion dynamics representation.
+
+VidTok, trained on a large-scale video dataset, outperforms previous models across all metrics, including PSNR, SSIM, LPIPS, and FVD.
+
+https://github.com/user-attachments/assets/a3341037-130d-4a83-aba6-c3daeaf66932
+
+## 🔥 News
+- August, 2025: 🚀 Introduced spatial tiling for large resolutions (>256), reducing GPU memory usage to ~6 GB when encoding and decoding a 17 × 768 × 768 video.
+* March, 2025: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) has been accepted by CVPR 2025, and the [checkpoint](https://huggingface.co/microsoft/vidtwin) was released!
+* March, 2025: 🚀 [VidTok v1.1](#-updates-in-vidtok-v11) was released! We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. [Relevant checkpoints](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints/vidtok_v1_1) are continuously updating.
+* December, 2024: 🚀 [VidTwin](https://github.com/microsoft/VidTok/tree/main/vidtwin) was released!
+* December, 2024: 🚀 [VidTok](https://github.com/microsoft/vidtok) was released!
+
+
+## 💥 Updates in VidTok v1.1
+> VidTok v1.1 is an update for causal models. We fine-tuned all causal models on long videos to support tokenization and reconstruction of videos of arbitrary length with fine temporal smoothness. See performance [here](#v11-performance).
+
+### v1.1: Long Video Reconstruction
+Run the following inference script to [reconstruct an input video](#reconstruct-an-input-video):
+```bash
+python scripts/inference_reconstruct.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --output_video_dir OUTPUT_DIR --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+and run the following inference script to [evaluate the reconstruction performance](#performance-evaluation):
+```bash
+python scripts/inference_evaluate.py --config CONFIG_v1_1 --ckpt CKPT_v1_1 --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30 --chunk_size CHUNK_SIZE --read_long_video
+# Set `CHUNK_SIZE` according to your GPU memory, recommendly 16.
+```
+
+For an easy usage of VidTok v1.1 models, refer to [this script](#easy-usage) and make the following revision:
+```python
+# Use VidTok v1.1 models
+cfg_path = "configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml"
+ckpt_path = "checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt"
+
+...
+
+model.to('cuda').eval()
+# Using tiling inference to save memory usage
+model.use_tiling = True
+model.t_chunk_enc = 16
+model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+model.use_overlap = True
+# random input: long video
+x_input = (torch.rand(1, 3, 129, 256, 256) * 2 - 1).to('cuda')
+
+...
+
+if x_recon.shape[2] != x_input.shape[2]:
+ x_recon = x_recon[:, :, -x_input.shape[2]:, ...]
+```
+
+### v1.1: Long Video Fine-tuning
+Follow this [training guidance](#fine-tune-on-custom-data) to fine-tune on your custom long video data and note that:
+- Compared to VidTok v1.0, we tend to use longer sequences to fine-tune the model (for example, setting `NUM_FRAMES_1` to 33, 49, or larger).
+- The resolution and the sequence length of training data should be adjusted according to GPU memory.
+
+### v1.1: Performance
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.13 | 0.941 | 0.049 | 87.4 |
+| [vidtok_kl_causal_41616_16chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt) | KL-16chn | ✔️ | 4x16x16 | 29.61 | 0.854 | 0.113 | 162.7 |
+| [vidtok_kl_causal_288_8chn_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt) | KL-8chn | ✔️ | 2x8x8 | 34.59 | 0.935 | 0.051 | 78.2 |
+| [vidtok_fsq_causal_488_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.39 | 0.856 | 0.114 | 168.5 |
+| [vidtok_fsq_causal_888_32768_v1_1](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt) | FSQ-32,768 | ✔️ | 8x8x8 | 27.95 | 0.817 | 0.142 | 293.2 |
+- This is the evaluation result of long video reconstruction conducted on each complete video in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30 and a resolution of `256x256`.
+
+
+## 🔧 Setup
+1. Clone this repository and navigate to VidTok folder:
+```bash
+git clone https://github.com/microsoft/VidTok
+cd VidTok
+```
+2. We provide an `environment.yaml` file for setting up a Conda environment. Conda's installation instructions are available [here](https://docs.anaconda.com/miniconda/index.html).
+```bash
+# 1. Prepare conda environment
+conda env create -f environment.yaml
+# 2. Activate the environment
+conda activate vidtok
+```
+
+We recommend using 1+ high-end GPU for training and inference. We have done all testing and development using A100 and MI300X GPUs. For convenience, we also provide prebuilt [Docker](https://hub.docker.com/) images with required dependencies. You can use it as follows:
+
+```bash
+# NVIDIA GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-cuda12.1-python3.10-pytorch2.5:orig-vidtok bash
+# AMD GPUs
+docker run -it --gpus all --shm-size 256G --rm -v `pwd`:/workspace --workdir /workspace \
+ deeptimhe/ubuntu22.04-rocm6.2.4-python3.10-pytorch2.5:orig-vidtok bash
+```
+
+## 🎈 Checkpoints
+Download pre-trained models [here](https://huggingface.co/microsoft/VidTok/tree/main/checkpoints), and put them in `checkpoints` folder, like:
+```
+└── checkpoints
+ ├── vidtok_v1_1
+ │ ├── vidtok_kl_causal_488_16chn_v1_1.ckpt
+ │ └── ...
+ ├── vidtok_fsq_causal_41616_262144.ckpt
+ ├── vidtok_fsq_causal_488_262144.ckpt
+ ├── vidtok_fsq_causal_488_32768.ckpt
+ ├── vidtok_fsq_causal_488_4096.ckpt
+ ├── vidtok_fsq_noncausal_41616_262144.ckpt
+ ├── vidtok_fsq_noncausal_488_262144.ckpt
+ ├── vidtok_kl_causal_288_8chn.ckpt
+ ├── vidtok_kl_causal_41616_4chn.ckpt
+ ├── vidtok_kl_causal_444_4chn.ckpt
+ ├── vidtok_kl_causal_488_16chn.ckpt
+ ├── vidtok_kl_causal_488_4chn.ckpt
+ ├── vidtok_kl_causal_488_8chn.ckpt
+ ├── vidtok_kl_noncausal_41616_16chn.ckpt
+ ├── vidtok_kl_noncausal_41616_4chn.ckpt
+ ├── vidtok_kl_noncausal_488_16chn.ckpt
+ └── vidtok_kl_noncausal_488_4chn.ckpt
+```
+Each checkpoint has a corresponding config file with the same name in `configs` folder.
+
+
+## 🔆 Performance
+
+| Model | Regularizer | Causal | VCR | PSNR | SSIM | LPIPS | FVD |
+|------|------|------|------|------|------|------|------|
+| [vidtok_kl_causal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_4chn.ckpt) | KL-4chn | ✔️ | 4x8x8 | 29.64 | 0.852| 0.114| 194.2|
+| [vidtok_kl_causal_488_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_8chn.ckpt) | KL-8chn | ✔️ |4x8x8 | 31.83 | 0.897| 0.083| 109.3|
+| [vidtok_kl_causal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_488_16chn.ckpt) | KL-16chn | ✔️ | 4x8x8 | 35.04 |0.942 |0.047 | 78.9|
+| [vidtok_kl_causal_288_8chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_288_8chn.ckpt) | KL-8chn | ✔️ | 2x8x8 | 33.86 | 0.928 |0.057 | 80.7 |
+| [vidtok_kl_causal_444_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_444_4chn.ckpt) | KL-4chn | ✔️ | 4x4x4 | 34.78 | 0.941 | 0.051| 87.2|
+| [vidtok_kl_causal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_causal_41616_4chn.ckpt) | KL-4chn | ✔️ | 4x16x16 | 25.05 | 0.711| 0.228| 549.1|
+| [vidtok_kl_noncausal_488_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_4chn.ckpt) | KL-4chn | ✖️ | 4x8x8 | 30.60 | 0.876 | 0.098| 157.9|
+| [vidtok_kl_noncausal_488_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_488_16chn.ckpt) | KL-16chn | ✖️ | 4x8x8 | 36.13 | 0.950 | 0.044| 60.5|
+| [vidtok_kl_noncausal_41616_4chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt) | KL-4chn | ✖️ | 4x16x16 | 26.06 | 0.751 | 0.190|423.2 |
+| [vidtok_kl_noncausal_41616_16chn](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt) | KL-16chn | ✖️ | 4x16x16 | 30.69 | 0.878 | 0.095| 147.1|
+| [vidtok_fsq_causal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_262144.ckpt) | FSQ-262,144 | ✔️ | 4x8x8 | 29.82 | 0.867 |0.106 | 160.1|
+| [vidtok_fsq_causal_488_32768](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_32768.ckpt) | FSQ-32,768 | ✔️ | 4x8x8 | 29.16 | 0.854 | 0.117| 196.9|
+| [vidtok_fsq_causal_488_4096](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_488_4096.ckpt) | FSQ-4096 | ✔️ | 4x8x8 | 28.36 | 0.832 | 0.133| 218.1|
+| [vidtok_fsq_causal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_causal_41616_262144.ckpt) | FSQ-262,144 | ✔️ | 4x16x16 | 25.38 | 0.738 |0.206 | 430.1|
+| [vidtok_fsq_noncausal_488_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_488_262144.ckpt) | FSQ-262,144 | ✖️ | 4x8x8 | 30.78 | 0.889| 0.091| 132.1|
+| [vidtok_fsq_noncausal_41616_262144](https://huggingface.co/microsoft/VidTok/blob/main/checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt) | FSQ-262,144 | ✖️ | 4x16x16 | 26.37 | 0.772| 0.171| 357.0|
+
+- `VCR` indicates the video compression ratio `TxHxW`.
+- The above table shows model performance evaluated on 30 test videos in [MCL_JCL](https://mcl.usc.edu/mcl-jcv-dataset/) dataset, with a sample fps of 30. The input size is `17x256x256` for causal models and `16x256x256` for non-causal models.
+
+## 🔛 Training
+
+### Data Preparation
+1. Put all training videos under `DATA_DIR`:
+```
+└── DATA_DIR
+ ├── subset1
+ │ ├── videoname11.mp4
+ │ └── videoname12.mp4
+ ├── subset2
+ │ ├── videoname21.mp4
+ │ ├── videoname22.mp4
+ │ └── subsubset1
+ │ ├── videoname211.mp4
+ │ └── videoname212.mp4
+ └── ...
+```
+2. Prepare a `.csv` meta file to record the relative paths of these videos with respect to `DATA_DIR`, like:
+```
+videos
+subset1/videoname11.mp4
+subset2/videoname21.mp4
+subset2/subsubset1/videoname211.mp4
+```
+
+> Validation data is also prepared following the above steps.
+
+### Fine-tune on Custom Data
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and modify the following parameters:
+ - Specify the `ckpt_path` parameter to initialize the model with pre-trained checkpoint parameters:
+ ```yaml
+ model:
+ params:
+ ckpt_path: PATH_TO_CHECKPOINT # train from existing checkpoint
+ ```
+ - Specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+ - Set `fix_encoder` and `fix_decoder` to be `False` to enable full model fine-tuning:
+ ```yaml
+ model:
+ params:
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+ ```
+ - Other hyperparameters according to your needs.
+
+3. Run the following command to start training:
+```bash
+python main.py -b CONFIG --logdir LOGDIR
+
+# You can also use `torchrun` to start the training code.
+```
+Training logs and checkpoints are saved in `LOGDIR`.
+
+It is recommended to use [Weights & Biases](https://wandb.ai/site) as the data visualization tool ([TensorBoard](https://www.tensorflow.org/tensorboard) by default). Use `wandb login` to log in first, and then run:
+```bash
+python main.py -b CONFIG --logdir LOGDIR --wandb --wandb_entity ENTITY --wandb_project PROJECT
+```
+
+
+### Train from Scratch
+
+Two-stage Training
+We adopt a two-stage training strategy to improve training efficiency: initially pre-training the full model on low-resolution videos, followed by fine-tuning only the decoder on high-resolution videos.
+
+| First Stage | Second Stage | Fix encoder | PSNR | SSIM | LPIPS | GPU Hours|
+|------|------|------|------|------|------|------|
+| 256 x 256 | - | - | 29.19 | 0.843 | 0.127| 3,072|
+| 128 x 128 | 256 x 256 | ✔️ | 29.21 | 0.843 | 0.125| 1,536|
+
+1. Prepare your own training and validation data following [Data Preparation](#data-preparation).
+2. Select the appropriate `CONFIG` file from `configs` folder based on your needs, and specify the `data` section to use your own training and validation data:
+ ```yaml
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1 # vary in different training stages
+ input_width: INPUT_WIDTH_1 # vary in different training stages
+ sample_num_frames: NUM_FRAMES_1 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_1 # sample fps for training data
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: NUM_FRAMES_2 # typically set to 17 for causal models and 16 for non-causal models
+ sample_fps: SAMPLE_FPS_2 # sample fps for validation data
+ start_index: 0 # fixed value to ensure the same sampled data
+ ```
+
+3. Start the first stage of training. First, revise the `CONFIG` file to enable full model training with low-resolution data:
+```yaml
+model:
+ params:
+ # ckpt_path: # disable this parameter so as to train from scratch
+ encoder_config:
+ params:
+ fix_encoder: false
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 128
+ input_width: 128
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 50,000 steps with batch size 16 in this stage.
+
+4. Start the second stage of training. First, revise the `CONFIG` file to enable the fine-tuning of the decoder with high-resolution data:
+```yaml
+model:
+ params:
+ ckpt_path: CKPT_PATH # path to the saved checkpoint after the first stage of training
+ encoder_config:
+ params:
+ fix_encoder: true
+ fix_decoder: false
+data:
+ params:
+ train:
+ params:
+ video_params:
+ input_height: 256
+ input_width: 256
+```
+Then revise other hyperparameters according to your needs, and run the training command to start training as in [Fine-tune on Custom Data](#fine-tune-on-custom-data). We train VidTok for 30,000 steps with batch size 8 in this stage.
+
+
+
+## 🚀 Inference
+
+### Easy Usage
+We provide the following example for a quick usage of our models. It works for both continuous and discrete tokenization and both causal and non-causal models.
+Just provide the path to the configuration file `cfg_path` and checkpoint file `ckpt_path`.
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+# model forward
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+assert x_input.shape == x_recon.shape
+```
+If you want to directly infer from latent tokens, run the following code:
+```python
+z, reg_log = model.encode(x_input, return_reg_log=True)
+# infer from continuous latent space
+x_recon = model.decode(z)
+# infer from discrete latent tokens
+x_recon = model.decode(reg_log['indices'], decode_from_indices=True)
+```
+
+### Use Torch Compile to Speed Up Inference
+Use compiled components in VidTok can speed up inference by as much as 2X. The following code snippet demonstrates how to compile our models.
+
+```python
+import torch
+from scripts.inference_evaluate import load_model_from_config
+
+torch._inductor.config.cpp.weight_prepack=True
+torch._inductor.config.freezing=True
+
+cfg_path = "configs/vidtok_kl_causal_488_4chn.yaml"
+ckpt_path = "checkpoints/vidtok_kl_causal_488_4chn.ckpt"
+
+# load pre-trained model
+model = load_model_from_config(cfg_path, ckpt_path)
+model.to('cuda').eval()
+model.encoder = torch.compile(model.encoder)
+model.decoder = torch.compile(model.decoder)
+
+# random input
+num_frames = 17 if model.is_causal else 16
+x_input = (torch.rand(1, 3, num_frames, 256, 256) * 2 - 1).to('cuda') # [B,C,T,H,W], range -1~1
+
+# Warm Up
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ _, x_recon, _ = model(x_input)
+
+torch.cuda.synchronize()
+import time
+start = time.time()
+with torch.no_grad(), torch.autocast(device_type='cuda', dtype=torch.float16):
+ for i in range(10):
+ _, x_recon, _ = model(x_input)
+torch.cuda.synchronize()
+print(f"Average inference time: {(time.time() - start)/10 :.4f} seconds")
+```
+
+### Reconstruct an Input Video
+```bash
+python scripts/inference_reconstruct.py --config CONFIG --ckpt CKPT --input_video_path VIDEO_PATH --input_height 256 --input_width 256 --sample_fps 30 --output_video_dir OUTPUT_DIR
+```
+- Specify `VIDEO_PATH` to the path of your test video. We provide an example video in `assets/example.mp4`.
+- The reconstructed video is saved in `OUTPUT_DIR`.
+- For causal models, you can choose to add `--pad_gen_frames` to the command line, which may improve the smoothness of the reconstructed video.
+
+### Performance Evaluation
+We also provide a manuscript `scripts/inference_evaluate.py` to evaluate the video reconstruction performance in PSNR, SSIM and LPIPS.
+
+1. Put all of your test videos under `DATA_DIR`.
+2. Run the following command, and all `.mp4` videos under `DATA_DIR` will be tested:
+```bash
+python scripts/inference_evaluate.py --config CONFIG --ckpt CKPT --data_dir DATA_DIR --input_height 256 --input_width 256 --sample_fps 30
+```
+(Optional) If you only want to test certain videos under `DATA_DIR`, you need to prepare a `.csv` meta file
+to indicate the video files to be tested (refer to [Data Preparation](#data-preparation)). And add `--meta_path META_PATH` to the above command to specify the path to the `.csv` meta file.
+
+## 💡 Intended Uses
+
+We are sharing our model with the research community to foster further research in this area:
+* Training your own video tokenizers for research purpose.
+* Video tokenization with various compression rates.
+
+
+## 🪧 Out-of-scope Uses
+
+Our models are not specifically designed or evaluated for all downstream purposes. Developers should consider common limitations of video tokenizers (e.g., performance degradation on out-of-domain data) as they select use cases, and evaluate and mitigate for privacy, safety, and fairness before using within a specific downstream use case, particularly for high-risk scenarios.
+
+Developers should be aware of and adhere to applicable laws or regulations (including privacy, trade compliance laws, etc.) that are relevant to their use case.
+
+
+## 🤖️ Risks and Limitations
+
+Some of the limitations of this model to be aware of include:
+* VidTok may lose detailed information on the reconstructed content.
+* VidTok inherits any biases, errors, or omissions characteristic of its training data.
+* VidTok was developed for research and experimental purposes. Further testing and validation are needed before considering its application in commercial or real-world scenarios.
+
+
+## 🤗 Acknowledgments
+
+This codebase borrows code from [generative-models](https://github.com/Stability-AI/generative-models). We thank Stability AI for its efforts and innovations, which have made the development process more efficient and convenient.
+
+Thank you to everyone who contributed their wisdom and efforts to this project.
+
+## ✏️ BibTeX
+
+```bibtex
+@article{tang2024vidtok,
+ title={VidTok: A Versatile and Open-Source Video Tokenizer},
+ author={Tang, Anni and He, Tianyu and Guo, Junliang and Cheng, Xinle and Song, Li and Bian, Jiang},
+ year={2024},
+ journal={arXiv preprint arXiv:2412.13061},
+}
+```
+
+## ☎️ Contact
+
+We welcome feedback and collaboration from our audience. If you have suggestions, questions, or observe unexpected/offensive behavior in our technology, please contact us at tianyuhe@microsoft.com.
+
+## 📄 Contributing
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+
+## 📍 Trademarks
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
+trademarks or logos is subject to and must follow
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.
diff --git a/Meissonic/vidtok_cache/VidTok/SECURITY.md b/Meissonic/vidtok_cache/VidTok/SECURITY.md
new file mode 100644
index 0000000000000000000000000000000000000000..b3c89efc852e22f71eabf5dfbc6ac62493425eb6
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/SECURITY.md
@@ -0,0 +1,41 @@
+
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+ * Full paths of source file(s) related to the manifestation of the issue
+ * The location of the affected source code (tag/branch/commit or direct URL)
+ * Any special configuration required to reproduce the issue
+ * Step-by-step instructions to reproduce the issue
+ * Proof-of-concept or exploit code (if possible)
+ * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd).
+
+
diff --git a/Meissonic/vidtok_cache/VidTok/SUPPORT.md b/Meissonic/vidtok_cache/VidTok/SUPPORT.md
new file mode 100644
index 0000000000000000000000000000000000000000..291d4d43733f4c15a81ff598ec1c99fd6c18f64c
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/SUPPORT.md
@@ -0,0 +1,25 @@
+# TODO: The maintainer of this repo has not yet edited this file
+
+**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
+
+- **No CSS support:** Fill out this template with information about how to file issues and get help.
+- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
+- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
+
+*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
+
+# Support
+
+## How to file issues and get help
+
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates. For new issues, file your bug or
+feature request as a new Issue.
+
+For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE
+FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
+CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
+
+## Microsoft Support Policy
+
+Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
diff --git a/Meissonic/vidtok_cache/VidTok/assets/example.mp4 b/Meissonic/vidtok_cache/VidTok/assets/example.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..21be26c8a8f5dbb76de5a225e5b284fdbb024904
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/assets/example.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:588ca89fae7320a079d4f77cf963f88075959f06310594ab35d3e04b844c4d50
+size 540937
diff --git a/Meissonic/vidtok_cache/VidTok/assets/gemini.png b/Meissonic/vidtok_cache/VidTok/assets/gemini.png
new file mode 100644
index 0000000000000000000000000000000000000000..fe326dc4b4b4b6db82575b15e9b4bf2f3b63e63d
Binary files /dev/null and b/Meissonic/vidtok_cache/VidTok/assets/gemini.png differ
diff --git a/Meissonic/vidtok_cache/VidTok/assets/radar.png b/Meissonic/vidtok_cache/VidTok/assets/radar.png
new file mode 100644
index 0000000000000000000000000000000000000000..6eff9338928faf73c0e39e859237bc6c588ec183
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/assets/radar.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cfdef783e26264ff671f81845b54471b237f5cd7df0dbd63642fde0c20f935e
+size 424581
diff --git a/Meissonic/vidtok_cache/VidTok/assets/vidtwin.png b/Meissonic/vidtok_cache/VidTok/assets/vidtwin.png
new file mode 100644
index 0000000000000000000000000000000000000000..a7ae41105384d077a7d128d1d687e3ee8e17fbd8
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/assets/vidtwin.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e58210f7ca7784a4df737f8a2ece2b5e9f85fff3bf423b24aa6aeeb0b196cef
+size 549933
diff --git a/Meissonic/vidtok_cache/VidTok/assets/vidtwin_demo.png b/Meissonic/vidtok_cache/VidTok/assets/vidtwin_demo.png
new file mode 100644
index 0000000000000000000000000000000000000000..ca79126104a7cbfb67c0acb70f03c6e66546675c
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/assets/vidtwin_demo.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af8e15c46050cc7957bc5d334bd1902d58b9be9aa7669df82f2ae9ba08c90585
+size 6371846
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..80b2edbebf97fd747933000d32ccbdfe0e135702
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_41616_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f9c3e94ed8b64a681b2ed3887929109b6ccc53
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_262144.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5e661c533b367c7be4eea8de629e074b395e1684
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_32768.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_32768.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9bf654bb21c1bd67ddde3f4878497c6ad780503a
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_causal_488_4096.yaml
@@ -0,0 +1,118 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_causal_488_4096.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8] # codebook size: 8*8*8*8=4096
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e400ff097155465e2a477bfbc3ca32f346a0ca12
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_41616_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_41616_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6c8731af435aae0efbde9716ee696ce3efd30d25
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_fsq_noncausal_488_262144.yaml
@@ -0,0 +1,117 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_fsq_noncausal_488_262144.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4c20bf1aaf2aa97f67d19d0de1dbac47cf2d55ac
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_288_8chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_288_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1db41cd612f5e6b1390bebda33e559e347b60907
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_41616_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bcbbd11cba27e60bd2f44866655adcfff91f91a6
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_444_4chn.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_444_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ spatial_ds: [1, 2]
+ spatial_us: [1, 2]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a7527679fe0766616df45223fe1e9595101fadc5
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_16chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d14b0d75435b24affad90095e4d6e42c6525a8d
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_4chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..336cbac8319dc3f9c232c02889e598fdf098a777
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_causal_488_8chn.yaml
@@ -0,0 +1,112 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_causal_488_8chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 17
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 17
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b32064f1589eb3b21cb978f808a95fd6ade31ff9
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bdd33a5ebc635cbf9a04fb94006d14a473d87a3c
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_41616_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_41616_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..47a1090f42b07655626f0d522d756157fdeb1d6a
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_16chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_16chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05bb8c54df1374c0f67c79d256195dd6451acfbd
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_kl_noncausal_488_4chn.yaml
@@ -0,0 +1,111 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_kl_noncausal_488_4chn.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+
+ encoder_config:
+ target: vidtok.modules.model_3dnoncausal.Encoder3D
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ norm_type: layernorm # layernorm, groupnorm
+ fix_encoder: false
+ fix_decoder: false
+
+ decoder_config:
+ target: vidtok.modules.model_3dnoncausal.Decoder3D
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 16
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 16
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..74364fa3209abbda65b0c23311d92ad0975570dd
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_41616_262144_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 6
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8*8=262144
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..334749091ff2c561208f169029eae4704d4213e3
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.yaml
@@ -0,0 +1,120 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_488_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2966072ff2a073d404d0cb438674acfc92319033
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.yaml
@@ -0,0 +1,122 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_fsq_causal_888_32768_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: false
+ z_channels: 5
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [0, 1, 2]
+ tempo_us: [1, 2, 3]
+ time_downsample_factor: 8
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.FSQRegularizer
+ params:
+ levels: [8, 8, 8, 8, 8] # codebook size: 8*8*8*8*8=32768
+ entropy_loss_weight: 0.1
+ entropy_loss_annealing_steps: 2000
+ entropy_loss_annealing_factor: 3
+ commitment_loss_weight: 0.25
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..deefcc29ecd8212e8bd3edd9d9870d8c64079db7
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.yaml
@@ -0,0 +1,116 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_288_8chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 8
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ tempo_ds: [1]
+ tempo_us: [2]
+ time_downsample_factor: 2
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..965b243859e513244d3d3fd9cc68aa27aee887da
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_41616_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26f68342b48dd5fcae8e5bfcd2fcb5cc5bd1ab1c
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_16chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 16
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b4badafed742c03fd7850f197cdb8207c59b992e
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.yaml
@@ -0,0 +1,114 @@
+model:
+ base_learning_rate: 1e-5
+ target: vidtok.models.autoencoder_v1_1.AutoencodingEngine
+ params:
+ monitor: val/rec_loss
+ mode: min
+ # ckpt_path: checkpoints/vidtok_v1_1/vidtok_kl_causal_488_4chn_v1_1.ckpt # train from existing checkpoint
+ ignore_keys: []
+ # ema_decay: 0.999
+ use_tiling: False
+
+ encoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.EncoderCausal3DPadding
+ params:
+ double_z: true
+ z_channels: 4
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult: [1, 2, 4, 4]
+ time_downsample_factor: 4
+ num_res_blocks: 2
+ dropout: 0.0
+ use_checkpoint: false
+ init_pad_mode: replicate
+ norm_type: layernorm # layernorm, groupnorm
+ interpolation_mode: trilinear # nearest, trilinear
+ fix_encoder: false # if True, fix it without updating params
+ fix_decoder: false # if True, fix it without updating params
+
+ decoder_config:
+ target: vidtok.modules.model_3dcausal_v1_1.DecoderCausal3DPadding
+ params: ${model.params.encoder_config.params}
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ dims: 3 # video - [t,h,w]
+ perceptual_weight: 1.0
+ disc_start: 20001
+ disc_weight: 0.2
+ disc_type: 2d # 2d, 3d
+ learn_logvar: true
+ gen_loss_cross_entropy: true
+ lecam_loss_weight: 0.005
+ regularization_weights: {'aux_loss': 1.0, 'kl_loss': 0.000001}
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 1
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: INPUT_HEIGHT_1
+ input_width: INPUT_WIDTH_1
+ sample_num_frames: 33
+ sample_fps: 3
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: INPUT_HEIGHT_2
+ input_width: INPUT_WIDTH_2
+ sample_num_frames: 33
+ sample_fps: 8
+ start_index: 0
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: true
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 17
+
+ trainer:
+ precision: bf16-mixed
+ devices: auto
+ num_nodes: 1
+ benchmark: true
+ num_sanity_val_steps: 10
+ val_check_interval: 2000
+ check_val_every_n_epoch: null # default: 1
+ accumulate_grad_batches: 1
+ max_epochs: 1000
diff --git a/Meissonic/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml b/Meissonic/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8bcd59650c229ac57560602854015262969d658b
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/configs/vidtwin/vidtwin_structure_7_7_8_dynamics_7_8.yaml
@@ -0,0 +1,154 @@
+model:
+ base_learning_rate: 1.6e-4
+ target: vidtwin.models.vidtwin_ae.VidAutoEncoderQformerCompactSymVidVAE
+ params:
+ input_key: jpg
+ monitor: val/rec_loss
+ ckpt_path: PATH_TO_CHECKPOINT
+ ignore_keys: []
+ expect_ch: 8
+ cont_num_blocks: 1
+ downsample_motion: True
+ motion_num_blocks: 1
+ d_dim: 8
+
+ temporal_qformer_config:
+ target: vidtwin.modules.qformer.MyQformerInterface
+ params:
+ num_query_tokens: 16
+ query_hidden_size: 64
+ encoder_hidden_size: 768
+
+ encoder_config:
+ target: vidtwin.modules.st_transformer.STTEncoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ decoder_config:
+ target: vidtwin.modules.st_transformer.STTDecoder
+ params:
+ in_channels: 3
+ input_size: [16, 224, 224]
+ patch_size: [1, 16, 16]
+ hidden_size: 768
+ depth: 16
+ num_heads: 12
+ temporal_casual: true
+
+ loss_config:
+ target: vidtok.modules.losses.GeneralLPIPSWithDiscriminator
+ params:
+ perceptual_weight: 0.05
+ disc_start: 20001
+ disc_weight: 0.05
+ learn_logvar: True
+ dims: 3
+ disc_type: 2d
+ regularization_weights:
+ kl_loss: 0.001
+
+ regularizer_config:
+ target: vidtok.modules.regularizers.DiagonalGaussianRegularizer
+ params:
+ sample: True
+
+
+ lr_scheduler_config_d:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 1.5e-05
+ lr_start: 1.0e-05
+ warmup_steps: 5000
+ lr_scheduler_config_g:
+ target: vidtok.models.vidtwin_ae.LambdaWarmUpCosineScheduler
+ params:
+ lr_min: 0
+ lr_max: 3.0e-05
+ lr_start: 0
+ warmup_steps: 5000
+ optimizer_config:
+ target: torch.optim.AdamW
+ params:
+ betas:
+ - 0
+ - 0.9
+ weight_decay: 0.0001
+ lr_scheduler_config:
+ target: inverse_sqrt
+ params:
+ num_warmup_steps: 2000
+ frequency: 1
+
+data:
+ target: vidtok.data.datamodule.DataModuleFromConfig
+ params:
+ batch_size: 2
+ num_workers: 12
+
+ train:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_1 # DATA_DIR for training data
+ meta_path: META_PATH_1 # path to the .csv meta file of training data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+
+ validation:
+ target: vidtok.data.vidtok.VidTokDataset
+ params:
+ data_dir: DATA_DIR_2 # DATA_DIR for validation data
+ meta_path: META_PATH_2 # path to the .csv meta file of validation data
+ video_params:
+ input_height: 224
+ input_width: 224
+ sample_num_frames: 16
+ sample_fps: 8
+ start_index: 0
+
+
+lightning:
+ strategy:
+ target: lightning.pytorch.strategies.DDPStrategy
+ params:
+ find_unused_parameters: True
+
+ modelcheckpoint:
+ params:
+ every_n_train_steps: 5000
+
+
+ callbacks:
+ image_logger:
+ target: vidtok.modules.logger.ImageVideoLogger
+ params:
+ disabled: false
+ rescale: true
+ enable_autocast: false
+ batch_frequency: 5000
+ max_samples: 2
+ increase_log_steps: false
+ log_first_step: false
+ log_before_first_step: false
+ log_images_kwargs:
+ n_rows: 2
+
+
+
+ trainer:
+ # precision: bf16-mixed # 16-mixed
+ benchmark: True
+ devices: 4
+ num_sanity_val_steps: 10
+ val_check_interval: 5000
+ accumulate_grad_batches: 1
+ max_epochs: 10
diff --git a/Meissonic/vidtok_cache/VidTok/environment.yaml b/Meissonic/vidtok_cache/VidTok/environment.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a6abf1dcc7cefdab27b272681c0395791b3d432f
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/environment.yaml
@@ -0,0 +1,114 @@
+name: vidtok
+channels:
+ - defaults
+dependencies:
+ - _libgcc_mutex=0.1=main
+ - _openmp_mutex=5.1=1_gnu
+ - bzip2=1.0.8=h5eee18b_6
+ - ca-certificates=2024.11.26=h06a4308_0
+ - ld_impl_linux-64=2.40=h12ee557_0
+ - libffi=3.4.4=h6a678d5_1
+ - libgcc-ng=11.2.0=h1234567_1
+ - libgomp=11.2.0=h1234567_1
+ - libstdcxx-ng=11.2.0=h1234567_1
+ - libuuid=1.41.5=h5eee18b_0
+ - ncurses=6.4=h6a678d5_0
+ - openssl=3.0.15=h5eee18b_0
+ - pip=24.2=py310h06a4308_0
+ - python=3.10.15=he870216_1
+ - readline=8.2=h5eee18b_0
+ - setuptools=75.1.0=py310h06a4308_0
+ - sqlite=3.45.3=h5eee18b_0
+ - tk=8.6.14=h39e8969_0
+ - wheel=0.44.0=py310h06a4308_0
+ - xz=5.4.6=h5eee18b_1
+ - zlib=1.2.13=h5eee18b_1
+ - pip:
+ - absl-py==2.1.0
+ - aiohappyeyeballs==2.4.4
+ - aiohttp==3.11.9
+ - aiosignal==1.3.1
+ - antlr4-python3-runtime==4.9.3
+ - appdirs==1.4.4
+ - async-timeout==5.0.1
+ - attrs==24.2.0
+ - av==12.0.0
+ - beartype==0.18.2
+ - certifi==2024.8.30
+ - charset-normalizer==3.4.0
+ - click==8.1.7
+ - contourpy==1.3.1
+ - cycler==0.12.1
+ - decord==0.6.0
+ - docker-pycreds==0.4.0
+ - einops==0.8.0
+ - filelock==3.16.1
+ - fonttools==4.55.1
+ - frozenlist==1.5.0
+ - fsspec==2024.10.0
+ - gitdb==4.0.11
+ - gitpython==3.1.43
+ - grpcio==1.68.1
+ - idna==3.10
+ - imageio==2.34.0
+ - jinja2==3.1.4
+ - kiwisolver==1.4.7
+ - lightning==2.2.4
+ - lightning-utilities==0.11.9
+ - markdown==3.7
+ - markdown-it-py==3.0.0
+ - markupsafe==3.0.2
+ - matplotlib==3.8.4
+ - mdurl==0.1.2
+ - mpmath==1.3.0
+ - multidict==6.1.0
+ - natsort==8.4.0
+ - networkx==3.4.2
+ - numpy==1.26.4
+ - nvidia-cublas-cu12==12.1.3.1
+ - nvidia-cuda-cupti-cu12==12.1.105
+ - nvidia-cuda-nvrtc-cu12==12.1.105
+ - nvidia-cuda-runtime-cu12==12.1.105
+ - nvidia-cudnn-cu12==8.9.2.26
+ - nvidia-cufft-cu12==11.0.2.54
+ - nvidia-curand-cu12==10.3.2.106
+ - nvidia-cusolver-cu12==11.4.5.107
+ - nvidia-cusparse-cu12==12.1.0.106
+ - nvidia-nccl-cu12==2.19.3
+ - nvidia-nvjitlink-cu12==12.6.85
+ - nvidia-nvtx-cu12==12.1.105
+ - omegaconf==2.3.0
+ - opencv-python==4.6.0.66
+ - packaging==24.2
+ - pandas==2.1.4
+ - pillow==11.0.0
+ - propcache==0.2.1
+ - protobuf==4.25.5
+ - psutil==6.1.0
+ - pygments==2.18.0
+ - pyparsing==3.2.0
+ - python-dateutil==2.9.0.post0
+ - pytorch-lightning==2.2.4
+ - pytz==2024.2
+ - pyyaml==6.0.2
+ - requests==2.32.3
+ - rich==13.5.3
+ - safetensors==0.4.2
+ - sentry-sdk==2.19.0
+ - setproctitle==1.3.4
+ - six==1.17.0
+ - smmap==5.0.1
+ - sympy==1.13.3
+ - tensorboard==2.16.2
+ - tensorboard-data-server==0.7.2
+ - torch==2.2.2
+ - torchmetrics==1.6.0
+ - torchvision==0.17.2
+ - tqdm==4.67.1
+ - triton==2.2.0
+ - typing-extensions==4.12.2
+ - tzdata==2024.2
+ - urllib3==2.2.3
+ - wandb==0.16.6
+ - werkzeug==3.1.3
+ - yarl==1.18.3
diff --git a/Meissonic/vidtok_cache/VidTok/main.py b/Meissonic/vidtok_cache/VidTok/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..e03f0de8a1c683fbdde7177083c87a0a3df85f83
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/main.py
@@ -0,0 +1,1124 @@
+import argparse
+import datetime
+import pytz
+import glob
+import inspect
+import os
+import re
+import sys
+import numpy as np
+import warnings
+warnings.filterwarnings("ignore")
+from rich import print
+from inspect import Parameter
+from typing import Union
+from matplotlib import pyplot as plt
+from natsort import natsorted
+from omegaconf import OmegaConf
+from packaging import version
+from PIL import Image
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch import seed_everything
+from lightning.pytorch.trainer import Trainer
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from vidtok.modules.util import (exists, instantiate_from_config, isheatmap,
+ print0, seed_anything)
+
+MULTINODE_HACKS = True
+
+
+def default_trainer_args():
+ argspec = dict(inspect.signature(Trainer.__init__).parameters)
+ argspec.pop("self")
+ default_args = {
+ param: argspec[param].default
+ for param in argspec
+ if argspec[param] != Parameter.empty
+ }
+ return default_args
+
+
+def get_step_value(folder_name):
+ match = re.search(r"step=(\d+)", folder_name)
+ if match:
+ return int(match.group(1))
+ return 0
+
+
+def get_parser(**parser_kwargs):
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser(**parser_kwargs)
+ parser.add_argument(
+ "-n",
+ "--name",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="postfix for logdir",
+ )
+ parser.add_argument(
+ "--no_date",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="if True, skip date generation for logdir and only use naming via opt.base or opt.name (+ opt.postfix, optionally)",
+ )
+ parser.add_argument(
+ "-r",
+ "--resume",
+ type=str,
+ const=True,
+ default="",
+ nargs="?",
+ help="resume from logdir or checkpoint in logdir",
+ )
+ parser.add_argument(
+ "-b",
+ "--base",
+ nargs="*",
+ metavar="base_config.yaml",
+ help="paths to base configs. Loaded from left-to-right. "
+ "Parameters can be overwritten or added with command-line options of the form `--key value`.",
+ default=list(),
+ )
+ parser.add_argument(
+ "-t",
+ "--train",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="train",
+ )
+ parser.add_argument(
+ "--no-test",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="disable test",
+ )
+ parser.add_argument(
+ "-p", "--project", help="name of new or path to existing project"
+ )
+ parser.add_argument(
+ "-d",
+ "--debug",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="enable post-mortem debugging",
+ )
+ parser.add_argument(
+ "-s",
+ "--seed",
+ type=int,
+ default=23,
+ help="seed for seed_everything",
+ )
+ parser.add_argument(
+ "--seed_rank",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="reset seed every rank on fit start",
+ )
+ parser.add_argument(
+ "-f",
+ "--postfix",
+ type=str,
+ default="",
+ help="post-postfix for default name",
+ )
+ parser.add_argument(
+ "-l",
+ "--logdir",
+ type=str,
+ default="logs",
+ help="directory for logging dat shit",
+ )
+ parser.add_argument(
+ "--scale_lr",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="scale base-lr by ngpu * batch_size * n_accumulate",
+ )
+ parser.add_argument(
+ "--legacy_naming",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="name run based on config file name if true, else by whole path",
+ )
+ parser.add_argument(
+ "--enable_tf32",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="enables the TensorFloat32 format both for matmuls and cuDNN for pytorch 1.12",
+ )
+ parser.add_argument(
+ "--startup",
+ type=str,
+ default=None,
+ help="Startuptime from distributed script",
+ )
+ parser.add_argument(
+ "--wandb",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False,
+ help="log to wandb",
+ )
+ parser.add_argument(
+ "--wandb_entity",
+ type=str,
+ default="",
+ help="Wandb entity name string",
+ )
+ parser.add_argument(
+ "--wandb_key",
+ type=str,
+ default="",
+ help="Wandb key",
+ )
+ parser.add_argument(
+ "--wandb_project",
+ type=str,
+ default="vidtok",
+ )
+ parser.add_argument(
+ "--wandb_id",
+ type=str,
+ default=None,
+ help="automatically resume from the same wandb id"
+ "must be used in combination with --wandb_auto_resume False",
+ )
+ parser.add_argument(
+ "--wandb_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest run id in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--checkpoint_auto_resume",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=True,
+ help="will find the latest checkpoint in the logdir"
+ "if checkpoint_auto_resume is False, wandb_auto_resume will be ignored",
+ )
+ parser.add_argument(
+ "--no_base_name",
+ type=str2bool,
+ nargs="?",
+ const=True,
+ default=False, # TODO: later default to True
+ help="log to wandb",
+ )
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ parser.add_argument(
+ "--resume_from_checkpoint",
+ type=str,
+ default=None,
+ help="single checkpoint file to resume from",
+ )
+ default_args = default_trainer_args()
+ for key in default_args:
+ # parameters in the pl.Trainer are passed as --key value
+ parser.add_argument("--" + key, default=default_args[key])
+ return parser
+
+
+def get_checkpoint_name(logdir):
+ ckpt = os.path.join(logdir, "checkpoints", "last**.ckpt")
+ ckpt = natsorted(glob.glob(ckpt))
+ print0('available "last" checkpoints:')
+ print0(ckpt)
+ if len(ckpt) > 1:
+ print0("got most recent checkpoint")
+ ckpt = sorted(ckpt, key=lambda x: os.path.getmtime(x))[-1]
+ print0(f"Most recent ckpt is {ckpt}")
+ with open(os.path.join(logdir, "most_recent_ckpt.txt"), "w") as f:
+ f.write(ckpt + "\n")
+ try:
+ version = int(ckpt.split("/")[-1].split("-v")[-1].split(".")[0])
+ except Exception as e:
+ print0("version confusion but not bad")
+ print0(e)
+ version = 1
+ # version = last_version + 1
+ else:
+ # in this case, we only have one "last.ckpt"
+ ckpt = ckpt[0]
+ version = 1
+ melk_ckpt_name = f"last-v{version}.ckpt"
+ print0(f"Current melk ckpt name: {melk_ckpt_name}")
+ return ckpt, melk_ckpt_name
+
+
+class SetupCallback(Callback):
+ def __init__(
+ self,
+ resume,
+ now,
+ logdir,
+ ckptdir,
+ cfgdir,
+ config,
+ lightning_config,
+ debug,
+ save_ckpt_on_exception=False,
+ ckpt_name=None,
+ seed=None,
+ seed_rank=False,
+ ):
+ super().__init__()
+ self.resume = resume
+ self.now = now
+ self.logdir = logdir
+ self.ckptdir = ckptdir
+ self.cfgdir = cfgdir
+ self.config = config
+ self.lightning_config = lightning_config
+ self.debug = debug
+ self.save_ckpt_on_exception = save_ckpt_on_exception
+ self.ckpt_name = ckpt_name
+ self.seed = seed
+ self.seed_rank = seed_rank
+
+ def on_exception(self, trainer: pl.Trainer, pl_module, exception):
+ if self.save_ckpt_on_exception and (not self.debug) and (trainer.global_rank == 0):
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Saving checkpoint to {self.ckptdir}")
+ if self.ckpt_name is None:
+ ckpt_path = os.path.join(self.ckptdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(self.ckptdir, self.ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def on_fit_start(self, trainer, pl_module):
+ if self.seed_rank:
+ # current_seed = torch.initial_seed()
+ seed_anything(self.seed + trainer.global_rank)
+ print(f"[bold red]\[main][SetupCallback][/bold red] Rank {trainer.global_rank}: Reset GLOBAL seed to {self.seed + trainer.global_rank}")
+ elif hasattr(pl_module, "set_seed") and callable(pl_module.set_seed):
+ pl_module.set_seed(self.seed)
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Set pl_module seed to {self.seed} with pl_module.set_seed")
+ if trainer.global_rank == 0:
+ # Create logdirs and save configs
+ print0(f"[bold red]\[main][SetupCallback][/bold red] Creating logdir: {self.logdir}, ckptdir: {self.ckptdir}, cfgdir: {self.cfgdir}")
+ os.makedirs(self.logdir, exist_ok=True)
+ os.makedirs(self.ckptdir, exist_ok=True)
+ os.makedirs(self.cfgdir, exist_ok=True)
+
+ if "callbacks" in self.lightning_config:
+ if (
+ "metrics_over_trainsteps_checkpoint"
+ in self.lightning_config["callbacks"]
+ ):
+ os.makedirs(
+ os.path.join(self.ckptdir, "trainstep_checkpoints"),
+ exist_ok=True,
+ )
+ print0("[bold red]\[main][SetupCallback][/bold red] Project config")
+ print0(OmegaConf.to_yaml(self.config))
+ if MULTINODE_HACKS and not self.debug:
+ import time
+ time.sleep(5)
+ OmegaConf.save(
+ self.config,
+ os.path.join(self.cfgdir, "{}-project.yaml".format(self.now)),
+ )
+
+ print0("[bold red]\[main][SetupCallback][/bold red] Lightning config")
+ print0(OmegaConf.to_yaml(self.lightning_config))
+ OmegaConf.save(
+ OmegaConf.create({"lightning": self.lightning_config}),
+ os.path.join(self.cfgdir, "{}-lightning.yaml".format(self.now)),
+ )
+
+ else:
+ # ModelCheckpoint callback created log directory --- remove it
+ if not MULTINODE_HACKS and not self.resume and os.path.exists(self.logdir):
+ dst, name = os.path.split(self.logdir)
+ dst = os.path.join(dst, "child_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ try:
+ os.rename(self.logdir, dst)
+ except FileNotFoundError:
+ pass
+
+
+class ImageLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ rescale=True,
+ disabled=True,
+ log_on_batch_idx=False,
+ log_first_step=False,
+ log_images_kwargs=None,
+ log_before_first_step=False,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ # TODO: support wandb
+ else:
+ grid = torchvision.utils.make_grid(images[k], nrow=4)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ self.check_frequency(check_idx)
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ gpu_autocast_kwargs = {
+ "enabled": self.enable_autocast, # torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad(), torch.cuda.amp.autocast(**gpu_autocast_kwargs):
+ images = pl_module.log_images(
+ batch, split=split, **self.log_images_kwargs
+ )
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ print0("[bold red]\[main][ImageLogger][/bold red]", e)
+ pass
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ print0(f"[bold red]\[main][ImageLogger][/bold red] {self.__class__.__name__}: logging before training")
+ self.log_img(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+@rank_zero_only
+def init_wandb(save_dir, opt, config, group_name, name_str):
+ print0(f"[bold red]\[main][init_wandb][/bold red] Creating WANDB_DIR: {save_dir}")
+ os.makedirs(save_dir, exist_ok=True)
+
+ # os.environ["WANDB_DIR"] = save_dir
+ gitcmd = f'git config --global --add safe.directory {os.path.dirname(os.path.abspath(__file__))}'
+ os.system(gitcmd)
+ print0(f"[bold red]\[main][init_wandb][/bold red] wandb_id is set to {opt.wandb_id}")
+ wandb_id = opt.wandb_id if opt.wandb_id is not None else name_str
+
+ if not wandb.api.api_key:
+ wandb.login(key=opt.wandb_key)
+ if opt.debug:
+ wandb.init(project=opt.wandb_project, mode="offline", group=group_name)
+ else:
+ wandb.init(
+ project=opt.wandb_project,
+ entity=opt.wandb_entity,
+ config=dict(config),
+ group=group_name,
+ name=name_str,
+ resume='auto',
+ id=wandb_id,
+ )
+
+
+if __name__ == "__main__":
+ # custom parser to specify config files, train, test and debug mode,
+ # postfix, resume.
+ # `--key value` arguments are interpreted as arguments to the trainer.
+ # `nested.key=value` arguments are interpreted as config parameters.
+ # configs are merged from left-to-right followed by command line parameters.
+
+ # model:
+ # base_learning_rate: float
+ # target: path to lightning module
+ # params:
+ # key: value
+ # data:
+ # target: main.DataModuleFromConfig
+ # params:
+ # batch_size: int
+ # wrap: bool
+ # train:
+ # target: path to train dataset
+ # params:
+ # key: value
+ # validation:
+ # target: path to validation dataset
+ # params:
+ # key: value
+ # test:
+ # target: path to test dataset
+ # params:
+ # key: value
+ # lightning: (optional, has sane defaults and can be specified on cmdline)
+ # trainer:
+ # additional arguments to trainer
+ # logger:
+ # logger to instantiate
+ # modelcheckpoint:
+ # modelcheckpoint to instantiate
+ # callbacks:
+ # callback1:
+ # target: importpath
+ # params:
+ # key: value
+
+ now = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+
+ # add cwd for convenience and to make classes in this file available when
+ # running as `python main.py`
+ # (in particular `main.DataModuleFromConfig`)
+ sys.path.append(os.getcwd())
+
+ parser = get_parser()
+
+ opt, unknown = parser.parse_known_args()
+
+ if opt.name and opt.resume:
+ raise ValueError(
+ "-n/--name and -r/--resume cannot be specified both."
+ "If you want to resume training in a new log folder, "
+ "use -n/--name in combination with --resume_from_checkpoint"
+ )
+ melk_ckpt_name = None
+ name = None
+ if opt.resume:
+ if not os.path.exists(opt.resume):
+ raise ValueError("Cannot find {}".format(opt.resume))
+ if os.path.isfile(opt.resume):
+ paths = opt.resume.split("/")
+ # idx = len(paths)-paths[::-1].index("logs")+1
+ # logdir = "/".join(paths[:idx])
+ logdir = "/".join(paths[:-2])
+ ckpt = opt.resume
+ _, melk_ckpt_name = get_checkpoint_name(logdir)
+ else:
+ assert os.path.isdir(opt.resume), opt.resume
+ logdir = opt.resume.rstrip("/")
+ ckpt, melk_ckpt_name = get_checkpoint_name(logdir)
+
+ print0("-" * 80)
+ print0(f'[bold red][main][/bold red] Resuming from checkpoint "{ckpt}"')
+
+ opt.resume_from_checkpoint = ckpt
+ base_configs = sorted(glob.glob(os.path.join(logdir, "configs/*.yaml")))
+ opt.base = base_configs + opt.base
+ _tmp = logdir.split("/")
+ nowname = _tmp[-1]
+ else:
+ if opt.name:
+ name = "_" + opt.name
+ elif opt.base:
+ if opt.no_base_name:
+ name = ""
+ else:
+ if opt.legacy_naming:
+ cfg_fname = os.path.split(opt.base[0])[-1]
+ cfg_name = os.path.splitext(cfg_fname)[0]
+ else:
+ assert "configs" in os.path.split(opt.base[0])[0], os.path.split(
+ opt.base[0]
+ )[0]
+ cfg_path = os.path.split(opt.base[0])[0].split(os.sep)[
+ os.path.split(opt.base[0])[0].split(os.sep).index("configs")
+ + 1 :
+ ] # cut away the first one (we assert all configs are in "configs")
+ cfg_name = os.path.splitext(os.path.split(opt.base[0])[-1])[0]
+ cfg_name = "-".join(cfg_path) + f"-{cfg_name}"
+ name = "_" + cfg_name
+ else:
+ name = ""
+ # automatic resume last checkpoint if available
+ if os.path.exists(opt.logdir):
+ auto_resumed = False
+ for sub_dir in sorted(os.listdir(opt.logdir)):
+ if sub_dir.endswith(name + opt.postfix):
+ ## checkpoint resume
+ if opt.checkpoint_auto_resume and not opt.debug:
+ checkpoint_dir = os.path.join(opt.logdir, sub_dir, "checkpoints")
+ # Use the max step checkpoint file
+ ckpt_files1 = glob.glob(os.path.join(checkpoint_dir, "*/*.ckpt"))
+ ckpt_files2 = glob.glob(os.path.join(checkpoint_dir, "*.ckpt"))
+ ckpt_files = ckpt_files1 + ckpt_files2
+ ckpt_files.sort(key=get_step_value, reverse=True)
+ if ckpt_files:
+ ckpt = ckpt_files[0]
+ else:
+ # If no checkpoint files found, use a random initialized model
+ ckpt = None
+ if ckpt is not None and os.path.isfile(ckpt):
+ opt.resume_from_checkpoint = ckpt
+ auto_resumed = True
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous log dir and checkpoint: {ckpt}")
+ ## wandb resume
+ if opt.wandb_auto_resume:
+ wandb_dir = Path(os.path.join(opt.logdir, sub_dir)) / "wandb"
+ if wandb_dir.exists() and any((wandb_dir / "latest-run").iterdir()):
+ # Parse unique `run_id` from the `.wandb.` file...
+ wandb_fns = [f.name for f in (wandb_dir / "latest-run").iterdir() if f.name.endswith(".wandb")]
+ assert len(wandb_fns) == 1, f"There should only be 1 `.wandb.` file... found {len(wandb_fns)}!"
+ # Regex Match on `run-{id}.wandb`
+ opt.wandb_id = re.search("run-(.+?).wandb", wandb_fns[0]).group(1)
+ # print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Find previous wandb run id: {opt.wandb_id}")
+ if auto_resumed:
+ print0(f"[bold red]\[main][/bold red] Auto-resuming from checkpoint: {opt.resume_from_checkpoint} and wandb id: {opt.wandb_id}")
+ ckpt_basename = os.path.basename(opt.resume_from_checkpoint)
+ seed_str = ''.join(re.findall(r'\d+', ckpt_basename))
+ if len(seed_str) > 0:
+ opt.seed = int(seed_str)
+ print0(f"[bold red]\[main][/bold red] Auto-reseting seed to {opt.seed} from checkpoint name")
+
+ if not opt.no_date:
+ nowname = now + name + opt.postfix
+ else:
+ nowname = name + opt.postfix
+ if nowname.startswith("_"):
+ nowname = nowname[1:]
+ logdir = os.path.join(opt.logdir, nowname)
+ print0(f"[bold red]\[main][/bold red] LOGDIR: {logdir}")
+
+ ckptdir = os.path.join(logdir, "checkpoints")
+ cfgdir = os.path.join(logdir, "configs")
+ if not opt.seed_rank:
+ seed_everything(opt.seed, workers=True) # torch.initial_seed()
+
+ # move before model init, in case a torch.compile(...) is called somewhere
+ if opt.enable_tf32:
+ # pt_version = version.parse(torch.__version__)
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.backends.cudnn.allow_tf32 = True
+ print0(f"[bold red]\[main][/bold red] Enabling TF32 for PyTorch {torch.__version__}")
+ else:
+ print0(f"[bold red]\[main][/bold red] Using default TF32 settings for PyTorch {torch.__version__}:")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cuda.matmul.allow_tf32={torch.backends.cuda.matmul.allow_tf32}")
+ print0(f"[bold red]\[main][/bold red] torch.backends.cudnn.allow_tf32={torch.backends.cudnn.allow_tf32}")
+
+ try:
+ # init and save configs
+ configs = [OmegaConf.load(cfg) for cfg in opt.base]
+ # deal with the unknown args, e.g., --model.base_learning_rate=1.0e-4
+ for i, u in enumerate(unknown):
+ if u.startswith("--"):
+ unknown[i] = u[2:]
+ # merge all configs and cli args
+ cli = OmegaConf.from_dotlist(unknown)
+ config = OmegaConf.merge(*configs, cli)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged input config: {config}")
+ lightning_config = config.pop("lightning", OmegaConf.create())
+ # merge trainer cli with config
+ trainer_config = lightning_config.get("trainer", OmegaConf.create())
+
+ # debug: default to one node
+ if opt.debug:
+ trainer_config["num_nodes"] = 1
+
+ # default profiler
+ trainer_config["profiler"] = None if not opt.debug else "simple"
+
+ # default to gpu
+ trainer_config["accelerator"] = "gpu"
+ #
+ standard_args = default_trainer_args()
+ for k in standard_args:
+ if getattr(opt, k) != standard_args[k]:
+ trainer_config[k] = getattr(opt, k)
+
+ if not "devices" in trainer_config and trainer_config["accelerator"] != "gpu":
+ del trainer_config["accelerator"]
+ cpu = True
+ else:
+ gpuinfo = trainer_config["devices"]
+ print0(f"[bold red]\[main][/bold red] Running on {gpuinfo} GPUs")
+ cpu = False
+ trainer_opt = argparse.Namespace(**trainer_config)
+ lightning_config.trainer = trainer_config
+
+ # model
+ model = instantiate_from_config(config.model)
+
+ # trainer and callbacks
+ trainer_kwargs = dict()
+
+ # default logger configs
+ default_logger_cfgs = {
+ "wandb": {
+ "target": "lightning.pytorch.loggers.WandbLogger",
+ "params": {
+ "name": nowname,
+ "save_dir": logdir,
+ "offline": opt.debug,
+ "id": nowname,
+ "project": opt.wandb_project,
+ "log_model": False,
+ "entity": opt.wandb_entity,
+ },
+ },
+ "csv": {
+ "target": "lightning.pytorch.loggers.CSVLogger",
+ "params": {
+ "name": "testtube", # hack for sbord fanatics
+ "save_dir": logdir,
+ },
+ },
+ "tensorboard": {
+ "target": "lightning.pytorch.loggers.TensorBoardLogger",
+ "params": {
+ "save_dir": logdir,
+ "name": 'tensorboard',
+ "version": nowname,
+ }
+ },
+ }
+ default_logger_cfg = default_logger_cfgs["wandb" if opt.wandb else "tensorboard"]
+ if opt.wandb:
+ # change once leaving "swiffer" config directory
+ try:
+ group_name = nowname.split(now)[-1].split("-")[1]
+ except:
+ group_name = nowname
+ default_logger_cfg["params"]["group"] = group_name
+
+ wandb_save_dir = os.path.join(os.getcwd(), logdir)
+ os.environ["WANDB_DIR"] = wandb_save_dir
+
+ init_wandb(
+ wandb_save_dir,
+ opt=opt,
+ group_name=group_name,
+ config=config,
+ name_str=nowname,
+ )
+ if "logger" in lightning_config:
+ logger_cfg = lightning_config.logger
+ else:
+ logger_cfg = OmegaConf.create()
+ logger_cfg = OmegaConf.merge(default_logger_cfg, logger_cfg)
+ trainer_kwargs["logger"] = instantiate_from_config(logger_cfg)
+
+ ckpt_resume_path = opt.resume_from_checkpoint
+
+ # modelcheckpoint - use TrainResult/EvalResult(checkpoint_on=metric) to
+ # specify which metric is used to determine best models
+ default_modelckpt_cfg = {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": ckptdir,
+ "filename": "{epoch:04}-{step:08}", # "epoch={epoch:06}-step={step:07}"
+ "verbose": True,
+ "save_last": True,
+ "auto_insert_metric_name": True,
+ },
+ }
+ if hasattr(model, "monitor"):
+ print0(f"[bold red]\[main][/bold red] Monitoring {model.monitor} as checkpoint metric.")
+ default_modelckpt_cfg["params"]["monitor"] = model.monitor
+ default_modelckpt_cfg["params"]["save_top_k"] = 3
+
+ if "modelcheckpoint" in lightning_config:
+ modelckpt_cfg = lightning_config.modelcheckpoint
+ else:
+ modelckpt_cfg = OmegaConf.create()
+ modelckpt_cfg = OmegaConf.merge(default_modelckpt_cfg, modelckpt_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] Merged modelckpt-cfg: {modelckpt_cfg}")
+
+ # https://pytorch-lightning.readthedocs.io/en/stable/extensions/strategy.html
+ # default to ddp if not further specified
+ default_strategy_config = {"target": "lightning.pytorch.strategies.DDPStrategy"}
+
+ if "strategy" in lightning_config:
+ strategy_cfg = lightning_config.strategy
+ else:
+ strategy_cfg = OmegaConf.create()
+ default_strategy_config["params"] = {
+ "find_unused_parameters": False,
+ # "static_graph": True,
+ # "ddp_comm_hook": default.fp16_compress_hook # experiment with this, also for DDPSharded
+ }
+ strategy_cfg = OmegaConf.merge(default_strategy_config, strategy_cfg)
+ print0("-" * 80)
+ print0(f"[bold red]\[main][/bold red] strategy config: {strategy_cfg}")
+ trainer_kwargs["strategy"] = instantiate_from_config(strategy_cfg)
+ if hasattr(trainer_kwargs["strategy"], "_timeout"):
+ trainer_kwargs["strategy"]._timeout = datetime.timedelta(seconds=5400) # 3600s = 1h
+
+ # add callback which sets up log directory
+ default_callbacks_cfg = {
+ "setup_callback": {
+ "target": "main.SetupCallback",
+ "params": {
+ "resume": opt.resume,
+ "now": now,
+ "logdir": logdir,
+ "ckptdir": ckptdir,
+ "cfgdir": cfgdir,
+ "config": config,
+ "lightning_config": lightning_config,
+ "debug": opt.debug,
+ "ckpt_name": melk_ckpt_name,
+ "seed": opt.seed,
+ "seed_rank": opt.seed_rank
+ },
+ },
+ "image_logger": {
+ "target": "main.ImageLogger",
+ "params": {"batch_frequency": 1000, "max_samples": 4, "clamp": True},
+ },
+ "learning_rate_logger": {
+ "target": "lightning.pytorch.callbacks.LearningRateMonitor",
+ "params": {
+ "logging_interval": "step",
+ # "log_momentum": True
+ },
+ },
+ }
+ if version.parse(pl.__version__) >= version.parse("1.4.0"):
+ default_callbacks_cfg.update({"checkpoint_callback": modelckpt_cfg})
+
+ if "callbacks" in lightning_config:
+ callbacks_cfg = lightning_config.callbacks
+ else:
+ callbacks_cfg = OmegaConf.create()
+
+ if "metrics_over_trainsteps_checkpoint" in callbacks_cfg:
+ print0(
+ "[bold red]\[main][/bold red] Caution: Saving checkpoints every n train steps without deleting. This might require some free space."
+ )
+ default_metrics_over_trainsteps_ckpt_dict = {
+ "metrics_over_trainsteps_checkpoint": {
+ "target": "lightning.pytorch.callbacks.ModelCheckpoint",
+ "params": {
+ "dirpath": os.path.join(ckptdir, "trainstep_checkpoints"),
+ "filename": "{epoch:04}-{step:08}", # "{epoch:06}-{step:09}"
+ "verbose": True,
+ "save_top_k": -1,
+ "every_n_train_steps": 10000,
+ "save_weights_only": True,
+ },
+ }
+ }
+ default_callbacks_cfg.update(default_metrics_over_trainsteps_ckpt_dict)
+
+ callbacks_cfg = OmegaConf.merge(default_callbacks_cfg, callbacks_cfg)
+ if "ignore_keys_callback" in callbacks_cfg and ckpt_resume_path is not None:
+ callbacks_cfg.ignore_keys_callback.params["ckpt_path"] = ckpt_resume_path
+ elif "ignore_keys_callback" in callbacks_cfg:
+ del callbacks_cfg["ignore_keys_callback"]
+
+ trainer_kwargs["callbacks"] = [
+ instantiate_from_config(callbacks_cfg[k]) for k in callbacks_cfg
+ ]
+ if not "plugins" in trainer_kwargs:
+ trainer_kwargs["plugins"] = list()
+
+ # cmd line trainer args (which are in trainer_opt) have always priority over config-trainer-args (which are in trainer_kwargs)
+ trainer_opt = vars(trainer_opt)
+ trainer_kwargs = {
+ key: val for key, val in trainer_kwargs.items() if key not in trainer_opt
+ }
+ trainer = Trainer(**trainer_opt, **trainer_kwargs)
+
+ trainer.logdir = logdir
+
+ # data
+ if ((not opt.train) or opt.debug) and hasattr(config.data.params, "validation"):
+ config.data.params.train = config.data.params.validation
+ print0("[bold red]\[main][/bold red] Using validation data as training data for fast loading.")
+ data = instantiate_from_config(config.data)
+ # NOTE according to https://pytorch-lightning.readthedocs.io/en/latest/datamodules.html
+ # calling these ourselves should not be necessary but it is.
+ # lightning still takes care of proper multiprocessing though
+ data.prepare_data()
+ # data.setup()
+ try:
+ for k in data.datasets:
+ print0(
+ f"[bold red]\[main][/bold red] {k}, {data.datasets[k].__class__.__name__}, {len(data.datasets[k])}"
+ )
+ except:
+ print0("[bold red]\[main][/bold red] datasets not yet initialized.")
+
+ # configure learning rate
+ if "batch_size" in config.data.params:
+ bs, base_lr = config.data.params.batch_size, config.model.base_learning_rate
+ else:
+ bs, base_lr = (
+ config.data.params.train.loader.batch_size,
+ config.model.base_learning_rate,
+ )
+ if not cpu:
+ # add for different device input type
+ if isinstance(lightning_config.trainer.devices, int):
+ ngpu = lightning_config.trainer.devices
+ elif isinstance(lightning_config.trainer.devices, list):
+ ngpu = len(lightning_config.trainer.devices)
+ elif isinstance(lightning_config.trainer.devices, str):
+ ngpu = len(lightning_config.trainer.devices.strip(",").split(","))
+ else:
+ ngpu = 1
+ if "accumulate_grad_batches" in lightning_config.trainer:
+ accumulate_grad_batches = lightning_config.trainer.accumulate_grad_batches
+ else:
+ accumulate_grad_batches = 1
+ print0(f"[bold red]\[main][/bold red] accumulate_grad_batches = {accumulate_grad_batches}")
+ lightning_config.trainer.accumulate_grad_batches = accumulate_grad_batches
+
+ if opt.scale_lr:
+ model.learning_rate = accumulate_grad_batches * ngpu * bs * base_lr
+ print0(
+ "[bold red]\[main][/bold red] Setting learning rate to {:.2e} = {} (accumulate_grad_batches) * {} (num_gpus) * {} (batchsize) * {:.2e} (base_lr)".format(
+ model.learning_rate, accumulate_grad_batches, ngpu, bs, base_lr
+ )
+ )
+ else:
+ model.learning_rate = base_lr
+ print0("[bold red]\[main][/bold red] NOT using learning rate scaling")
+ print0(f"[bold red]\[main][/bold red] Setting learning rate to {model.learning_rate:.2e}")
+
+ # allow checkpointing via USR1
+ def melk(*args, **kwargs):
+ # run all checkpoint hooks
+ if trainer.global_rank == 0:
+ melkdir = os.path.join(logdir, "melk")
+ os.makedirs(melkdir, exist_ok=True)
+ print0(f"[bold red]\[main][/bold red] Saving checkpoint to {melkdir}")
+ if melk_ckpt_name is None:
+ ckpt_path = os.path.join(melkdir, "last.ckpt")
+ else:
+ ckpt_path = os.path.join(melkdir, melk_ckpt_name)
+ trainer.save_checkpoint(ckpt_path)
+
+ def divein(*args, **kwargs):
+ if trainer.global_rank == 0:
+ import pudb
+ pudb.set_trace()
+
+ import signal
+ signal.signal(signal.SIGUSR1, melk)
+ signal.signal(signal.SIGUSR2, divein)
+
+ # run
+ if opt.train:
+ try:
+ trainer.fit(model, data, ckpt_path=ckpt_resume_path)
+ print0(f"[bold red]\[main][/bold red] Finish training with logdir: {logdir}")
+ except Exception as e:
+ print(f"")
+ print(f"[bold red]\[main][/bold red] Exception: {e}")
+ print(f"[bold red]\[main][/bold red] Beijing Time {datetime.datetime.now(tz=pytz.timezone('Asia/Shanghai'))}")
+ if not opt.debug:
+ melk()
+ raise
+ else:
+ trainer.validate(model, data, ckpt_path=ckpt_resume_path)
+ exit()
+ if not opt.no_test and not trainer.interrupted:
+ trainer.test(model, data)
+ except RuntimeError as err:
+ if MULTINODE_HACKS:
+ import datetime
+ import os
+ import socket
+ import requests
+
+ device = os.environ.get("CUDA_VISIBLE_DEVICES", "?")
+ hostname = socket.gethostname()
+ ts = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+ resp = requests.get("http://169.254.169.254/latest/meta-data/instance-id")
+ print(
+ f"[bold red]\[main][/bold red] ERROR at {ts} on {hostname}/{resp.text} (CUDA_VISIBLE_DEVICES={device}): {type(err).__name__}: {err}",
+ flush=True,
+ )
+ raise err
+ except Exception:
+ if opt.debug and trainer.global_rank == 0:
+ try:
+ import pudb as debugger
+ except ImportError:
+ import pdb as debugger
+ # debugger.post_mortem()
+ raise
+ finally:
+ # move newly created debug project to debug_runs
+ if opt.debug and not opt.resume and trainer.global_rank == 0:
+ dst, name = os.path.split(logdir)
+ dst = os.path.join(dst, "debug_runs", name)
+ os.makedirs(os.path.split(dst)[0], exist_ok=True)
+ os.rename(logdir, dst)
+
+ if opt.wandb:
+ wandb.finish()
+
+ # clean up
+ # dist.barrier()
+ # torch.cuda.empty_cache()
+ dist.destroy_process_group()
+
+ if trainer.global_rank == 0 and opt.debug:
+ print0(f"[bold red]\[main][/bold red] Current logdir: {logdir}")
+ # print0(f"[bold red]\[main][/bold red] Profiler summary:")
+ # print(trainer.profiler.summary())
+ print0(f"[bold red]\[main][/bold red] Memory summary:")
+ num_params = sum([p.numel() for p in model.parameters()])
+ print0(f"[bold red]\[main][/bold red] Expected bf16 memory usage from params: {num_params * 2 / 1e9:.2f} GB")
+ print0(f"[bold red]\[main][/bold red] Current memory usage with model on device {torch.cuda.max_memory_allocated() / 1e9:.2f} GB")
+ # trainer.print(torch.cuda.memory_summary())
diff --git a/Meissonic/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc b/Meissonic/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c3073fc0399093458876a613ed5102383d038820
Binary files /dev/null and b/Meissonic/vidtok_cache/VidTok/scripts/__pycache__/inference_evaluate.cpython-310.pyc differ
diff --git a/Meissonic/vidtok_cache/VidTok/scripts/inference_evaluate.py b/Meissonic/vidtok_cache/VidTok/scripts/inference_evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4152046727305af7b32e0a3e5728072b7085ca4
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/scripts/inference_evaluate.py
@@ -0,0 +1,198 @@
+import argparse
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from torch import autocast
+from tqdm import tqdm
+
+import numpy as np
+import torch
+from einops import rearrange
+from lightning.pytorch import seed_everything
+
+from vidtok.data.vidtok import VidTokValDataset
+from vidtok.modules.lpips import LPIPS
+from vidtok.modules.util import (compute_psnr, compute_ssim,
+ instantiate_from_config, print0)
+
+
+def load_model_from_config(config, ckpt, ignore_keys=[], verbose=False):
+ config = OmegaConf.load(config)
+ config.model.params.ckpt_path = ckpt
+ config.model.params.ignore_keys = ignore_keys
+ config.model.params.verbose = verbose
+ model = instantiate_from_config(config.model)
+ return model
+
+
+class MultiVideoDataset(VidTokValDataset):
+ def __init__(
+ self,
+ data_dir,
+ meta_path=None,
+ input_height=256,
+ input_width=256,
+ sample_fps=30,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ super().__init__(
+ data_dir=data_dir,
+ meta_path=meta_path,
+ video_params={
+ "input_height": input_height,
+ "input_width": input_width,
+ "sample_num_frames": chunk_size + 1 if is_causal else chunk_size,
+ "sample_fps": sample_fps,
+ },
+ pre_load_frames=True,
+ last_frames_handle="repeat",
+ read_long_video=read_long_video,
+ chunk_size=chunk_size,
+ is_causal=is_causal,
+ )
+
+ def __getitem__(self, idx):
+ frames = super().__getitem__(idx)["jpg"]
+ return frames
+
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--data_dir",
+ type=str,
+ default="./",
+ help="root folder",
+ )
+ parser.add_argument(
+ "--meta_path",
+ type=str,
+ default=None,
+ help="path to the .csv meta file",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.enable_tiling()
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+
+ if args.input_width > 256:
+ model.enable_tiling()
+
+ dataset = MultiVideoDataset(
+ data_dir=args.data_dir,
+ meta_path=args.meta_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ perceptual_loss = LPIPS().eval()
+ perceptual_loss = perceptual_loss.to(device)
+
+ psnrs, ssims, lpipss = [], [], []
+
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+ _, output, reg_log = model(input)
+ output = output.clamp(-1, 1)
+ input, output = map(lambda x: (x + 1) / 2, (input, output))
+
+ if input.dim() == 5:
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ assert output.dim() == 5
+ output = rearrange(output, "b c t h w -> (b t) c h w")
+
+ for inp, out in zip(torch.split(input, 16), torch.split(output, 16)):
+ psnrs += [compute_psnr(inp, out).item()] * inp.shape[0]
+ ssims += [compute_ssim(inp, out).item()] * inp.shape[0]
+ lpipss += [perceptual_loss(inp * 2 - 1, out * 2 - 1).mean().item()] * inp.shape[0]
+
+ toc = time.time()
+ print0(
+ f"[bold red]\[scripts.inference_evaluate][/bold red] PSNR: {np.mean(psnrs):.4f}, SSIM: {np.mean(ssims):.4f}, LPIPS: {np.mean(lpipss):.4f}"
+ )
+ print0(f"[bold red]\[scripts.inference_evaluate][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/vidtok_cache/VidTok/scripts/inference_reconstruct.py b/Meissonic/vidtok_cache/VidTok/scripts/inference_reconstruct.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a26b9475339f6675e01fd052637c8465ca37caf
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/scripts/inference_reconstruct.py
@@ -0,0 +1,246 @@
+import os
+import sys
+sys.path.append(os.getcwd())
+
+import argparse
+import warnings
+warnings.filterwarnings("ignore")
+
+import time
+from contextlib import nullcontext
+from omegaconf import OmegaConf
+from pathlib import Path
+from tqdm import tqdm
+
+import numpy as np
+import torch
+import decord
+from einops import rearrange
+from lightning.pytorch import seed_everything
+from torch import autocast
+from torchvision import transforms
+from torchvision.io import write_video
+
+from vidtok.modules.util import print0
+from scripts.inference_evaluate import load_model_from_config
+
+
+class SingleVideoDataset(torch.utils.data.Dataset):
+ def __init__(
+ self,
+ video_path,
+ input_height=128,
+ input_width=128,
+ sample_fps=8,
+ chunk_size=16,
+ is_causal=True,
+ read_long_video=False
+ ):
+ decord.bridge.set_bridge("torch")
+ self.video_path = video_path
+ self.transform = transforms.Compose(
+ [
+ transforms.Resize(input_height, antialias=True),
+ transforms.CenterCrop((input_height, input_width)),
+ transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
+ ]
+ )
+
+ self.video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(self.video_reader)
+ fps = self.video_reader.get_avg_fps() # float
+
+ interval = round(fps / sample_fps)
+ frame_ids = list(range(0, total_frames, interval))
+ self.frame_ids_batch = []
+ if read_long_video:
+ video_length = len(frame_ids)
+ if is_causal and video_length > chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * ((video_length - 1) // chunk_size) + 1])
+ elif not is_causal and video_length >= chunk_size:
+ self.frame_ids_batch.append(frame_ids[:chunk_size * (video_length // chunk_size)])
+ else:
+ num_frames_per_batch = chunk_size + 1 if is_causal else chunk_size
+ for x in range(0, len(frame_ids), num_frames_per_batch):
+ if len(frame_ids[x : x + num_frames_per_batch]) == num_frames_per_batch:
+ self.frame_ids_batch.append(frame_ids[x : x + num_frames_per_batch])
+
+ def __len__(self):
+ return len(self.frame_ids_batch)
+
+ def __getitem__(self, idx):
+ frame_ids = self.frame_ids_batch[idx]
+ frames = self.video_reader.get_batch(frame_ids).permute(0, 3, 1, 2).float() / 255.0
+ frames = self.transform(frames).permute(1, 0, 2, 3)
+ return frames
+
+
+def tensor_to_uint8(tensor):
+ tensor = torch.clamp(tensor, -1.0, 1.0)
+ tensor = (tensor + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ tensor = (tensor.cpu().numpy() * 255).astype(np.uint8)
+ return tensor
+
+
+def main():
+ def str2bool(v):
+ if isinstance(v, bool):
+ return v
+ if v.lower() in ("yes", "true", "t", "y", "1"):
+ return True
+ elif v.lower() in ("no", "false", "f", "n", "0"):
+ return False
+ else:
+ raise argparse.ArgumentTypeError("Boolean value expected.")
+
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=42,
+ help="the seed (for reproducible sampling)",
+ )
+ parser.add_argument(
+ "--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="full"
+ )
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="configs/vidtok_kl_causal_488_4chn.yaml",
+ help="path to config which constructs model",
+ )
+ parser.add_argument(
+ "--ckpt",
+ type=str,
+ default="checkpoints/vidtok_kl_causal_488_4chn.ckpt",
+ help="path to checkpoint of model",
+ )
+ parser.add_argument(
+ "--output_video_dir",
+ type=str,
+ default="tmp",
+ help="path to save the outputs",
+ )
+ parser.add_argument(
+ "--input_video_path",
+ type=str,
+ default="assets/example.mp4",
+ help="path to the input video",
+ )
+ parser.add_argument(
+ "--input_height",
+ type=int,
+ default=256,
+ help="height of the input video",
+ )
+ parser.add_argument(
+ "--input_width",
+ type=int,
+ default=256,
+ help="width of the input video",
+ )
+ parser.add_argument(
+ "--sample_fps",
+ type=int,
+ default=30,
+ help="sample fps",
+ )
+ parser.add_argument(
+ "--chunk_size",
+ type=int,
+ default=16,
+ help="the size of a chunk - we split a long video into several chunks",
+ )
+ parser.add_argument(
+ "--read_long_video",
+ action='store_true'
+ )
+ parser.add_argument(
+ "--pad_gen_frames",
+ action="store_true",
+ help="Used only in causal mode. If True, pad frames generated in the last batch, else replicate the first frame instead",
+ )
+ parser.add_argument(
+ "--concate_input",
+ type=str2bool,
+ const=True,
+ default=True,
+ nargs="?",
+ help="",
+ )
+
+ args = parser.parse_args()
+ seed_everything(args.seed)
+
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Evaluating model {args.ckpt}")
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ precision_scope = autocast if args.precision == "autocast" else nullcontext
+ config = OmegaConf.load(args.config)
+
+ os.makedirs(args.output_video_dir, exist_ok=True)
+
+ model = load_model_from_config(args.config, args.ckpt)
+ model.to(device).eval()
+ assert args.chunk_size % model.encoder.time_downsample_factor == 0
+
+ if args.read_long_video:
+ assert hasattr(model, 'use_tiling'), "Tiling inference is needed to conduct long video reconstruction."
+ print(f"Using tiling inference to save memory usage...")
+ model.use_tiling = True
+ model.t_chunk_enc = args.chunk_size
+ model.t_chunk_dec = model.t_chunk_enc // model.encoder.time_downsample_factor
+ model.use_overlap = True
+
+ dataset = SingleVideoDataset(
+ video_path=args.input_video_path,
+ input_height=args.input_height,
+ input_width=args.input_width,
+ sample_fps=args.sample_fps,
+ chunk_size=args.chunk_size,
+ is_causal=model.is_causal,
+ read_long_video=args.read_long_video
+ )
+ dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)
+
+ inputs = []
+ outputs = []
+ with torch.no_grad(), precision_scope("cuda"):
+ tic = time.time()
+ for i, input in tqdm(enumerate(dataloader)):
+ input = input.to(device)
+
+ if model.is_causal and not args.read_long_video and args.pad_gen_frames:
+ if i == 0:
+ _, xrec, _ = model(input)
+ else:
+ _, xrec, _ = model(torch.cat([last_gen_frames, input], dim=2))
+ xrec = xrec[:, :, -input.shape[2]:].clamp(-1, 1)
+ last_gen_frames = xrec[:, :, (1 - model.encoder.time_downsample_factor):, :, :]
+ else:
+ _, xrec, _ = model(input)
+
+ input = rearrange(input, "b c t h w -> (b t) c h w")
+ inputs.append(input)
+ xrec = rearrange(xrec.clamp(-1, 1), "b c t h w -> (b t) c h w")
+ outputs.append(xrec)
+
+ toc = time.time()
+
+ # save the outputs as videos
+ inputs = tensor_to_uint8(torch.cat(inputs, dim=0))
+ inputs = rearrange(inputs, "t c h w -> t h w c")
+ outputs = tensor_to_uint8(torch.cat(outputs, dim=0))
+ outputs = rearrange(outputs, "t c h w -> t h w c")
+ min_len = min(inputs.shape[0], outputs.shape[0])
+ final = np.concatenate([inputs[:min_len], outputs[:min_len]], axis=2) if args.concate_input else outputs[:min_len]
+
+ output_video_path = os.path.join(args.output_video_dir, f"{Path(args.input_video_path).stem}_reconstructed.mp4")
+ write_video(output_video_path, final, args.sample_fps)
+
+ print0(f"[bold red]Results saved in: {output_video_path}[/bold red]")
+ print0(f"[bold red]\[scripts.inference_reconstruct][/bold red] Time taken: {toc - tic:.2f}s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc b/Meissonic/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a60105308bbef7ac0e229301aaf664728e0dba28
Binary files /dev/null and b/Meissonic/vidtok_cache/VidTok/vidtok/data/__pycache__/vidtok.cpython-310.pyc differ
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/data/datamodule.py b/Meissonic/vidtok_cache/VidTok/vidtok/data/datamodule.py
new file mode 100644
index 0000000000000000000000000000000000000000..c405b84703735b9a076b9ffb4e74de673470c15b
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/data/datamodule.py
@@ -0,0 +1,150 @@
+import numpy as np
+from functools import partial
+
+import torch
+import lightning.pytorch as pl
+from torch.utils.data import DataLoader, Dataset, IterableDataset
+
+from vidtok.modules.util import instantiate_from_config
+
+
+class WrappedDataset(Dataset):
+ """Wraps an arbitrary object with __len__ and __getitem__ into a pytorch dataset"""
+
+ def __init__(self, dataset):
+ self.data = dataset
+
+ def __len__(self):
+ return len(self.data)
+
+ def __getitem__(self, idx):
+ return self.data[idx]
+
+
+def worker_init_fn(_):
+ worker_info = torch.utils.data.get_worker_info()
+
+ dataset = worker_info.dataset
+ worker_id = worker_info.id
+
+ if isinstance(dataset, IterableDataset):
+ split_size = dataset.num_records // worker_info.num_workers
+ # reset num_records to the true number to retain reliable length information
+ dataset.sample_ids = dataset.valid_ids[
+ worker_id * split_size : (worker_id + 1) * split_size
+ ]
+ current_id = np.random.choice(len(np.random.get_state()[1]), 1)
+ return np.random.seed(np.random.get_state()[1][current_id] + worker_id)
+ else:
+ return np.random.seed(np.random.get_state()[1][0] + worker_id)
+
+
+class DataModuleFromConfig(pl.LightningDataModule):
+ def __init__(
+ self,
+ batch_size,
+ train=None,
+ validation=None,
+ test=None,
+ predict=None,
+ wrap=False,
+ num_workers=None,
+ pin_train_memory=True,
+ is_iterable_dataset=False,
+ shuffle_test_loader=False,
+ use_worker_init_fn=False,
+ shuffle_val_dataloader=False,
+ ):
+ super().__init__()
+ self.batch_size = batch_size
+ self.dataset_configs = dict()
+ self.num_workers = num_workers if num_workers is not None else batch_size * 2
+ self.pin_train_memory = pin_train_memory
+ self.is_iterable_dataset = is_iterable_dataset
+ self.use_worker_init_fn = use_worker_init_fn
+ if train is not None:
+ self.dataset_configs["train"] = train
+ self.train_dataloader = self._train_dataloader
+ if validation is not None:
+ self.dataset_configs["validation"] = validation
+ self.val_dataloader = partial(
+ self._val_dataloader, shuffle=shuffle_val_dataloader
+ )
+ if test is not None:
+ self.dataset_configs["test"] = test
+ self.test_dataloader = partial(
+ self._test_dataloader, shuffle=shuffle_test_loader
+ )
+ if predict is not None:
+ self.dataset_configs["predict"] = predict
+ self.predict_dataloader = self._predict_dataloader
+ self.wrap = wrap
+
+ def prepare_data(self):
+ for data_cfg in self.dataset_configs.values():
+ instantiate_from_config(data_cfg)
+
+ def setup(self, stage=None):
+ self.datasets = dict(
+ (k, instantiate_from_config(self.dataset_configs[k]))
+ for k in self.dataset_configs
+ )
+ if self.wrap:
+ for k in self.datasets:
+ self.datasets[k] = WrappedDataset(self.datasets[k])
+
+ def _train_dataloader(self):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["train"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ pin_memory=self.pin_train_memory,
+ shuffle=False if self.is_iterable_dataset else True,
+ worker_init_fn=init_fn,
+ )
+
+ def _val_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["validation"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _test_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+
+ # do not shuffle dataloader for iterable dataset
+ shuffle = shuffle and (not self.is_iterable_dataset)
+
+ return DataLoader(
+ self.datasets["test"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ shuffle=shuffle,
+ )
+
+ def _predict_dataloader(self, shuffle=False):
+ if self.is_iterable_dataset or self.use_worker_init_fn:
+ init_fn = worker_init_fn
+ else:
+ init_fn = None
+ return DataLoader(
+ self.datasets["predict"],
+ batch_size=self.batch_size,
+ num_workers=self.num_workers,
+ worker_init_fn=init_fn,
+ )
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/data/video_read.py b/Meissonic/vidtok_cache/VidTok/vidtok/data/video_read.py
new file mode 100644
index 0000000000000000000000000000000000000000..357cd48305141ee70924fd6a79adbe5cb7f6ca5b
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/data/video_read.py
@@ -0,0 +1,88 @@
+import os
+import random
+import decord
+import numpy as np
+import torch
+
+from vidtok.modules.util import print0
+
+decord.bridge.set_bridge("torch")
+
+
+def sample_frames_with_fps(
+ total_frames,
+ video_fps,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+):
+ """sample frames proportional to the length of the frames in one second
+ e.g., 1s video has 30 frames, when 'fps'=3, we sample frames with spacing of 30/3=10
+ return the frame indices
+
+ Parameters
+ ----------
+ total_frames : length of the video
+ video_fps : original fps of the video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frame indices
+ """
+ sample_num_frames = min(sample_num_frames, total_frames)
+ interval = round(video_fps / sample_fps)
+ frames_range = (sample_num_frames - 1) * interval + 1
+
+ if start_index is not None:
+ start = start_index
+ elif total_frames - frames_range - 1 < 0:
+ start = 0
+ else:
+ start = random.randint(0, total_frames - frames_range - 1)
+
+ frame_idxs = np.linspace(
+ start=start, stop=min(total_frames - 1, start + frames_range), num=sample_num_frames
+ ).astype(int)
+
+ return frame_idxs
+
+
+def read_frames_with_decord(
+ video_path,
+ sample_num_frames,
+ sample_fps,
+ start_index=None
+) -> tuple[torch.Tensor, list[int]]:
+ """read frames from video path using decord
+
+ Parameters
+ ----------
+ video_path : path to video
+ sample_num_frames : number of frames to sample
+ sample_fps : the fps to sample frames
+ start_index : the starting frame index. If it is not None, it will be used as the starting frame index
+
+ Returns
+ -------
+ frames (tensor 0~1), frame indices
+ """
+ video_reader = decord.VideoReader(video_path, num_threads=0)
+ total_frames = len(video_reader)
+ video_fps = video_reader.get_avg_fps() # note that the fps here is float.
+ frame_idxs = sample_frames_with_fps(
+ total_frames=total_frames,
+ video_fps=video_fps,
+ sample_num_frames=sample_num_frames,
+ sample_fps=sample_fps,
+ start_index=start_index
+ )
+ frames = video_reader.get_batch(frame_idxs)
+ frames = frames.float() / 255
+ frames = frames.permute(0, 3, 1, 2)
+ if (frames.shape[0] != sample_num_frames) or (len(frame_idxs) != sample_num_frames):
+ print0(f"[bold yellow]\[vidtok.data.video_read][read_frames_with_decord][/bold yellow] Warning: need {sample_num_frames} frames, "
+ f"but got {frames.shape[0]} frames, {len(frame_idxs)} frame indices, video_path={video_path}.")
+ return frames, frame_idxs
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/data/vidtok.py b/Meissonic/vidtok_cache/VidTok/vidtok/data/vidtok.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b1898d4718de1450da96f25e9134dd4e1084cc1
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/data/vidtok.py
@@ -0,0 +1,333 @@
+import os
+import glob
+from typing import Union
+
+import decord
+import numpy as np
+import pandas as pd
+import torch
+from PIL import Image
+from torch.utils.data import Dataset
+from torchvision.transforms import v2
+from tqdm import trange
+
+from vidtok.modules.util import print0
+from .video_read import read_frames_with_decord
+
+
+class VidTokDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ meta_path: str,
+ video_params: dict,
+ data_frac: float = 1.0,
+ is_strict_loading: bool = False,
+ skip_missing_files: bool = True,
+ start_index: Union[None, int] = None
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use data dir: {self.data_dir}")
+
+ self.meta_path = meta_path
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Use meta path: {self.meta_path}")
+
+ self.video_params = video_params
+
+ self.data_frac = data_frac
+ self.is_strict_loading = is_strict_loading
+ self.skip_missing_files = skip_missing_files
+ self.start_index = start_index
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",)
+
+ if self.data_frac < 1:
+ metadata = metadata.sample(frac=self.data_frac)
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk
+ """
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.metadata)
+
+ def __getitem__(self, item):
+ item = item % len(self.metadata)
+ sample = self.metadata.iloc[item]
+ video_fp, _ = self._get_video_path(sample)
+
+ try:
+ if os.path.isfile(video_fp):
+ imgs, idxs = read_frames_with_decord(
+ video_path=video_fp,
+ sample_num_frames=self.video_params["sample_num_frames"],
+ sample_fps=self.video_params["sample_fps"],
+ start_index=self.start_index
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(f"[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video.")
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}.") from e
+ else:
+ print0("[bold yellow]\[vidtok.data.vidtok][VidTokDataset][/bold yellow] Warning: using the pure black image as the frame sample")
+ imgs = Image.new("RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0))
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ # imgs: (T, C, H, W)
+ imgs = self.transforms(imgs)
+
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ imgs = torch.cat([imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)], dim=0)
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ 'jpg': imgs,
+ "path": video_fp
+ }
+
+
+class VidTokValDataset(Dataset):
+ def __init__(
+ self,
+ data_dir: str,
+ video_params: dict,
+ meta_path: Union[None, str] = None,
+ pre_load_frames: bool = True,
+ is_strict_loading: bool = True,
+ last_frames_handle: str = "repeat", # 'repeat', 'drop'
+ skip_missing_files: bool = False,
+ read_long_video: bool = False,
+ chunk_size: int = 16,
+ is_causal: bool = True,
+ ):
+ super().__init__()
+
+ self.data_dir = data_dir
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use data dir: {self.data_dir}"
+ )
+
+ self.meta_path = meta_path
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Use meta path: {self.meta_path}"
+ )
+
+ self.video_params = video_params
+ self.read_long_video = read_long_video
+ self.chunk_size = chunk_size
+ self.is_causal = is_causal
+
+ self.is_strict_loading = is_strict_loading
+ self.last_frames_handle = last_frames_handle
+ self.skip_missing_files = skip_missing_files
+ self.transforms = self._get_transforms(
+ video_params["input_height"],
+ video_params["input_width"],
+ )
+
+ self.missing_files = []
+ self._load_metadata()
+ self._load_every_frame_from_meta()
+
+ if pre_load_frames:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Pre-loading all frames into CPU..."
+ )
+ self._pre_load_frames()
+
+ def _get_transforms(self, input_height, input_width, norm_mean=[0.5, 0.5, 0.5], norm_std=[0.5, 0.5, 0.5]):
+ normalize = v2.Normalize(mean=norm_mean, std=norm_std)
+ return v2.Compose(
+ [
+ v2.Resize(input_height, antialias=True),
+ v2.CenterCrop((input_height, input_width)),
+ normalize,
+ ]
+ )
+
+ def _load_metadata(self):
+ if self.meta_path is not None:
+ metadata = pd.read_csv(
+ self.meta_path,
+ on_bad_lines="skip",
+ encoding="ISO-8859-1",
+ engine="python",
+ sep=",",
+ )
+ self.metadata = metadata
+ self.metadata.dropna(inplace=True)
+ else:
+ self.metadata = glob.glob(os.path.join(self.data_dir, '**', '*.mp4'), recursive=True)
+
+ def _load_every_frame_from_meta(self):
+ decord.bridge.set_bridge("torch")
+ self.frames_batch = []
+ for video_idx in range(len(self.metadata)):
+ try:
+ sample = self.metadata.iloc[video_idx]
+ video_fp, _ = self._get_video_path(sample)
+ except:
+ video_fp = self.metadata[video_idx]
+ if os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ total_frames = len(video_reader)
+ fps = video_reader.get_avg_fps() # float
+ interval = round(fps / self.video_params["sample_fps"])
+ frame_ids = list(range(0, total_frames, interval))
+
+ if self.read_long_video:
+ video_length = len(frame_ids)
+ if self.is_causal and video_length > self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * ((video_length - 1) // self.chunk_size) + 1]
+ elif not self.is_causal and video_length >= self.chunk_size:
+ num_frames_ids = frame_ids[:self.chunk_size * (video_length // self.chunk_size)]
+ else:
+ continue
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ else:
+ for x in range(0, len(frame_ids), self.video_params["sample_num_frames"]):
+ num_frames_ids = frame_ids[x : x + self.video_params["sample_num_frames"]]
+ if len(num_frames_ids) < self.video_params["sample_num_frames"]:
+ if self.last_frames_handle == "repeat":
+ num_frames_ids += [num_frames_ids[-1]] * (
+ self.video_params["sample_num_frames"] - len(num_frames_ids)
+ )
+ elif self.last_frames_handle == "drop":
+ continue
+ else:
+ raise ValueError(f"Invalid last_frames_handle: {self.last_frames_handle}")
+ self.frames_batch.append(
+ {
+ "video_fp": video_fp,
+ "num_frames_ids": num_frames_ids,
+ }
+ )
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Loaded all frames index from {len(self.metadata)} videos."
+ )
+
+ def _pre_load_frames(self):
+ last_video_fp = None
+ for idx in trange(len(self.frames_batch), desc="Pre-loading all frames"):
+ if self.frames_batch[idx]["video_fp"] != last_video_fp:
+ video_reader = decord.VideoReader(self.frames_batch[idx]["video_fp"], num_threads=0)
+ last_video_fp = self.frames_batch[idx]["video_fp"]
+ self.frames_batch[idx]["frames"] = (
+ video_reader.get_batch(self.frames_batch[idx]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+
+ def _get_video_path(self, sample):
+ """reduce the access to the disk"""
+ rel_video_fp = str(sample["videos"])
+ abs_video_fp = os.path.join(self.data_dir, rel_video_fp)
+ return abs_video_fp, rel_video_fp
+
+ def __len__(self):
+ return len(self.frames_batch)
+
+ def __getitem__(self, item):
+ video_fp = self.frames_batch[item]["video_fp"]
+
+ try:
+ if "frames" in self.frames_batch[item]:
+ imgs = self.frames_batch[item]["frames"]
+ elif os.path.isfile(video_fp):
+ video_reader = decord.VideoReader(video_fp, num_threads=0)
+ imgs = (
+ video_reader.get_batch(self.frames_batch[item]["num_frames_ids"]).permute(0, 3, 1, 2).float()
+ / 255.0
+ )
+ else:
+ # if the video file is missing
+ if video_fp not in self.missing_files:
+ self.missing_files.append(video_fp)
+ # resample another video or not
+ if self.skip_missing_files:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: missing video file {video_fp}. Resampling another video."
+ )
+ return self.__getitem__(np.random.choice(self.__len__()))
+ else:
+ raise ValueError(f"Video file {video_fp} is missing, skip_missing_files={self.skip_missing_files}.")
+ except Exception as e:
+ # if the video exists, but loading failed
+ if self.is_strict_loading:
+ raise ValueError(
+ f"Video loading failed for {video_fp}, is_strict_loading={self.is_strict_loading}."
+ ) from e
+ else:
+ print0(
+ "[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: using the pure black image as the frame sample"
+ )
+ imgs = Image.new(
+ "RGB", (self.video_params["input_width"], self.video_params["input_height"]), (0, 0, 0)
+ )
+ imgs = v2.ToTensor()(imgs).unsqueeze(0)
+
+ if self.transforms is not None:
+ imgs = self.transforms(imgs)
+
+ if not self.read_long_video:
+ if imgs.shape[0] < self.video_params["sample_num_frames"]:
+ print0(
+ f"[bold yellow]\[vidtok.data.vidtok][VidTokValDataset][/bold yellow] Warning: video {video_fp} has less frames {imgs.shape[0]} than sample_num_frames {self.video_params['sample_num_frames']}."
+ )
+ imgs = torch.cat(
+ [imgs, imgs[-1].unsqueeze(0).repeat(self.video_params["sample_num_frames"] - imgs.shape[0], 1, 1, 1)],
+ dim=0,
+ )
+
+ imgs = imgs.permute(1, 0, 2, 3) # (C, T, H, W)
+
+ return {
+ "jpg": imgs,
+ "path": video_fp,
+ }
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder.py b/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..96da5e6c74621b4f82538453e850567245adbbb0
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder.py
@@ -0,0 +1,517 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+ self.use_tiling = False
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ token_indices = rearrange(token_indices, "... -> ... 1")
+ token_indices, ps = pack_one(token_indices, "b * d")
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = rearrange(codes, "b d n c -> b n (c d)")
+ z = self.regularization.project_out(codes)
+ z = unpack_one(z, ps, "b * d")
+ z = rearrange(z, "b ... d -> b d ...")
+ return z
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ z = self.indices_to_latent(z)
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = [[0, num_frames]]
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ tile = z[
+ :,
+ :,
+ start_frame : end_frame,
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+
+ dec = self.decode(z)
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py b/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c1182f573eee05b634daeb9a065b7adaacc600b
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/models/autoencoder_v1_1.py
@@ -0,0 +1,588 @@
+import re
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Any, Dict, Tuple, Union, Optional, List
+from omegaconf import ListConfig
+from packaging import version
+
+import torch
+import lightning.pytorch as pl
+
+from safetensors.torch import load_file as load_safetensors
+from vidtok.modules.ema import LitEma
+from vidtok.modules.util import (default, get_obj_from_str,
+ instantiate_from_config, print0)
+from vidtok.modules.regularizers import pack_one, unpack_one, rearrange
+
+
+class AbstractAutoencoder(pl.LightningModule):
+ """
+ This is the base class for all autoencoders
+ """
+
+ def __init__(
+ self,
+ ema_decay: Union[None, float] = None,
+ monitor: Union[None, str] = None,
+ mode: Union[None, str] = None,
+ input_key: str = "jpg",
+ ):
+ super().__init__()
+
+ self.input_key = input_key
+ self.use_ema = ema_decay is not None
+ self.ema_decay = ema_decay
+ if monitor is not None:
+ self.monitor = monitor
+ if mode is not None:
+ self.mode = mode
+
+ if version.parse(torch.__version__) >= version.parse("2.0.0"):
+ self.automatic_optimization = False
+
+ @abstractmethod
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_input(self, batch) -> Any:
+ raise NotImplementedError()
+
+ def on_train_batch_end(self, *args, **kwargs):
+ # for EMA computation
+ if self.use_ema:
+ self.model_ema(self)
+
+ @contextmanager
+ def ema_scope(self, context=None):
+ if self.use_ema:
+ self.model_ema.store(self.parameters())
+ self.model_ema.copy_to(self)
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Switched to EMA weights"
+ )
+ try:
+ yield None
+ finally:
+ if self.use_ema:
+ self.model_ema.restore(self.parameters())
+ if context is not None:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] {context}: Restored training weights"
+ )
+
+ @abstractmethod
+ def encode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] encode()-method of abstract base class called"
+ )
+
+ @abstractmethod
+ def decode(self, *args, **kwargs) -> torch.Tensor:
+ raise NotImplementedError(
+ "[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] decode()-method of abstract base class called"
+ )
+
+ def instantiate_optimizer_from_config(self, params, lr, cfg):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AbstractAutoencoder][/bold magenta] loading >>> {cfg['target']} <<< optimizer from config"
+ )
+ return get_obj_from_str(cfg["target"])(params, lr=lr, **cfg.get("params", dict()))
+
+ @abstractmethod
+ def configure_optimizers(self) -> Any:
+ raise NotImplementedError()
+
+
+class AutoencodingEngine(AbstractAutoencoder):
+ """
+ Base class for all video tokenizers that we train
+ """
+
+ def __init__(
+ self,
+ *args,
+ encoder_config: Dict,
+ decoder_config: Dict,
+ loss_config: Dict,
+ regularizer_config: Dict,
+ optimizer_config: Union[Dict, None] = None,
+ lr_g_factor: float = 1.0,
+ compile_model: bool = False,
+ use_tiling: bool = False,
+ **kwargs,
+ ):
+ ckpt_path = kwargs.pop("ckpt_path", None)
+ ignore_keys = kwargs.pop("ignore_keys", ())
+ verbose = kwargs.pop("verbose", True)
+ self.use_tiling = kwargs.pop("use_tiling", False)
+ self.t_chunk_enc = kwargs.pop("t_chunk_enc", 16)
+ super().__init__(*args, **kwargs)
+
+ compile = (
+ torch.compile
+ if (version.parse(torch.__version__) >= version.parse("2.0.0")) and compile_model
+ else lambda x: x
+ )
+
+ self.encoder = compile(instantiate_from_config(encoder_config))
+ self.decoder = compile(instantiate_from_config(decoder_config))
+ self.loss = instantiate_from_config(loss_config)
+ self.regularization = instantiate_from_config(regularizer_config)
+ self.optimizer_config = default(optimizer_config, {"target": "torch.optim.Adam"})
+ self.lr_g_factor = lr_g_factor
+
+ self.t_chunk_dec = self.t_chunk_enc // self.encoder.time_downsample_factor
+ self.use_overlap = False
+ self.is_causal = self.encoder.is_causal
+
+ self.temporal_compression_ratio = 2 ** len(self.encoder.tempo_ds)
+
+ self.use_tiling = use_tiling
+ # Decode more latent frames at once
+ self.num_sample_frames_batch_size = 16
+ self.num_latent_frames_batch_size = self.num_sample_frames_batch_size // self.temporal_compression_ratio
+
+ # We make the minimum height and width of sample for tiling half that of the generally supported
+ self.tile_sample_min_height = 256
+ self.tile_sample_min_width = 256
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = 0 # 1 / 8
+ self.tile_overlap_factor_width = 0 # 1 / 8
+
+ if self.use_ema:
+ self.model_ema = LitEma(self, decay=self.ema_decay)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Keeping EMAs of {len(list(self.model_ema.buffers()))}."
+ )
+
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Use ckpt_path: {ckpt_path}"
+ )
+ if ckpt_path is not None:
+ self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, verbose=verbose)
+
+ def init_from_ckpt(self, path: str, ignore_keys: Union[Tuple, list, ListConfig] = tuple(), verbose: bool = True) -> None:
+ if path.endswith("ckpt"):
+ ckpt = torch.load(path, map_location="cpu")
+ weights = ckpt["state_dict"] if "state_dict" in ckpt else ckpt
+ elif path.endswith("safetensors"):
+ weights = load_safetensors(path)
+ else:
+ raise NotImplementedError(f"Unknown checkpoint: {path}")
+
+ keys = list(weights.keys())
+ for k in keys:
+ for ik in ignore_keys:
+ if re.match(ik, k):
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Deleting key {k} from state_dict."
+ )
+ del weights[k]
+
+ missing, unexpected = self.load_state_dict(weights, strict=False)
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys"
+ )
+ if verbose:
+ if len(missing) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Missing Keys: {missing}"
+ )
+ if len(unexpected) > 0:
+ print0(
+ f"[bold magenta]\[vidtok.models.autoencoder][AutoencodingEngine][/bold magenta] Unexpected Keys: {unexpected}"
+ )
+
+ def get_input(self, batch: Dict) -> torch.Tensor:
+ return batch[self.input_key]
+
+ def get_autoencoder_params(self) -> list:
+ params = (
+ list(filter(lambda p: p.requires_grad, self.encoder.parameters()))
+ + list(filter(lambda p: p.requires_grad, self.decoder.parameters()))
+ + list(self.regularization.get_trainable_parameters())
+ + list(self.loss.get_trainable_autoencoder_parameters())
+ )
+ return params
+
+ def get_discriminator_params(self) -> list:
+ params = list(self.loss.get_trainable_parameters())
+ return params
+
+ def get_last_layer(self):
+ return self.decoder.get_last_layer()
+
+ def _empty_causal_cached(self, parent):
+ for name, module in parent.named_modules():
+ if hasattr(module, 'causal_cache'):
+ module.causal_cache = None
+
+ def _set_first_chunk(self, is_first_chunk=True):
+ for module in self.modules():
+ if hasattr(module, 'is_first_chunk'):
+ module.is_first_chunk = is_first_chunk
+
+ def _set_cache_offset(self, modules, cache_offset=0):
+ for module in modules:
+ for submodule in module.modules():
+ if hasattr(submodule, 'cache_offset'):
+ submodule.cache_offset = cache_offset
+
+ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[3], b.shape[3], blend_extent)
+ for y in range(blend_extent):
+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
+ y / blend_extent
+ )
+ return b
+
+ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor:
+ blend_extent = min(a.shape[4], b.shape[4], blend_extent)
+ for x in range(blend_extent):
+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
+ x / blend_extent
+ )
+ return b
+
+ def build_chunk_start_end(self, t, decoder_mode=False):
+ start_end = [[0, 1]]
+ start = 1
+ end = start
+ while True:
+ if start >= t:
+ break
+ end = min(t, end + (self.t_chunk_dec if decoder_mode else self.t_chunk_enc))
+ start_end.append([start, end])
+ start = end
+ return start_end
+
+ def enable_tiling(
+ self,
+ tile_sample_min_height: Optional[int] = None,
+ tile_sample_min_width: Optional[int] = None,
+ tile_overlap_factor_height: Optional[float] = None,
+ tile_overlap_factor_width: Optional[float] = None,
+ ) -> None:
+ self.use_tiling = True
+ self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height
+ self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width
+ self.tile_latent_min_height = int(self.tile_sample_min_height / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** len(self.encoder.spatial_ds)))
+ self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height
+ self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width
+
+ def disable_tiling(self) -> None:
+ self.use_tiling = False
+
+ def encode(self, x: Any, return_reg_log: bool = False) -> Any:
+ self._empty_causal_cached(self.encoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ z = self.tile_encode(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+ else:
+ z = self.encoder(x)
+ z, reg_log = self.regularization(z, n_steps=self.global_step // 2)
+
+ if return_reg_log:
+ return z, reg_log
+ return z
+
+ def tile_encode(self, x: Any) -> Any:
+
+ num_frames, height, width = x.shape[-3:]
+
+ overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_latent_min_height - blend_extent_height
+ row_limit_width = self.tile_latent_min_width - blend_extent_width
+ rows = []
+
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ start_end = self.build_chunk_start_end(num_frames)
+ result_z = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = x[
+ :,
+ :,
+ start_frame:end_frame,
+ i : i + self.tile_sample_min_height,
+ j : j + self.tile_sample_min_width,
+ ]
+ tile = self.encoder(tile)
+ result_z.append(tile)
+ row.append(torch.cat(result_z, dim=2))
+ rows.append(row)
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+ enc = torch.cat(result_rows, dim=3)
+
+ return enc
+
+ def indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ assert token_indices.dim() == 4, "token_indices should be of shape (b, t, h, w)"
+ b, t, h, w = token_indices.shape
+ token_indices = token_indices.unsqueeze(-1).reshape(b, -1, 1)
+ codes = self.regularization.indices_to_codes(token_indices)
+ codes = codes.permute(0, 2, 3, 1).reshape(b, codes.shape[2], -1)
+ z = self.regularization.project_out(codes)
+ return z.reshape(b, t, h, w, -1).permute(0, 4, 1, 2, 3)
+
+ def tile_indices_to_latent(self, token_indices: torch.Tensor) -> torch.Tensor:
+ num_frames = token_indices.shape[1]
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ result_z = []
+ for (start, end) in start_end:
+ chunk = token_indices[:, start:end, :, :]
+ chunk_z = self.indices_to_latent(chunk)
+ result_z.append(chunk_z.clone())
+ return torch.cat(result_z, dim=2)
+
+ def decode(self, z: Any, decode_from_indices: bool = False) -> torch.Tensor:
+ if decode_from_indices:
+ if self.use_tiling:
+ z = self.tile_indices_to_latent(z)
+ else:
+ z = self.indices_to_latent(z)
+ self._empty_causal_cached(self.decoder)
+ self._set_first_chunk(True)
+
+ if self.use_tiling:
+ x = self.tile_decode(z)
+ else:
+ x = self.decoder(z)
+ return x
+
+
+ def tile_decode(self, z: Any) -> torch.Tensor:
+
+ num_frames, height, width = z.shape[-3:]
+
+ overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height))
+ overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width))
+ blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height)
+ blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width)
+ row_limit_height = self.tile_sample_min_height - blend_extent_height
+ row_limit_width = self.tile_sample_min_width - blend_extent_width
+
+ # Split z into overlapping tiles and decode them separately.
+ # The tiles have an overlap to avoid seams between tiles.
+ rows = []
+ for i in range(0, height, overlap_height):
+ row = []
+ for j in range(0, width, overlap_width):
+ if self.is_causal:
+ assert self.encoder.time_downsample_factor in [2, 4, 8], "Only support 2x, 4x or 8x temporal downsampling now."
+ if self.encoder.time_downsample_factor == 4:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 4)
+ elif self.encoder.time_downsample_factor == 2:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1], self.decoder.up_temporal[0], self.decoder.conv_out], 2)
+ else:
+ self._set_cache_offset([self.decoder], 1)
+ self._set_cache_offset([self.decoder.up_temporal[3].upsample, self.decoder.up_temporal[2]], 2)
+ self._set_cache_offset([self.decoder.up_temporal[2].upsample, self.decoder.up_temporal[1]], 4)
+ self._set_cache_offset([self.decoder.up_temporal[1].upsample, self.decoder.up_temporal[0], self.decoder.conv_out], 8)
+
+ start_end = self.build_chunk_start_end(num_frames, decoder_mode=True)
+ time = []
+ for idx, (start_frame, end_frame) in enumerate(start_end):
+ self._set_first_chunk(idx == 0)
+ tile = z[
+ :,
+ :,
+ start_frame : (end_frame + 1 if self.is_causal and end_frame + 1 <= num_frames else end_frame),
+ i : i + self.tile_latent_min_height,
+ j : j + self.tile_latent_min_width,
+ ]
+ tile = self.decoder(tile)
+ if self.is_causal and end_frame + 1 <= num_frames:
+ tile = tile[:, :, : -self.encoder.time_downsample_factor]
+ time.append(tile)
+ row.append(torch.cat(time, dim=2))
+ rows.append(row)
+
+ result_rows = []
+ for i, row in enumerate(rows):
+ result_row = []
+ for j, tile in enumerate(row):
+ # blend the above tile and the left tile
+ # to the current tile and add the current tile to the result row
+ if i > 0:
+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height)
+ if j > 0:
+ tile = self.blend_h(row[j - 1], tile, blend_extent_width)
+ result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width])
+ result_rows.append(torch.cat(result_row, dim=4))
+
+ dec = torch.cat(result_rows, dim=3)
+ return dec
+
+ def forward(self, x: Any) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ if self.encoder.fix_encoder:
+ with torch.no_grad():
+ z, reg_log = self.encode(x, return_reg_log=True)
+ else:
+ z, reg_log = self.encode(x, return_reg_log=True)
+ dec = self.decode(z)
+ if dec.shape[2] != x.shape[2]:
+ dec = dec[:, :, -x.shape[2]:, ...]
+ return z, dec, reg_log
+
+ def training_step(self, batch, batch_idx) -> Any:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ opt_g, opt_d = self.optimizers()
+
+ # autoencode loss
+ self.toggle_optimizer(opt_g)
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_g.zero_grad()
+ self.manual_backward(aeloss)
+
+ # gradient clip
+ torch.nn.utils.clip_grad_norm_(self.get_autoencoder_params(), 20.0)
+ opt_g.step()
+ self.untoggle_optimizer(opt_g)
+
+ # discriminator loss
+ self.toggle_optimizer(opt_d)
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="train",
+ )
+ opt_d.zero_grad()
+ self.manual_backward(discloss)
+ torch.nn.utils.clip_grad_norm_(self.get_discriminator_params(), 20.0)
+ opt_d.step()
+ self.untoggle_optimizer(opt_d)
+
+ # logging
+ log_dict = {
+ "train/aeloss": aeloss,
+ "train/discloss": discloss,
+ }
+ log_dict.update(log_dict_ae)
+ log_dict.update(log_dict_disc)
+
+ self.log_dict(log_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True)
+ lr = opt_g.param_groups[0]["lr"]
+ self.log(
+ "lr_abs",
+ lr,
+ prog_bar=True,
+ logger=True,
+ on_step=True,
+ on_epoch=False,
+ sync_dist=True,
+ )
+
+ def validation_step(self, batch, batch_idx) -> Dict:
+ log_dict = self._validation_step(batch, batch_idx)
+ with self.ema_scope():
+ log_dict_ema = self._validation_step(batch, batch_idx, postfix="_ema")
+ log_dict.update(log_dict_ema)
+ return log_dict
+
+ def _validation_step(self, batch, batch_idx, postfix="") -> Dict:
+ x = self.get_input(batch)
+
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+
+ z, xrec, regularization_log = self(x)
+
+ if x.ndim == 5 and xrec.ndim == 4:
+ xrec = xrec.unsqueeze(2)
+
+ aeloss, log_dict_ae = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 0,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ discloss, log_dict_disc = self.loss(
+ regularization_log,
+ x,
+ xrec,
+ 1,
+ self.global_step,
+ last_layer=self.get_last_layer(),
+ split="val" + postfix,
+ )
+
+ self.log(f"val{postfix}/rec_loss", log_dict_ae[f"val{postfix}/rec_loss"])
+ log_dict_ae.update(log_dict_disc)
+ self.log_dict(log_dict_ae)
+ return log_dict_ae
+
+ def configure_optimizers(self) -> Any:
+ ae_params = self.get_autoencoder_params()
+ disc_params = self.get_discriminator_params()
+
+ opt_ae = self.instantiate_optimizer_from_config(
+ ae_params,
+ default(self.lr_g_factor, 1.0) * self.learning_rate,
+ self.optimizer_config,
+ )
+ opt_disc = self.instantiate_optimizer_from_config(disc_params, self.learning_rate, self.optimizer_config)
+
+ return [opt_ae, opt_disc], []
+
+ @torch.no_grad()
+ def log_images(self, batch: Dict) -> Dict:
+ log = dict()
+ x = self.get_input(batch)
+ _, xrec, _ = self(x)
+ log["inputs"] = x
+ log["recs"] = xrec
+ with self.ema_scope():
+ _, xrec_ema, _ = self(x)
+ log["recs_ema"] = xrec_ema
+ return log
\ No newline at end of file
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc b/Meissonic/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..33f2b721c289174afb26321194a97156b77c328b
Binary files /dev/null and b/Meissonic/vidtok_cache/VidTok/vidtok/modules/__pycache__/util.cpython-310.pyc differ
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/discriminator.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/discriminator.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9d94b21b22f5019f3cdcc4cbf2e98bd0ce0ee02
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/discriminator.py
@@ -0,0 +1,201 @@
+import functools
+
+import torch
+import torch.nn as nn
+
+
+def weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ nn.init.normal_(m.weight.data, 0.0, 0.02)
+ elif classname.find("BatchNorm") != -1:
+ nn.init.normal_(m.weight.data, 1.0, 0.02)
+ nn.init.constant_(m.bias.data, 0)
+
+
+class ActNorm(nn.Module):
+ def __init__(self, num_features, logdet=False, affine=True, allow_reverse_init=False):
+ assert affine
+ super().__init__()
+ self.logdet = logdet
+ self.loc = nn.Parameter(torch.zeros(1, num_features, 1, 1))
+ self.scale = nn.Parameter(torch.ones(1, num_features, 1, 1))
+ self.allow_reverse_init = allow_reverse_init
+
+ self.register_buffer("initialized", torch.tensor(0, dtype=torch.uint8))
+
+ def initialize(self, input):
+ with torch.no_grad():
+ flatten = input.permute(1, 0, 2, 3).contiguous().view(input.shape[1], -1)
+ mean = flatten.mean(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+ std = flatten.std(1).unsqueeze(1).unsqueeze(2).unsqueeze(3).permute(1, 0, 2, 3)
+
+ self.loc.data.copy_(-mean)
+ self.scale.data.copy_(1 / (std + 1e-6))
+
+ def forward(self, input, reverse=False):
+ if reverse:
+ return self.reverse(input)
+ if len(input.shape) == 2:
+ input = input[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ _, _, height, width = input.shape
+
+ if self.training and self.initialized.item() == 0:
+ self.initialize(input)
+ self.initialized.fill_(1)
+
+ h = self.scale * (input + self.loc)
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+
+ if self.logdet:
+ log_abs = torch.log(torch.abs(self.scale))
+ logdet = height * width * torch.sum(log_abs)
+ logdet = logdet * torch.ones(input.shape[0]).to(input)
+ return h, logdet
+
+ return h
+
+ def reverse(self, output):
+ if self.training and self.initialized.item() == 0:
+ if not self.allow_reverse_init:
+ raise RuntimeError(
+ "Initializing ActNorm in reverse direction is "
+ "disabled by default. Use allow_reverse_init=True to enable."
+ )
+ else:
+ self.initialize(output)
+ self.initialized.fill_(1)
+
+ if len(output.shape) == 2:
+ output = output[:, :, None, None]
+ squeeze = True
+ else:
+ squeeze = False
+
+ h = output / self.scale - self.loc
+
+ if squeeze:
+ h = h.squeeze(-1).squeeze(-1)
+ return h
+
+
+class NLayerDiscriminator(nn.Module):
+ """Defines a PatchGAN discriminator as in Pix2Pix."""
+ # https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py
+ def __init__(self, input_nc=3, ndf=64, n_layers=3, use_actnorm=False):
+ """Construct a PatchGAN discriminator
+ Parameters:
+ input_nc (int) -- the number of channels in input images
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ """
+ super(NLayerDiscriminator, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm2d
+ else:
+ norm_layer = ActNorm
+ if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters
+ use_bias = norm_layer.func != nn.BatchNorm2d
+ else:
+ use_bias = norm_layer != nn.BatchNorm2d
+
+ kw = 4
+ padw = 1
+ sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
+
+
+class NLayerDiscriminator3D(nn.Module):
+ """Defines a 3D PatchGAN discriminator as in Pix2Pix but for 3D inputs."""
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/losses/discriminator.py
+ def __init__(self, input_nc=1, ndf=64, n_layers=3, use_actnorm=False):
+ """
+ Construct a 3D PatchGAN discriminator
+
+ Parameters:
+ input_nc (int) -- the number of channels in input volumes
+ ndf (int) -- the number of filters in the last conv layer
+ n_layers (int) -- the number of conv layers in the discriminator
+ use_actnorm (bool) -- flag to use actnorm instead of batchnorm
+ """
+ super(NLayerDiscriminator3D, self).__init__()
+ if not use_actnorm:
+ norm_layer = nn.BatchNorm3d
+ else:
+ raise NotImplementedError("Not implemented.")
+ if type(norm_layer) == functools.partial:
+ use_bias = norm_layer.func != nn.BatchNorm3d
+ else:
+ use_bias = norm_layer != nn.BatchNorm3d
+
+ kw = 3
+ padw = 1
+ sequence = [nn.Conv3d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]
+ nf_mult = 1
+ nf_mult_prev = 1
+ for n in range(1, n_layers): # gradually increase the number of filters
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev,
+ ndf * nf_mult,
+ kernel_size=(kw, kw, kw),
+ stride=(2 if n == 1 else 1, 2, 2),
+ padding=padw,
+ bias=use_bias,
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ nf_mult_prev = nf_mult
+ nf_mult = min(2**n_layers, 8)
+ sequence += [
+ nn.Conv3d(
+ ndf * nf_mult_prev, ndf * nf_mult, kernel_size=(kw, kw, kw), stride=1, padding=padw, bias=use_bias
+ ),
+ norm_layer(ndf * nf_mult),
+ nn.LeakyReLU(0.2, True),
+ ]
+
+ sequence += [
+ nn.Conv3d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)
+ ] # output 1 channel prediction map
+ self.main = nn.Sequential(*sequence)
+
+ def forward(self, input):
+ """Standard forward."""
+ return self.main(input)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/distributions.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/distributions.py
new file mode 100644
index 0000000000000000000000000000000000000000..76e814475d4d32b9f5ead736cce3a234bb5a0e5f
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/distributions.py
@@ -0,0 +1,49 @@
+import numpy as np
+import torch
+
+
+class DiagonalGaussianDistribution(object):
+ def __init__(self, parameters, deterministic=False):
+ self.parameters = parameters
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
+ self.deterministic = deterministic
+ self.std = torch.exp(0.5 * self.logvar)
+ self.var = torch.exp(self.logvar)
+ if self.deterministic:
+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device)
+
+ def sample(self):
+ x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device)
+ return x
+
+ def kl(self, other=None):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ else:
+ if other is None:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar,
+ dim=[1, 2, 3],
+ )
+ else:
+ return 0.5 * torch.sum(
+ torch.pow(self.mean - other.mean, 2) / other.var
+ + self.var / other.var
+ - 1.0
+ - self.logvar
+ + other.logvar,
+ dim=[1, 2, 3],
+ )
+
+ def nll(self, sample, dims=[1, 2, 3]):
+ if self.deterministic:
+ return torch.Tensor([0.0])
+ logtwopi = np.log(2.0 * np.pi)
+ return 0.5 * torch.sum(
+ logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
+ dim=dims,
+ )
+
+ def mode(self):
+ return self.mean
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/ema.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/ema.py
new file mode 100644
index 0000000000000000000000000000000000000000..9f1f7606c2c9b68ebd2302215a9e08f9f31ed8ab
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/ema.py
@@ -0,0 +1,82 @@
+import torch
+from torch import nn
+
+
+class LitEma(nn.Module):
+ def __init__(self, model, decay=0.9999, use_num_upates=True):
+ super().__init__()
+ if decay < 0.0 or decay > 1.0:
+ raise ValueError("Decay must be between 0 and 1")
+
+ self.m_name2s_name = {}
+ self.register_buffer("decay", torch.tensor(decay, dtype=torch.float32))
+ self.register_buffer(
+ "num_updates",
+ torch.tensor(0, dtype=torch.int) if use_num_upates else torch.tensor(-1, dtype=torch.int),
+ )
+
+ for name, p in model.named_parameters():
+ if p.requires_grad:
+ # remove as '.'-character is not allowed in buffers
+ s_name = name.replace(".", "")
+ self.m_name2s_name.update({name: s_name})
+ self.register_buffer(s_name, p.clone().detach().data)
+
+ self.collected_params = []
+
+ def reset_num_updates(self):
+ del self.num_updates
+ self.register_buffer("num_updates", torch.tensor(0, dtype=torch.int))
+
+ def forward(self, model):
+ decay = self.decay
+
+ if self.num_updates >= 0:
+ self.num_updates += 1
+ decay = min(self.decay, (1 + self.num_updates) / (10 + self.num_updates))
+
+ one_minus_decay = 1.0 - decay
+
+ with torch.no_grad():
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+
+ for key in m_param:
+ if m_param[key].requires_grad:
+ sname = self.m_name2s_name[key]
+ shadow_params[sname] = shadow_params[sname].type_as(m_param[key])
+ shadow_params[sname].sub_(one_minus_decay * (shadow_params[sname] - m_param[key]))
+ else:
+ assert not key in self.m_name2s_name
+
+ def copy_to(self, model):
+ m_param = dict(model.named_parameters())
+ shadow_params = dict(self.named_buffers())
+ for key in m_param:
+ if m_param[key].requires_grad:
+ m_param[key].data.copy_(shadow_params[self.m_name2s_name[key]].data)
+ else:
+ assert not key in self.m_name2s_name
+
+ def store(self, parameters):
+ """
+ Save the current parameters for restoring later.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ temporarily stored.
+ """
+ self.collected_params = [param.clone() for param in parameters]
+
+ def restore(self, parameters):
+ """
+ Restore the parameters stored with the `store` method.
+ Useful to validate the model with EMA parameters without affecting the
+ original optimization process. Store the parameters before the
+ `copy_to` method. After validation (or model saving), use this to
+ restore the former parameters.
+ Args:
+ parameters: Iterable of `torch.nn.Parameter`; the parameters to be
+ updated with the stored parameters.
+ """
+ for c_param, param in zip(self.collected_params, parameters):
+ param.data.copy_(c_param.data)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/logger.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..23c7d066e6508433cb2141c59dbd80cb0030ab6d
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/logger.py
@@ -0,0 +1,289 @@
+import os
+import numpy as np
+import einops
+import imageio
+from typing import Union
+from matplotlib import pyplot as plt
+from PIL import Image, ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True # UnidentifiedImageError: https://github.com/python-pillow/Pillow/issues/5631
+from pathlib import Path
+
+import torch
+import torchvision
+import wandb
+
+import lightning.pytorch as pl
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.loggers import WandbLogger
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+from .util import exists, isheatmap
+
+
+class ImageVideoLogger(Callback):
+ def __init__(
+ self,
+ batch_frequency,
+ max_samples,
+ clamp=True,
+ increase_log_steps=True,
+ batch_frequency_val=None,
+ video_fps=8,
+ rescale=True,
+ disabled=False,
+ log_on_batch_idx=True, # log on batch_idx instead of global_step. global_step is fixed in validation. batch_idx restarts at each validation
+ log_first_step=True,
+ log_images_kwargs=None,
+ log_videos_kwargs=None,
+ log_before_first_step=True,
+ enable_autocast=True,
+ ):
+ super().__init__()
+ self.enable_autocast = enable_autocast
+ self.rescale = rescale
+ self.batch_freq = batch_frequency
+ self.batch_freq_val = batch_frequency_val if batch_frequency_val is not None else batch_frequency
+ self.video_fps = video_fps
+ self.max_samples = max_samples
+ self.log_steps = [2**n for n in range(int(np.log2(self.batch_freq)) + 1)]
+ if not increase_log_steps:
+ self.log_steps = [self.batch_freq]
+ self.clamp = clamp
+ self.disabled = disabled
+ self.log_on_batch_idx = log_on_batch_idx
+ self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}
+ self.log_videos_kwargs = log_videos_kwargs if log_videos_kwargs else {}
+ self.log_first_step = log_first_step
+ self.log_before_first_step = log_before_first_step
+
+ @rank_zero_only
+ def log_img_local(
+ self,
+ save_dir,
+ split,
+ images,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "images", split)
+ for k in images:
+ if isheatmap(images[k]):
+ fig, ax = plt.subplots()
+ ax = ax.matshow(
+ images[k].cpu().numpy(), cmap="hot", interpolation="lanczos"
+ )
+ plt.colorbar(ax)
+ plt.axis("off")
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ plt.savefig(path)
+ plt.close()
+ else:
+ if images[k].ndim == 5:
+ images[k] = einops.rearrange(images[k], "b c t h w -> (b t) c h w")
+ nrow = self.log_images_kwargs.get("n_rows", 8)
+ grid = torchvision.utils.make_grid(images[k], nrow=nrow)
+ if self.rescale:
+ grid = (grid + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)
+ grid = grid.numpy()
+ grid = (grid * 255).astype(np.uint8)
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.png".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ path = os.path.join(root, filename)
+ os.makedirs(os.path.split(path)[0], exist_ok=True)
+ img = Image.fromarray(grid)
+ img.save(path)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "logger_log_image only supports WandbLogger currently"
+ pl_module.logger.log_image(
+ key=f"{split}/{k}",
+ images=[
+ img,
+ ],
+ step=pl_module.global_step,
+ )
+
+ @rank_zero_only
+ def log_vid_local(
+ self,
+ save_dir,
+ split,
+ videos,
+ global_step,
+ current_epoch,
+ batch_idx,
+ pl_module: Union[None, pl.LightningModule] = None,
+ ):
+ root = os.path.join(save_dir, "videos", split)
+ for k in videos:
+ # if is video, we can add captions
+ if isinstance(videos[k], torch.Tensor) and videos[k].ndim == 5:
+ if self.rescale:
+ videos[k] = (videos[k] + 1.0) / 2.0 # -1,1 -> 0,1; c,h,w
+ frames = [videos[k][:, :, i] for i in range(videos[k].shape[2])]
+ frames = [torchvision.utils.make_grid(each, nrow=4) for each in frames]
+ frames = [einops.rearrange(each, "c h w -> 1 c h w") for each in frames]
+ frames = torch.clamp(torch.cat(frames, dim=0), min=0.0, max=1.0)
+ frames = (frames.numpy() * 255).astype(np.uint8)
+
+ filename = "{}_gs-{:06}_e-{:06}_b-{:06}.gif".format(
+ k, global_step, current_epoch, batch_idx
+ )
+ os.makedirs(root, exist_ok=True)
+ path = os.path.join(root, filename)
+ save_numpy_as_gif(frames, path, duration=1 / self.video_fps)
+ if exists(pl_module):
+ assert isinstance(
+ pl_module.logger, WandbLogger
+ ), "log_videos only supports WandbLogger currently"
+ wandb.log({f"{split}/{k}": wandb.Video(frames, fps=self.video_fps)}) # k is str
+
+ @rank_zero_only
+ def log_img(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_images") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_images)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ images = pl_module.log_images(batch)
+
+ for k in images:
+ N = min(images[k].shape[0], self.max_samples)
+ if not isheatmap(images[k]):
+ images[k] = images[k][:N]
+ if isinstance(images[k], torch.Tensor):
+ images[k] = images[k].detach().float().cpu()
+ if self.clamp and not isheatmap(images[k]):
+ images[k] = torch.clamp(images[k], -1.0, 1.0)
+
+ self.log_img_local(
+ pl_module.logger.save_dir,
+ split,
+ images,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ @rank_zero_only
+ def log_vid(self, pl_module, batch, batch_idx, split="train"):
+ check_idx = batch_idx if self.log_on_batch_idx else pl_module.global_step
+ if (
+ (self.check_frequency(check_idx) or self.check_frequency_val(batch_idx, split))
+ and hasattr(pl_module, "log_videos") # batch_idx % self.batch_freq == 0
+ and callable(pl_module.log_videos)
+ and self.max_samples > 0
+ ):
+ logger = type(pl_module.logger)
+ is_train = pl_module.training
+ if is_train:
+ pl_module.eval()
+
+ with torch.no_grad(), torch.autocast(enabled=self.enable_autocast, device_type="cuda"):
+ videos = pl_module.log_videos(
+ batch, split=split, **self.log_videos_kwargs
+ )
+
+ for k in videos:
+ N = min(videos[k].shape[0], self.max_samples)
+ videos[k] = videos[k][:N]
+ if isinstance(videos[k], torch.Tensor):
+ videos[k] = videos[k].detach().float().cpu()
+ if self.clamp:
+ videos[k] = torch.clamp(videos[k], -1.0, 1.0)
+
+ self.log_vid_local(
+ pl_module.logger.save_dir,
+ split,
+ videos,
+ pl_module.global_step,
+ pl_module.current_epoch,
+ batch_idx,
+ pl_module=pl_module
+ if isinstance(pl_module.logger, WandbLogger)
+ else None,
+ )
+
+ if is_train:
+ pl_module.train()
+
+ def check_frequency(self, check_idx):
+ if ((check_idx % self.batch_freq) == 0 or (check_idx in self.log_steps)) and (
+ check_idx > 0 or self.log_first_step
+ ):
+ try:
+ self.log_steps.pop(0)
+ except IndexError as e:
+ pass
+ return True
+ return False
+
+ def check_frequency_val(self, check_idx, split):
+ if 'val' in split:
+ if ((check_idx % self.batch_freq_val) == 0) and (
+ check_idx > 0 or self.log_first_step):
+ return True
+ return False
+
+ @rank_zero_only
+ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+ if not self.disabled and (pl_module.global_step > 0 or self.log_first_step):
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_train_batch_start(self, trainer, pl_module, batch, batch_idx):
+ if self.log_before_first_step and pl_module.global_step == 0:
+ self.log_img(pl_module, batch, batch_idx, split="train")
+ self.log_vid(pl_module, batch, batch_idx, split="train")
+
+ @rank_zero_only
+ def on_validation_batch_end(
+ self, trainer, pl_module, outputs, batch, batch_idx, *args, **kwargs
+ ):
+ if not self.disabled and pl_module.global_step > 0:
+ self.log_img(pl_module, batch, batch_idx, split="val")
+ self.log_vid(pl_module, batch, batch_idx, split="val")
+ if hasattr(pl_module, "calibrate_grad_norm"):
+ if (
+ pl_module.calibrate_grad_norm and batch_idx % 25 == 0
+ ) and batch_idx > 0:
+ self.log_gradients(trainer, pl_module, batch_idx=batch_idx)
+
+
+def save_numpy_as_gif(frames, path, duration=None):
+ """
+ save numpy array as gif file
+ """
+ image_list = []
+ for frame in frames:
+ image = frame.transpose(1, 2, 0)
+ image_list.append(image)
+ if duration:
+ imageio.mimsave(path, image_list, format="GIF", duration=duration, loop=0)
+ else:
+ imageio.mimsave(path, image_list, format="GIF", loop=0)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/losses.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..c96f471c72124c56025b40a17f7d7bda81446ca5
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/losses.py
@@ -0,0 +1,262 @@
+from typing import Any, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .discriminator import (NLayerDiscriminator, NLayerDiscriminator3D,
+ weights_init)
+from .lpips import LPIPS
+from .util import default, print0
+
+
+def hinge_d_loss(logits_real, logits_fake):
+ loss_real = torch.mean(F.relu(1.0 - logits_real))
+ loss_fake = torch.mean(F.relu(1.0 + logits_fake))
+ d_loss = 0.5 * (loss_real + loss_fake)
+ return d_loss
+
+
+def vanilla_d_loss(logits_real, logits_fake):
+ d_loss = 0.5 * (torch.mean(F.softplus(-logits_real)) + torch.mean(F.softplus(logits_fake)))
+ return d_loss
+
+
+def adopt_weight(weight, global_step, threshold=0, value=0.0):
+ if global_step < threshold:
+ weight = value
+ return weight
+
+
+def _sigmoid_cross_entropy_with_logits(labels, logits):
+ """
+ non-saturating loss
+ """
+ zeros = torch.zeros_like(logits, dtype=logits.dtype)
+ condition = logits >= zeros
+ relu_logits = torch.where(condition, logits, zeros)
+ neg_abs_logits = torch.where(condition, -logits, logits)
+ return relu_logits - logits * labels + torch.log1p(torch.exp(neg_abs_logits))
+
+
+def non_saturate_gen_loss(logits_fake):
+ """
+ logits_fake: [B 1 H W]
+ """
+ B = logits_fake.shape[0]
+ logits_fake = logits_fake.reshape(B, -1)
+ logits_fake = torch.mean(logits_fake, dim=-1)
+ gen_loss = torch.mean(_sigmoid_cross_entropy_with_logits(labels=torch.ones_like(logits_fake), logits=logits_fake))
+ return gen_loss
+
+
+def lecam_reg(real_pred, fake_pred, lecam_ema):
+ reg = torch.mean(F.relu(real_pred - lecam_ema.logits_fake_ema).pow(2)) + torch.mean(
+ F.relu(lecam_ema.logits_real_ema - fake_pred).pow(2)
+ )
+ return reg
+
+
+class LeCAM_EMA(object):
+ # https://github.com/TencentARC/SEED-Voken/blob/main/src/Open_MAGVIT2/modules/losses/vqperceptual.py
+ def __init__(self, init=0.0, decay=0.999):
+ self.logits_real_ema = init
+ self.logits_fake_ema = init
+ self.decay = decay
+
+ def update(self, logits_real, logits_fake):
+ self.logits_real_ema = self.logits_real_ema * self.decay + torch.mean(logits_real).item() * (1 - self.decay)
+ self.logits_fake_ema = self.logits_fake_ema * self.decay + torch.mean(logits_fake).item() * (1 - self.decay)
+
+
+class GeneralLPIPSWithDiscriminator(nn.Module):
+ def __init__(
+ self,
+ disc_start: int,
+ logvar_init: float = 0.0,
+ pixelloss_weight=1.0,
+ disc_num_layers: int = 3,
+ disc_in_channels: int = 3,
+ disc_factor: float = 1.0,
+ disc_weight: float = 1.0,
+ disc_type: str = "3d",
+ perceptual_weight: float = 1.0,
+ lecam_loss_weight: float = 0.0,
+ disc_loss: str = "hinge",
+ scale_input_to_tgt_size: bool = False,
+ dims: int = 2,
+ learn_logvar: bool = False,
+ regularization_weights: Union[None, dict] = None,
+ gen_loss_cross_entropy: bool = False,
+ ):
+ super().__init__()
+ self.dims = dims
+ if self.dims > 2:
+ print0(
+ f"[bold cyan]\[vidtok.modules.losses][GeneralLPIPSWithDiscriminator][/bold cyan] running with dims={dims}. This means that for perceptual loss calculation, "
+ f"the LPIPS loss will be applied to each frame independently. "
+ )
+ self.scale_input_to_tgt_size = scale_input_to_tgt_size
+ assert disc_loss in ["hinge", "vanilla"]
+ self.pixel_weight = pixelloss_weight
+ self.perceptual_loss = LPIPS().eval()
+ self.perceptual_weight = perceptual_weight
+ # output log variance
+ self.logvar = nn.Parameter(torch.ones(size=()) * logvar_init)
+ self.learn_logvar = learn_logvar
+ self.disc_type = disc_type
+ assert self.disc_type in ["2d", "3d"]
+
+ if self.disc_type == "2d":
+ self.discriminator = NLayerDiscriminator(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ else:
+ self.discriminator = NLayerDiscriminator3D(
+ input_nc=disc_in_channels, n_layers=disc_num_layers, use_actnorm=False
+ ).apply(weights_init)
+ self.discriminator_iter_start = disc_start
+ self.disc_loss = hinge_d_loss if disc_loss == "hinge" else vanilla_d_loss
+ self.disc_factor = disc_factor
+ self.discriminator_weight = disc_weight
+ self.regularization_weights = default(regularization_weights, {})
+ self.gen_loss_cross_entropy = gen_loss_cross_entropy
+ self.lecam_loss_weight = lecam_loss_weight
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema = LeCAM_EMA()
+
+ def get_trainable_parameters(self) -> Any:
+ return self.discriminator.parameters()
+
+ def get_trainable_autoencoder_parameters(self) -> Any:
+ if self.learn_logvar:
+ yield self.logvar
+ yield from ()
+
+ def calculate_adaptive_weight(self, nll_loss, g_loss, last_layer=None):
+ if last_layer is not None:
+ nll_grads = torch.autograd.grad(nll_loss, last_layer, retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, last_layer, retain_graph=True)[0]
+ else:
+ nll_grads = torch.autograd.grad(nll_loss, self.last_layer[0], retain_graph=True)[0]
+ g_grads = torch.autograd.grad(g_loss, self.last_layer[0], retain_graph=True)[0]
+
+ d_weight = torch.norm(nll_grads) / (torch.norm(g_grads) + 1e-4)
+ d_weight = torch.clamp(d_weight, 0.0, 1e4).detach()
+ d_weight = d_weight * self.discriminator_weight
+ return d_weight
+
+ def forward(
+ self,
+ regularization_log,
+ inputs,
+ reconstructions,
+ optimizer_idx,
+ global_step,
+ last_layer=None,
+ split="train",
+ weights=None,
+ ):
+ if self.scale_input_to_tgt_size:
+ inputs = torch.nn.functional.interpolate(inputs, reconstructions.shape[2:], mode="bicubic", antialias=True)
+
+ if optimizer_idx == 0:
+ bs = inputs.shape[0]
+ t = inputs.shape[2]
+ if self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ rec_loss = torch.abs(inputs.contiguous() - reconstructions.contiguous())
+ if self.perceptual_weight > 0:
+ p_loss = self.perceptual_loss(inputs.contiguous(), reconstructions.contiguous())
+ rec_loss = rec_loss + self.perceptual_weight * p_loss
+ else:
+ p_loss = torch.Tensor([0.0])
+
+ nll_loss = rec_loss / torch.exp(self.logvar) + self.logvar
+ weighted_nll_loss = nll_loss
+ if weights is not None:
+ weighted_nll_loss = weights * nll_loss
+ weighted_nll_loss = torch.sum(weighted_nll_loss) / weighted_nll_loss.shape[0]
+ nll_loss = torch.sum(nll_loss) / nll_loss.shape[0]
+
+ # now the GAN part
+ if self.disc_type == "3d":
+ reconstructions = rearrange(reconstructions, "(b t) c h w -> b c t h w", t=t).contiguous()
+
+ # generator update
+ logits_fake = self.discriminator(reconstructions)
+
+ if not self.gen_loss_cross_entropy:
+ g_loss = -torch.mean(logits_fake)
+ else:
+ g_loss = non_saturate_gen_loss(logits_fake)
+
+ if self.disc_factor > 0.0:
+ try:
+ d_weight = self.calculate_adaptive_weight(nll_loss, g_loss, last_layer=last_layer)
+ except RuntimeError:
+ assert not self.training
+ d_weight = torch.tensor(0.0)
+ else:
+ d_weight = torch.tensor(0.0)
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+ loss = weighted_nll_loss + d_weight * disc_factor * g_loss
+ log = dict()
+ for k in regularization_log:
+ if k in self.regularization_weights:
+ loss = loss + self.regularization_weights[k] * regularization_log[k]
+ log[f"{split}/{k}"] = regularization_log[k].detach().mean()
+
+ log.update(
+ {
+ "{}/total_loss".format(split): loss.clone().detach().mean(),
+ "{}/logvar".format(split): self.logvar.detach(),
+ "{}/nll_loss".format(split): nll_loss.detach().mean(),
+ "{}/rec_loss".format(split): rec_loss.detach().mean(),
+ "{}/p_loss".format(split): p_loss.detach().mean(),
+ "{}/d_weight".format(split): d_weight.detach(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/g_loss".format(split): g_loss.detach().mean(),
+ }
+ )
+ return loss, log
+
+ if optimizer_idx == 1:
+ if self.disc_type == "2d" and self.dims > 2:
+ inputs, reconstructions = map(
+ lambda x: rearrange(x, "b c t h w -> (b t) c h w"),
+ (inputs, reconstructions),
+ )
+
+ logits_real = self.discriminator(inputs.contiguous().detach())
+ logits_fake = self.discriminator(reconstructions.contiguous().detach())
+
+ disc_factor = adopt_weight(self.disc_factor, global_step, threshold=self.discriminator_iter_start)
+
+ non_saturate_d_loss = self.disc_loss(logits_real, logits_fake)
+
+ if self.lecam_loss_weight > 0:
+ self.lecam_ema.update(logits_real, logits_fake)
+ lecam_loss = lecam_reg(logits_real, logits_fake, self.lecam_ema)
+ d_loss = disc_factor * (lecam_loss * self.lecam_loss_weight + non_saturate_d_loss)
+ else:
+ d_loss = disc_factor * non_saturate_d_loss
+
+ log = {
+ "{}/disc_loss".format(split): d_loss.clone().detach().mean(),
+ "{}/logits_real".format(split): logits_real.detach().mean(),
+ "{}/logits_fake".format(split): logits_fake.detach().mean(),
+ "{}/disc_factor".format(split): torch.tensor(disc_factor),
+ "{}/non_saturated_d_loss".format(split): non_saturate_d_loss.detach(),
+ }
+
+ if self.lecam_loss_weight > 0:
+ log.update({"{}/lecam_loss".format(split): lecam_loss.detach()})
+
+ return d_loss, log
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/lpips.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/lpips.py
new file mode 100644
index 0000000000000000000000000000000000000000..22bb5fa315618c0d1f0463d67ea57c29083fe302
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/lpips.py
@@ -0,0 +1,172 @@
+import hashlib
+import os
+from collections import namedtuple
+from tqdm import tqdm
+
+import requests
+import torch
+import torch.nn as nn
+from torchvision import models
+
+from .util import print0
+
+URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"}
+
+CKPT_MAP = {"vgg_lpips": "vgg.pth"}
+
+MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"}
+
+
+def download(url, local_path, chunk_size=1024):
+ os.makedirs(os.path.split(local_path)[0], exist_ok=True)
+ with requests.get(url, stream=True) as r:
+ total_size = int(r.headers.get("content-length", 0))
+ with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
+ with open(local_path, "wb") as f:
+ for data in r.iter_content(chunk_size=chunk_size):
+ if data:
+ f.write(data)
+ pbar.update(chunk_size)
+
+
+def md5_hash(path):
+ with open(path, "rb") as f:
+ content = f.read()
+ return hashlib.md5(content).hexdigest()
+
+
+def get_ckpt_path(name, root, check=False):
+ assert name in URL_MAP
+ path = os.path.join(root, CKPT_MAP[name])
+ if os.path.exists(path) and not (check and not md5_hash(path) == MD5_MAP[name]):
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Using existing path for {} model: {}".format(
+ name, path
+ )
+ )
+ return path
+
+ # if not, download the model
+ print0(
+ "[bold cyan]\[vidtok.modules.lpips]\[get_ckpt_path][/bold cyan] Downloading {} model from {} to {}".format(
+ name, URL_MAP[name], path
+ )
+ )
+ download(URL_MAP[name], path)
+ md5 = md5_hash(path)
+ assert md5 == MD5_MAP[name], md5
+ return path
+
+
+class LPIPS(nn.Module):
+ # Learned perceptual metric
+ def __init__(self, use_dropout=True):
+ super().__init__()
+ self.scaling_layer = ScalingLayer()
+ self.chns = [64, 128, 256, 512, 512] # vg16 features
+ self.net = vgg16(pretrained=True, requires_grad=False)
+ self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout)
+ self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout)
+ self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout)
+ self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout)
+ self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout)
+ self.load_from_pretrained()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def load_from_pretrained(self, name="vgg_lpips"):
+ ckpt = get_ckpt_path(name, "checkpoints/lpips")
+ self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False)
+ print0("[bold cyan]\[vidtok.modules.lpips][LPIPS][/bold cyan] loaded pretrained LPIPS loss from {}".format(ckpt))
+
+ def forward(self, input, target):
+ in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target))
+ outs0, outs1 = self.net(in0_input), self.net(in1_input)
+ feats0, feats1, diffs = {}, {}, {}
+ lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
+ for kk in range(len(self.chns)):
+ feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk])
+ diffs[kk] = (feats0[kk] - feats1[kk]) ** 2
+
+ res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))]
+ val = res[0]
+ for l in range(1, len(self.chns)):
+ val += res[l]
+ return val
+
+
+class ScalingLayer(nn.Module):
+ def __init__(self):
+ super(ScalingLayer, self).__init__()
+ self.register_buffer("shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None])
+ self.register_buffer("scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None])
+
+ def forward(self, inp):
+ return (inp - self.shift) / self.scale
+
+
+class NetLinLayer(nn.Module):
+ """A single linear layer which does a 1x1 conv"""
+
+ def __init__(self, chn_in, chn_out=1, use_dropout=False):
+ super(NetLinLayer, self).__init__()
+ layers = (
+ [
+ nn.Dropout(),
+ ]
+ if (use_dropout)
+ else []
+ )
+ layers += [
+ nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False),
+ ]
+ self.model = nn.Sequential(*layers)
+
+
+class vgg16(torch.nn.Module):
+ def __init__(self, requires_grad=False, pretrained=True):
+ super(vgg16, self).__init__()
+ vgg_pretrained_features = models.vgg16(pretrained=pretrained).features
+ self.slice1 = torch.nn.Sequential()
+ self.slice2 = torch.nn.Sequential()
+ self.slice3 = torch.nn.Sequential()
+ self.slice4 = torch.nn.Sequential()
+ self.slice5 = torch.nn.Sequential()
+ self.N_slices = 5
+ for x in range(4):
+ self.slice1.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(4, 9):
+ self.slice2.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(9, 16):
+ self.slice3.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(16, 23):
+ self.slice4.add_module(str(x), vgg_pretrained_features[x])
+ for x in range(23, 30):
+ self.slice5.add_module(str(x), vgg_pretrained_features[x])
+ if not requires_grad:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, X):
+ h = self.slice1(X)
+ h_relu1_2 = h
+ h = self.slice2(h)
+ h_relu2_2 = h
+ h = self.slice3(h)
+ h_relu3_3 = h
+ h = self.slice4(h)
+ h_relu4_3 = h
+ h = self.slice5(h)
+ h_relu5_3 = h
+ vgg_outputs = namedtuple("VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"])
+ out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3)
+ return out
+
+
+def normalize_tensor(x, eps=1e-10):
+ norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True))
+ return x / (norm_factor + eps)
+
+
+def spatial_average(x, keepdim=True):
+ return x.mean([2, 3], keepdim=keepdim)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..c71a8b09e370451046eef9ba60315feab2459e35
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal.py
@@ -0,0 +1,885 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+ self.time_causal_padding = (self.time_pad, 0)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (height_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.time_causal_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ time_pad,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ def forward(self, x):
+ pad_mode = self.pad_mode if self.time_pad < x.shape[2] else "constant"
+
+ x = F.pad(x, self.time_causal_padding, mode=pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+ x1 = self.avg_pool(torch.nn.functional.pad(x, pad, mode="constant", value=0))
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest").to(
+ x.dtype
+ )
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ x = pad_at_dim(x, (self.time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in)
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.time_padding = self.time_downsample_factor - 1
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x[:, :, self.time_padding :, :, :]
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py
new file mode 100644
index 0000000000000000000000000000000000000000..44397a135334a58cf5774fb899152f701d56a37d
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dcausal_v1_1.py
@@ -0,0 +1,959 @@
+from typing import Callable
+from beartype import beartype
+from beartype.typing import Tuple, Union
+
+import einops
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+
+from .util import checkpoint
+
+
+def spatial_temporal_resblk(x, block_s, block_t, temb):
+ assert len(x.shape) == 5, "input should be 5D tensor, but got {}D tensor".format(len(x.shape))
+ B, C, T, H, W = x.shape
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ x = block_s(x, temb)
+ x = einops.rearrange(x, "(b t) c h w -> b c t h w", b=B, t=T)
+ x = einops.rearrange(x, "b c t h w -> (b h w) c t")
+ x = block_t(x, temb)
+ x = einops.rearrange(x, "(b h w) c t -> b c t h w", b=B, h=H, w=W)
+ return x
+
+
+def nonlinearity(x):
+ return x * torch.sigmoid(x)
+
+
+def Normalize(in_channels, num_groups=32, norm_type="groupnorm"):
+ if norm_type == "groupnorm":
+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)
+ elif norm_type == "layernorm":
+ return LayerNorm(num_channels=in_channels, eps=1e-6)
+
+
+def pad_at_dim(t, pad, dim=-1, pad_mode="constant", value=0.0):
+ assert pad_mode in ["constant", "replicate", "reflect"]
+ dims_from_right = (-dim - 1) if dim < 0 else (t.ndim - dim - 1)
+ zeros = (0, 0) * dims_from_right
+ if pad_mode == "constant":
+ return F.pad(t, (*zeros, *pad), value=value)
+ return F.pad(t, (*zeros, *pad), mode=pad_mode)
+
+
+def divisible_by(num, den):
+ return (num % den) == 0
+
+
+def is_odd(n):
+ return not divisible_by(n, 2)
+
+
+def cast_tuple(t, length=1):
+ return t if isinstance(t, tuple) else ((t,) * length)
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, num_channels, eps=1e-6, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.norm = torch.nn.LayerNorm(num_channels, eps=eps, elementwise_affine=True)
+
+ def forward(self, x):
+ if x.dim() == 5:
+ x = rearrange(x, "b c t h w -> b t h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b t h w c -> b c t h w")
+ elif x.dim() == 4:
+ x = rearrange(x, "b c h w -> b h w c")
+ x = self.norm(x)
+ x = rearrange(x, "b h w c -> b c h w")
+ else:
+ x = rearrange(x, "b c s -> b s c")
+ x = self.norm(x)
+ x = rearrange(x, "b s c -> b c s")
+ return x
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+
+ self.norm = Normalize(in_channels, norm_type=self.norm_type)
+ self.q = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c h w -> b 1 (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b 1 (h w) c -> b c h w", h=h, w=w, c=c, b=b)
+
+ def forward(self, x, **kwargs):
+ if self.use_checkpoint:
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x)
+
+ def _forward(self, x, **kwargs):
+ h_ = x
+ h_ = self.attention(h_)
+ h_ = self.proj_out(h_)
+ return x + h_
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.k = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.v = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+ self.proj_out = CausalConv3d(in_channels, in_channels, kernel_size=1, stride=1)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ B = h_.shape[0]
+ h_ = rearrange(h_, "b c t h w -> (b t) c h w")
+ h_ = self.norm(h_)
+ h_ = rearrange(h_, "(b t) c h w -> b c t h w", b=B)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class CausalConv1d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: int, pad_mode="constant", **kwargs):
+ super().__init__()
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ self.pad_mode = pad_mode
+ self.time_pad = dilation * (kernel_size - 1) + (1 - stride)
+
+ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1].repeat(
+ (1, 1, self.time_pad)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ return self.conv(x)
+
+
+class CausalConv3d(nn.Module):
+ @beartype
+ def __init__(self, chan_in, chan_out, kernel_size: Union[int, Tuple[int, int, int]], pad_mode="constant", **kwargs):
+ super().__init__()
+ kernel_size = cast_tuple(kernel_size, 3)
+ dilation = kwargs.pop("dilation", 1)
+ stride = kwargs.pop("stride", 1)
+ dilation = cast_tuple(dilation, 3)
+ stride = cast_tuple(stride, 3)
+
+ time_kernel_size, height_kernel_size, width_kernel_size = kernel_size
+
+ assert is_odd(height_kernel_size) and is_odd(width_kernel_size)
+
+ self.pad_mode = pad_mode
+ time_pad = dilation[0] * (time_kernel_size - 1) + (1 - stride[0])
+ height_pad = dilation[1] * (height_kernel_size - 1) + (1 - stride[1])
+ width_pad = dilation[2] * (width_kernel_size - 1) + (1 - stride[2])
+
+ self.time_pad = time_pad
+ self.spatial_padding = (
+ width_pad // 2,
+ width_pad - width_pad // 2,
+ height_pad // 2,
+ height_pad - height_pad // 2,
+ 0,
+ 0,
+ )
+
+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+ self.cache_offset = 0
+
+ def forward(self, x):
+ if self.is_first_chunk:
+ first_frame_pad = x[:, :, :1, :, :].repeat(
+ (1, 1, self.time_pad, 1, 1)
+ )
+ else:
+ first_frame_pad = self.causal_cache
+ if self.time_pad != 0:
+ first_frame_pad = first_frame_pad[:, :, -self.time_pad:]
+ else:
+ first_frame_pad = first_frame_pad[:, :, 0:0]
+
+ x = torch.concatenate((first_frame_pad, x), dim=2)
+
+ if self.cache_offset == 0:
+ self.causal_cache = x.clone()
+ else:
+ self.causal_cache = x[:,:,:-self.cache_offset].clone()
+
+ x = F.pad(x, self.spatial_padding, mode=self.pad_mode)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = CausalConv3d(in_channels, out_channels, 3, stride=(2, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 1, 0)
+
+ if self.is_first_chunk:
+ x_pad = torch.nn.functional.pad(x, pad, mode="replicate")
+ else:
+ x_pad = torch.concatenate((self.causal_cache, x), dim=2)
+
+ self.causal_cache = x_pad[:,:,-1:].clone()
+
+ x1 = self.avg_pool(x_pad)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleResCausal2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ interpolation_mode='nearest',
+ num_temp_upsample=1
+ ):
+ super().__init__()
+ self.conv = CausalConv3d(in_channels, out_channels, 3)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ self.interpolation_mode = interpolation_mode
+ self.num_temp_upsample = num_temp_upsample
+ self.enable_cached = (self.interpolation_mode == 'trilinear')
+ self.is_first_chunk = True
+ self.causal_cache = None
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ if not self.enable_cached:
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ elif not self.is_first_chunk:
+ x = torch.cat([self.causal_cache, x], dim=2)
+ self.causal_cache = x[:, :, -2*self.num_temp_upsample:-self.num_temp_upsample].clone()
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ x = x[:, :, 2*self.num_temp_upsample:]
+ else:
+ self.causal_cache = x[:, :, -self.num_temp_upsample:].clone()
+ x, _x = x[:, :, :self.num_temp_upsample], x[:, :, self.num_temp_upsample:]
+ x = F.interpolate(x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(x.dtype)
+ if _x.shape[-3] > 0:
+ _x = F.interpolate(_x.to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode=self.interpolation_mode).to(_x.dtype)
+ x = torch.concat([x, _x], dim=2)
+
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetCausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv3d(
+ out_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=3,
+ stride=1,
+ )
+ else:
+ self.nin_shortcut = CausalConv3d(
+ in_channels,
+ out_channels,
+ kernel_size=1,
+ stride=1,
+ )
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+ return x + h
+
+
+class ResnetCausalBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = CausalConv1d(out_channels, out_channels, kernel_size=3, stride=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=3, stride=1)
+ else:
+ self.nin_shortcut = CausalConv1d(in_channels, out_channels, kernel_size=1, stride=1)
+
+ if zero_init:
+ self.conv2.conv.weight.data.zero_()
+ self.conv2.conv.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ B = x.shape[0]
+ h = x
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = rearrange(h, "(b s) c t -> (b t) c s", b=B)
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = rearrange(h, "(b t) c s -> (b s) c t", b=B)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class EncoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_ds=None,
+ tempo_ds=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.norm_type = norm_type
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.is_causal = True
+
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) if spatial_ds is None else spatial_ds
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3] if tempo_ds is None else tempo_ds
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetCausalBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level in self.spatial_ds:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleResCausal2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(block_in, norm_type=self.norm_type)
+
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ )
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, H, W = x.shape
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+
+ if i_level in self.spatial_ds:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal downsample
+ B, _, T, H, W = htmp.shape
+ if i_level in self.tempo_ds:
+ htmp = self.down_temporal[i_level].downsample(htmp)
+
+ hs.append(htmp)
+ B, _, T, H, W = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = self.conv_out(h)
+
+ return h
+
+
+class EncoderCausal3DPadding(EncoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.init_pad_mode = ignore_kwargs.get("init_pad_mode", "replicate")
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ video_len = x.shape[2]
+ if video_len % self.time_downsample_factor != 0:
+ time_padding = self.time_downsample_factor - video_len % self.time_downsample_factor
+ x = pad_at_dim(x, (time_padding, 0), dim=2, pad_mode=self.init_pad_mode, value=0.0)
+ return super().forward(x)
+
+
+class DecoderCausal3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ spatial_us=None,
+ tempo_us=None,
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.norm_type = norm_type
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.interpolation_mode = ignorekwargs.get("interpolation_mode", 'nearest')
+ assert self.interpolation_mode in ['nearest', 'trilinear']
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.spatial_us = list(range(1, self.num_resolutions)) if spatial_us is None else spatial_us
+ self.tempo_us = [1, 2] if tempo_us is None else tempo_us
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level in self.spatial_us:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ num_temp_upsample = 1
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetCausalBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleResCausal2x(block_in, block_in, interpolation_mode=self.interpolation_mode, num_temp_upsample=num_temp_upsample)
+ num_temp_upsample *= 2
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1)
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetCausalBlock
+
+ def _make_conv(self) -> Callable:
+ return CausalConv3d
+
+ def get_last_layer(self, **kwargs):
+ try:
+ return self.conv_out.conv.weight
+ except:
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, H, W = z.shape
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+
+ if i_level in self.spatial_us:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+
+ # temporal upsample
+ B, _, T, H, W = h.shape
+ if i_level in self.tempo_us:
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, H, W = h.shape
+
+ # end
+ if self.give_pre_end:
+ return h
+
+ B, C, T, H, W = h.shape
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.norm_out(h)
+ h = rearrange(h, "(b t) c h w -> b c t h w", b=B)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+
+ if self.tanh_out:
+ h = torch.tanh(h)
+
+ return h
+
+
+class DecoderCausal3DPadding(DecoderCausal3D):
+ def __init__(self, *args, **ignore_kwargs):
+ super().__init__(*args, **ignore_kwargs)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, x):
+ x = super().forward(x)
+ return x
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py
new file mode 100644
index 0000000000000000000000000000000000000000..4223fb635eb88cd7e7292943c131965f2b814206
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/model_3dnoncausal.py
@@ -0,0 +1,652 @@
+from typing import Callable
+
+import einops
+import torch
+import torch.nn as nn
+from einops import rearrange
+
+from .model_3dcausal import (AttnBlock, Normalize, nonlinearity,
+ spatial_temporal_resblk)
+from .util import checkpoint
+
+
+def make_attn(in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ return AttnBlockWrapper(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+
+
+class AttnBlockWrapper(AttnBlock):
+ def __init__(self, in_channels, use_checkpoint=False, norm_type="groupnorm"):
+ super().__init__(in_channels, use_checkpoint=use_checkpoint, norm_type=norm_type)
+ self.q = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.k = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.v = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+ self.proj_out = torch.nn.Conv3d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)
+
+ def attention(self, h_: torch.Tensor) -> torch.Tensor:
+ h_ = self.norm(h_)
+ q = self.q(h_)
+ k = self.k(h_)
+ v = self.v(h_)
+
+ b, c, t, h, w = q.shape
+ q, k, v = map(lambda x: rearrange(x, "b c t h w -> b t (h w) c").contiguous(), (q, k, v))
+ h_ = torch.nn.functional.scaled_dot_product_attention(q, k, v) # scale is dim ** -0.5 per default
+ return rearrange(h_, "b t (h w) c -> b c t h w", h=h, w=w, c=c, b=b)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x):
+ x = torch.nn.functional.interpolate(x.to(torch.float32), scale_factor=2.0, mode="nearest").to(x.dtype)
+ if self.with_conv:
+ x = self.conv(x)
+ return x
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels, with_conv):
+ super().__init__()
+ self.in_channels = in_channels
+ self.with_conv = with_conv
+ if self.with_conv:
+ self.conv = torch.nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x):
+ if self.with_conv:
+ pad = (0, 1, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x = self.conv(x)
+ else:
+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)
+ return x
+
+
+class TimeDownsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.kernel_size = (3, 3, 3)
+ self.avg_pool = nn.AvgPool3d((3, 1, 1), stride=(2, 1, 1))
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, stride=(2, 1, 1), padding=(0, 1, 1))
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ pad = (0, 0, 0, 0, 0, 1)
+ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)
+ x1 = self.avg_pool(x)
+ x2 = self.conv(x)
+ return alpha * x1 + (1 - alpha) * x2
+
+
+class TimeUpsampleRes2x(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ out_channels,
+ mix_factor: float = 2.0,
+ ):
+ super().__init__()
+ self.conv = nn.Conv3d(in_channels, out_channels, 3, padding=1)
+ # https://github.com/PKU-YuanGroup/Open-Sora-Plan/blob/main/opensora/models/causalvideovae/model/modules/updownsample.py
+ self.mix_factor = torch.nn.Parameter(torch.Tensor([mix_factor]))
+
+ def forward(self, x):
+ alpha = torch.sigmoid(self.mix_factor)
+ xlst = [
+ torch.nn.functional.interpolate(
+ sx.unsqueeze(0).to(torch.float32), scale_factor=[2.0, 1.0, 1.0], mode="nearest"
+ ).to(x.dtype)
+ for sx in x
+ ]
+ x = torch.cat(xlst, dim=0)
+ x_ = self.conv(x)
+ return alpha * x + (1 - alpha) * x_
+
+
+class ResnetBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = torch.nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = torch.nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetBlock1D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ zero_init=False,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ if zero_init:
+ self.conv2.weight.data.zero_()
+ self.conv2.bias.data.zero_()
+
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class ResnetNoncausalBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout,
+ temb_channels=512,
+ use_checkpoint=False,
+ norm_type="groupnorm",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.norm_type = norm_type
+
+ self.norm1 = Normalize(in_channels, norm_type=self.norm_type)
+ self.conv1 = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if temb_channels > 0:
+ self.temb_proj = torch.nn.Linear(temb_channels, out_channels)
+ self.norm2 = Normalize(out_channels, norm_type=self.norm_type)
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = nn.Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ self.conv_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ else:
+ self.nin_shortcut = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=1)
+ self.use_checkpoint = use_checkpoint
+
+ def forward(self, x, temb):
+ if self.use_checkpoint:
+ assert temb is None, "checkpointing not supported with temb"
+ return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint)
+ else:
+ return self._forward(x, temb)
+
+ def _forward(self, x, temb=None):
+ h = x
+ h = self.norm1(h)
+ h = nonlinearity(h)
+ h = self.conv1(h)
+
+ if temb is not None:
+ h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None]
+
+ h = self.norm2(h)
+ h = nonlinearity(h)
+ h = self.dropout(h)
+ h = self.conv2(h)
+
+ if self.in_channels != self.out_channels:
+ if self.use_conv_shortcut:
+ x = self.conv_shortcut(x)
+ else:
+ x = self.nin_shortcut(x)
+
+ return x + h
+
+
+class Encoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch=8,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels,
+ z_channels,
+ double_z=True,
+ norm_type="groupnorm",
+ **ignore_kwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignore_kwargs.get("use_checkpoint", False)
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.fix_encoder = ignore_kwargs.get("fix_encoder", False)
+ self.time_downsample_factor = ignore_kwargs.get("time_downsample_factor", 4)
+ self.tempo_ds = [self.num_resolutions - 2, self.num_resolutions - 3]
+ self.spatial_ds = list(range(0, self.num_resolutions - 1)) # add for spatial tiling
+ self.norm_type = norm_type
+ self.is_causal = False
+
+ # downsampling
+ make_conv_cls = self._make_conv()
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+
+ self.conv_in = make_conv_cls(in_channels, self.ch, kernel_size=3, stride=1, padding=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.down = nn.ModuleList()
+ self.down_temporal = nn.ModuleList()
+ for i_level in range(self.num_resolutions):
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_temporal = nn.ModuleList()
+ attn_temporal = nn.ModuleList()
+
+ for i_block in range(self.num_res_blocks):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_temporal.append(
+ ResnetBlock1D(
+ in_channels=block_out,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+
+ down_temporal = nn.Module()
+ down_temporal.block = block_temporal
+ down_temporal.attn = attn_temporal
+
+ if i_level != self.num_resolutions - 1:
+ down.downsample = Downsample(block_in, resamp_with_conv)
+ if i_level in self.tempo_ds:
+ down_temporal.downsample = TimeDownsampleRes2x(block_in, block_in)
+
+ self.down.append(down)
+ self.down_temporal.append(down_temporal)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn(block_in, norm_type=self.norm_type)
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(
+ block_in,
+ 2 * z_channels if double_z else z_channels,
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ )
+
+ if self.fix_encoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def forward(self, x):
+ temb = None
+ B, _, T, _, _ = x.shape
+
+ # downsampling
+ if x.shape[1] == 4 and self.conv_in.in_channels == 3:
+ raise ValueError("Mismatched number of input channels")
+ hs = [self.conv_in(x)]
+
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ h = spatial_temporal_resblk(
+ hs[-1], self.down[i_level].block[i_block], self.down_temporal[i_level].block[i_block], temb
+ )
+ hs.append(h)
+ if i_level != self.num_resolutions - 1:
+ # spatial downsample
+ htmp = einops.rearrange(hs[-1], "b c t h w -> (b t) c h w")
+ htmp = self.down[i_level].downsample(htmp)
+ htmp = einops.rearrange(htmp, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_ds:
+ # temporal downsample
+ htmp = self.down_temporal[i_level].downsample(htmp)
+ hs.append(htmp)
+ B, _, T, _, _ = htmp.shape
+
+ # middle
+ h = hs[-1]
+ h = self.mid.block_1(h, temb)
+ h = self.mid.attn_1(h)
+ h = self.mid.block_2(h, temb)
+
+ # end
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h)
+ return h
+
+
+class Decoder3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ ch,
+ out_ch,
+ ch_mult=(1, 2, 4, 8),
+ num_res_blocks,
+ dropout=0.0,
+ resamp_with_conv=True,
+ in_channels=8,
+ z_channels,
+ give_pre_end=False,
+ tanh_out=False,
+ norm_type="groupnorm",
+ **ignorekwargs,
+ ):
+ super().__init__()
+ use_checkpoint = ignorekwargs.get("use_checkpoint", False)
+
+ self.ch = ch
+ self.temb_ch = 0
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.in_channels = in_channels
+ self.give_pre_end = give_pre_end
+ self.tanh_out = tanh_out
+ self.fix_decoder = ignorekwargs.get("fix_decoder", False)
+ self.tempo_us = [1, 2]
+ self.norm_type = norm_type
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+
+ make_attn_cls = self._make_attn()
+ make_resblock_cls = self._make_resblock()
+ make_conv_cls = self._make_conv()
+ self.conv_in = make_conv_cls(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+ # middle
+ self.mid = nn.Module()
+ self.mid.block_1 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ self.mid.attn_1 = make_attn_cls(
+ block_in, use_checkpoint=use_checkpoint, norm_type=self.norm_type
+ )
+ self.mid.block_2 = make_resblock_cls(
+ in_channels=block_in,
+ out_channels=block_in,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+
+ # upsampling
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level != 0:
+ up.upsample = Upsample(block_in, resamp_with_conv)
+ self.up.insert(0, up)
+
+ self.up_temporal = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for i_block in range(self.num_res_blocks + 1):
+ block.append(
+ ResnetBlock1D(
+ in_channels=block_in,
+ out_channels=block_out,
+ temb_channels=self.temb_ch,
+ dropout=dropout,
+ zero_init=True,
+ use_checkpoint=use_checkpoint,
+ norm_type=self.norm_type,
+ )
+ )
+ block_in = block_out
+ up_temporal = nn.Module()
+ up_temporal.block = block
+ up_temporal.attn = attn
+ if i_level in self.tempo_us:
+ up_temporal.upsample = TimeUpsampleRes2x(block_in, block_in)
+
+ self.up_temporal.insert(0, up_temporal)
+
+ # end
+ self.norm_out = Normalize(block_in, norm_type=self.norm_type)
+ self.conv_out = make_conv_cls(block_in, out_ch, kernel_size=3, stride=1, padding=1)
+
+ if self.fix_decoder:
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def _make_attn(self) -> Callable:
+ return make_attn
+
+ def _make_resblock(self) -> Callable:
+ return ResnetNoncausalBlock
+
+ def _make_conv(self) -> Callable:
+ return nn.Conv3d
+
+ def get_last_layer(self, **kwargs):
+ return self.conv_out.weight
+
+ def forward(self, z, **kwargs):
+ temb = None
+ B, _, T, _, _ = z.shape
+
+ h = self.conv_in(z)
+
+ # middle
+ h = self.mid.block_1(h, temb, **kwargs)
+ h = self.mid.attn_1(h, **kwargs)
+ h = self.mid.block_2(h, temb, **kwargs)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ h = spatial_temporal_resblk(
+ h, self.up[i_level].block[i_block], self.up_temporal[i_level].block[i_block], temb
+ )
+ if i_level != 0:
+ # spatial upsample
+ h = einops.rearrange(h, "b c t h w -> (b t) c h w")
+ h = self.up[i_level].upsample(h)
+ h = einops.rearrange(h, "(b t) c h w -> b c t h w", b=B, t=T)
+ if i_level in self.tempo_us:
+ # temporal upsample
+ h = self.up_temporal[i_level].upsample(h)
+ B, _, T, _, _ = h.shape
+ # end
+ if self.give_pre_end:
+ return h
+ h = self.norm_out(h)
+ h = nonlinearity(h)
+ h = self.conv_out(h, **kwargs)
+ if self.tanh_out:
+ h = torch.tanh(h)
+ return h
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/regularizers.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/regularizers.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f4f1a4fccdb873f640e637ecbaaaef389e75b41
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/regularizers.py
@@ -0,0 +1,268 @@
+from abc import abstractmethod
+from functools import cache
+from typing import Any, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import pack, rearrange, reduce, unpack
+from torch import Tensor, int32
+from torch.cuda.amp import autocast
+
+from .distributions import DiagonalGaussianDistribution
+
+
+def exists(v):
+ return v is not None
+
+
+def default(*args):
+ for arg in args:
+ if exists(arg):
+ return arg
+ return None
+
+
+def pack_one(t, pattern):
+ return pack([t], pattern)
+
+
+def unpack_one(t, ps, pattern):
+ return unpack(t, ps, pattern)[0]
+
+
+def round_ste(z: Tensor) -> Tensor:
+ """Round with straight through gradients."""
+ zhat = z.round()
+ return z + (zhat - z).detach()
+
+
+def log(t, eps=1e-5):
+ return t.clamp(min=eps).log()
+
+
+def entropy(prob):
+ return (-prob * log(prob)).sum(dim=-1)
+
+
+def maybe_distributed_mean(t):
+ if not is_distributed():
+ return t
+ dist.all_reduce(t)
+ t = t / dist.get_world_size()
+ return t
+
+
+@cache
+def is_distributed():
+ return dist.is_initialized() and dist.get_world_size() > 1
+
+
+class AbstractRegularizer(nn.Module):
+ def __init__(self):
+ super().__init__()
+
+ def forward(self, z: torch.Tensor) -> Tuple[torch.Tensor, dict]:
+ raise NotImplementedError()
+
+ @abstractmethod
+ def get_trainable_parameters(self) -> Any:
+ raise NotImplementedError()
+
+
+class DiagonalGaussianRegularizer(AbstractRegularizer):
+ def __init__(self, sample: bool = True):
+ super().__init__()
+ self.sample = sample
+
+ def get_trainable_parameters(self) -> Any:
+ yield from ()
+
+ def forward(self, z: torch.Tensor, n_steps=None) -> Tuple[torch.Tensor, dict]:
+ log = dict()
+ posterior = DiagonalGaussianDistribution(z)
+ if self.sample:
+ z = posterior.sample()
+ else:
+ z = posterior.mode()
+ kl_loss = posterior.kl()
+ kl_loss = torch.sum(kl_loss) / kl_loss.shape[0]
+ log["kl_loss"] = kl_loss
+ return z, log
+
+
+class FSQRegularizer(AbstractRegularizer):
+ # https://github.com/lucidrains/vector-quantize-pytorch/blob/master/vector_quantize_pytorch/finite_scalar_quantization.py
+ def __init__(
+ self,
+ levels: List[int],
+ dim: Optional[int] = None,
+ num_codebooks=1,
+ keep_num_codebooks_dim: Optional[bool] = None,
+ scale: Optional[float] = None,
+ entropy_loss_weight: float = 0.0,
+ entropy_loss_annealing_steps: int = 0,
+ entropy_loss_annealing_factor: float = 1.0,
+ commitment_loss_weight: float = 0.0,
+ diversity_gamma: float = 1.0,
+ ):
+ super().__init__()
+ _levels = torch.tensor(levels, dtype=int32)
+ self.register_buffer("_levels", _levels, persistent=False)
+
+ _basis = torch.cumprod(torch.tensor([1] + levels[:-1]), dim=0, dtype=int32)
+ self.register_buffer("_basis", _basis, persistent=False)
+
+ self.scale = scale
+ self.entropy_loss_weight = entropy_loss_weight
+ self.entropy_loss_annealing_steps = entropy_loss_annealing_steps
+ self.entropy_loss_annealing_factor = entropy_loss_annealing_factor
+ self.commitment_loss_weight = commitment_loss_weight
+ self.diversity_gamma = diversity_gamma
+
+ codebook_dim = len(levels)
+ self.codebook_dim = codebook_dim
+
+ effective_codebook_dim = codebook_dim * num_codebooks
+ self.num_codebooks = num_codebooks
+ self.effective_codebook_dim = effective_codebook_dim
+
+ keep_num_codebooks_dim = default(keep_num_codebooks_dim, num_codebooks > 1)
+ assert not (num_codebooks > 1 and not keep_num_codebooks_dim)
+ self.keep_num_codebooks_dim = keep_num_codebooks_dim
+
+ self.dim = default(dim, len(_levels) * num_codebooks)
+
+ has_projections = self.dim != effective_codebook_dim
+ self.project_in = nn.Linear(self.dim, effective_codebook_dim) if has_projections else nn.Identity()
+ self.project_out = nn.Linear(effective_codebook_dim, self.dim) if has_projections else nn.Identity()
+ self.has_projections = has_projections
+
+ self.codebook_size = self._levels.prod().item()
+
+ implicit_codebook = self.indices_to_codes(torch.arange(self.codebook_size), project_out=False)
+ self.register_buffer("implicit_codebook", implicit_codebook, persistent=False)
+ self.register_buffer("zero", torch.tensor(0.0), persistent=False)
+
+ self.global_codebook_usage = torch.zeros([2**self.codebook_dim, self.num_codebooks], dtype=torch.long)
+
+ def get_trainable_parameters(self) -> Any:
+ return self.parameters()
+
+ def bound(self, z: Tensor, eps: float = 1e-3) -> Tensor:
+ """Bound `z`, an array of shape (..., d)."""
+ half_l = (self._levels - 1) * (1 + eps) / 2
+ offset = torch.where(self._levels % 2 == 0, 0.5, 0.0)
+ shift = (offset / half_l).atanh()
+ return (z + shift).tanh() * half_l - offset
+
+ def quantize(self, z: Tensor) -> Tensor:
+ """Quantizes z, returns quantized zhat, same shape as z."""
+ quantized = round_ste(self.bound(z))
+ half_width = self._levels // 2
+ return quantized / half_width
+
+ def _scale_and_shift(self, zhat_normalized: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat_normalized * half_width) + half_width
+
+ def _scale_and_shift_inverse(self, zhat: Tensor) -> Tensor:
+ half_width = self._levels // 2
+ return (zhat - half_width) / half_width
+
+ def codes_to_indices(self, zhat: Tensor) -> Tensor:
+ """Converts a `code` to an index in the codebook."""
+ assert zhat.shape[-1] == self.codebook_dim
+ zhat = self._scale_and_shift(zhat)
+ return (zhat * self._basis).sum(dim=-1).to(int32)
+
+ def indices_to_codes(self, indices: Tensor, project_out=True) -> Tensor:
+ """Inverse of `codes_to_indices`."""
+
+ is_img_or_video = indices.ndim >= (3 + int(self.keep_num_codebooks_dim))
+
+ indices = rearrange(indices, "... -> ... 1")
+ codes_non_centered = (indices // self._basis) % self._levels
+ codes = self._scale_and_shift_inverse(codes_non_centered)
+
+ if self.keep_num_codebooks_dim:
+ codes = rearrange(codes, "... c d -> ... (c d)")
+
+ if project_out:
+ codes = self.project_out(codes)
+
+ if is_img_or_video:
+ codes = rearrange(codes, "b ... d -> b d ...")
+
+ return codes
+
+ def calculate_entropy_loss_weight(self, n_steps):
+ if n_steps >= self.entropy_loss_annealing_steps:
+ return self.entropy_loss_weight
+ start = self.entropy_loss_annealing_factor * self.entropy_loss_weight
+ return start - (n_steps / self.entropy_loss_annealing_steps) * (start - self.entropy_loss_weight)
+
+ @autocast(enabled=False)
+ def forward(self, z: Tensor, inv_temperature: float = 100.0, n_steps: int = 0) -> Tensor:
+ """
+ einstein notation
+ b - batch
+ n - sequence (or flattened spatial dimensions)
+ d - feature dimension
+ c - number of codebook dim
+ """
+ is_img_or_video = z.ndim >= 4
+ if is_img_or_video:
+ z = rearrange(z, "b d ... -> b ... d")
+ z, ps = pack_one(z, "b * d")
+
+ assert z.shape[-1] == self.dim, f"expected dimension of {self.dim} but found dimension of {z.shape[-1]}"
+
+ z = self.project_in(z)
+ z = rearrange(z, "b n (c d) -> b n c d", c=self.num_codebooks)
+
+ with torch.autocast("cuda", enabled=False):
+ orig_dtype = z.dtype
+ z = z.float()
+ original_input = z
+ codes = self.quantize(z)
+ indices = self.codes_to_indices(codes)
+
+ if self.entropy_loss_weight > 0 or self.commitment_loss_weight > 0:
+ # the same as euclidean distance up to a constant
+ distance = -2 * torch.einsum("... i d, j d -> ... i j", original_input, self.implicit_codebook)
+ prob = (-distance * inv_temperature).softmax(dim=-1)
+ per_sample_probs = rearrange(prob, "b n ... -> (b n) ...")
+ per_sample_entropy = entropy(per_sample_probs).mean()
+ # distribution over all available tokens in the batch
+ avg_prob = reduce(per_sample_probs, "... c d -> c d", "mean")
+ avg_prob = maybe_distributed_mean(avg_prob)
+ codebook_entropy = entropy(avg_prob).mean()
+ entropy_aux_loss = per_sample_entropy - self.diversity_gamma * codebook_entropy
+ # commit loss
+ commit_loss = F.mse_loss(original_input, codes.detach(), reduction="none")
+ commit_loss = commit_loss.mean()
+ else:
+ entropy_aux_loss = per_sample_entropy = codebook_entropy = commit_loss = self.zero
+
+ codes = codes.type(orig_dtype)
+
+ codes = rearrange(codes, "b n c d -> b n (c d)")
+ out = self.project_out(codes)
+
+ # reconstitute image or video dimensions
+ if is_img_or_video:
+ out = unpack_one(out, ps, "b * d")
+ out = rearrange(out, "b ... d -> b d ...")
+
+ indices = unpack_one(indices, ps, "b * c")
+
+ if not self.keep_num_codebooks_dim:
+ indices = rearrange(indices, "... 1 -> ...")
+
+ aux_loss = (
+ entropy_aux_loss * self.calculate_entropy_loss_weight(n_steps) + commit_loss * self.commitment_loss_weight
+ )
+
+ return out, dict(indices=indices, aux_loss=aux_loss)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtok/modules/util.py b/Meissonic/vidtok_cache/VidTok/vidtok/modules/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..9570016221e47b50d536d30352b5718a29bb0009
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtok/modules/util.py
@@ -0,0 +1,324 @@
+import importlib
+import random
+import os
+import einops
+import numpy as np
+from inspect import isfunction
+from rich import print
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from lightning.pytorch.utilities.rank_zero import rank_zero_only
+
+
+def get_valid_dirs(dir1: str, dir2: str, dir3: Union[None, str] = None) -> Union[None, str]:
+ if (dir1 is not None) and os.path.isdir(dir1):
+ return dir1
+ elif (dir2 is not None) and os.path.isdir(dir2):
+ return dir2
+ elif (dir3 is not None) and os.path.isdir(dir3):
+ return dir3
+ else:
+ return None
+
+
+def get_valid_paths(path1: str, path2: str, path3: Union[None, str] = None) -> Union[None, str]:
+ if (path1 is not None) and os.path.isfile(path1):
+ return path1
+ elif (path2 is not None) and os.path.isfile(path2):
+ return path2
+ elif (path3 is not None) and os.path.isfile(path3):
+ return path3
+ else:
+ return None
+
+
+@rank_zero_only
+def print0(*args, **kwargs):
+ print(*args, **kwargs)
+
+
+def seed_anything(seed: int):
+ os.environ['PYTHONHASHSEED'] = str(seed)
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+
+
+def isheatmap(x):
+ if not isinstance(x, torch.Tensor):
+ return False
+
+ return x.ndim == 2
+
+
+def exists(x):
+ return x is not None
+
+
+def default(val, d):
+ if exists(val):
+ return val
+ return d() if isfunction(d) else d
+
+
+def instantiate_from_config(config):
+ if not "target" in config:
+ if config == "__is_first_stage__":
+ return None
+ elif config == "__is_unconditional__":
+ return None
+ raise KeyError("Expected key `target` to instantiate.")
+ return get_obj_from_str(config["target"])(**config.get("params", dict()))
+
+
+def get_obj_from_str(string, reload=False, invalidate_cache=True):
+ module, cls = string.rsplit(".", 1)
+ if invalidate_cache:
+ importlib.invalidate_caches()
+ if reload:
+ module_imp = importlib.import_module(module)
+ importlib.reload(module_imp)
+ return getattr(importlib.import_module(module, package=None), cls)
+
+
+def checkpoint(func, inputs, params, flag):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ """
+ Evaluate a function without caching intermediate activations, allowing for
+ reduced memory at the expense of extra compute in the backward pass.
+ :param func: the function to evaluate.
+ :param inputs: the argument sequence to pass to `func`.
+ :param params: a sequence of parameters `func` depends on but does not
+ explicitly take as arguments.
+ :param flag: if False, disable gradient checkpointing.
+ """
+ if flag:
+ args = tuple(inputs) + tuple(params)
+ return CheckpointFunction.apply(func, len(inputs), *args)
+ else:
+ return func(*inputs)
+
+
+class CheckpointFunction(torch.autograd.Function):
+ # https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py
+ @staticmethod
+ def forward(ctx, run_function, length, *args):
+ ctx.run_function = run_function
+ ctx.input_tensors = list(args[:length])
+ ctx.input_params = list(args[length:])
+ ctx.gpu_autocast_kwargs = {
+ "enabled": torch.is_autocast_enabled(),
+ "dtype": torch.get_autocast_gpu_dtype(),
+ "cache_enabled": torch.is_autocast_cache_enabled(),
+ }
+ with torch.no_grad():
+ output_tensors = ctx.run_function(*ctx.input_tensors)
+ return output_tensors
+
+ @staticmethod
+ def backward(ctx, *output_grads):
+ ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
+ # Ensure all tensors have requires_grad set to True
+ ctx.input_params = [p.requires_grad_(True) for p in ctx.input_params]
+ with torch.enable_grad(), torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs):
+ # Fixes a bug where the first op in run_function modifies the
+ # Tensor storage in place, which is not allowed for detach()'d
+ # Tensors.
+ shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
+ output_tensors = ctx.run_function(*shallow_copies)
+ input_grads = torch.autograd.grad(
+ output_tensors,
+ ctx.input_tensors + ctx.input_params,
+ output_grads,
+ allow_unused=True,
+ )
+ del ctx.input_tensors
+ del ctx.input_params
+ del output_tensors
+ return (None, None) + input_grads
+
+
+def compute_psnr(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ EPS = 1e-8
+ mse = torch.mean((x - y) ** 2, dim=[1, 2, 3])
+ psnr = -10 * torch.log10(mse + EPS)
+ return psnr.mean(dim=0)
+
+
+def compute_ssim(x, y):
+ if x.dim() == 5:
+ x = einops.rearrange(x, "b c t h w -> (b t) c h w")
+ assert y.dim() == 5
+ y = einops.rearrange(y, "b c t h w -> (b t) c h w")
+ kernel_size = 11
+ kernel_sigma = 1.5
+ k1 = 0.01
+ k2 = 0.03
+
+ f = max(1, round(min(x.size()[-2:]) / 256))
+ if f > 1:
+ x = F.avg_pool2d(x, kernel_size=f)
+ y = F.avg_pool2d(y, kernel_size=f)
+
+ kernel = gaussian_filter(kernel_size, kernel_sigma, device=x.device, dtype=x.dtype).repeat(x.size(1), 1, 1, 1)
+
+ _compute_ssim_per_channel = _ssim_per_channel_complex if x.dim() == 5 else _ssim_per_channel
+ ssim_map, cs_map = _compute_ssim_per_channel(x=x, y=y, kernel=kernel, data_range=1, k1=k1, k2=k2)
+ ssim_val = ssim_map.mean(1)
+
+ return ssim_val.mean(dim=0)
+
+
+def _ssim_per_channel(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W)`.
+ y: A target tensor. Shape :math:`(N, C, H, W)`.
+ kernel: 2D Gaussian kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Structural Similarity (SSIM) index.
+ """
+ if x.size(-1) < kernel.size(-1) or x.size(-2) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. "
+ f"Input size: {x.size()}. Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+ n_channels = x.size(1)
+ mu_x = F.conv2d(x, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu_y = F.conv2d(y, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu_xx = mu_x**2
+ mu_yy = mu_y**2
+ mu_xy = mu_x * mu_y
+
+ sigma_xx = F.conv2d(x**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xx
+ sigma_yy = F.conv2d(y**2, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_yy
+ sigma_xy = F.conv2d(x * y, weight=kernel, stride=1, padding=0, groups=n_channels) - mu_xy
+
+ # Contrast sensitivity (CS) with alpha = beta = gamma = 1.
+ cs = (2.0 * sigma_xy + c2) / (sigma_xx + sigma_yy + c2)
+
+ # Structural similarity (SSIM)
+ ss = (2.0 * mu_xy + c1) / (mu_xx + mu_yy + c1) * cs
+
+ ssim_val = ss.mean(dim=(-1, -2))
+ cs = cs.mean(dim=(-1, -2))
+ return ssim_val, cs
+
+
+def _ssim_per_channel_complex(
+ x: torch.Tensor,
+ y: torch.Tensor,
+ kernel: torch.Tensor,
+ data_range: Union[float, int] = 1.0,
+ k1: float = 0.01,
+ k2: float = 0.03,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ r"""Calculate Structural Similarity (SSIM) index for Complex X and Y per channel.
+
+ Args:
+ x: An input tensor. Shape :math:`(N, C, H, W, 2)`.
+ y: A target tensor. Shape :math:`(N, C, H, W, 2)`.
+ kernel: 2-D gauss kernel.
+ data_range: Maximum value range of images (usually 1.0 or 255).
+ k1: Algorithm parameter, K1 (small constant, see [1]).
+ k2: Algorithm parameter, K2 (small constant, see [1]).
+ Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results.
+
+ Returns:
+ Full Value of Complex Structural Similarity (SSIM) index.
+ """
+ n_channels = x.size(1)
+ if x.size(-2) < kernel.size(-1) or x.size(-3) < kernel.size(-2):
+ raise ValueError(
+ f"Kernel size can't be greater than actual input size. Input size: {x.size()}. "
+ f"Kernel size: {kernel.size()}"
+ )
+
+ c1 = k1**2
+ c2 = k2**2
+
+ x_real = x[..., 0]
+ x_imag = x[..., 1]
+ y_real = y[..., 0]
+ y_imag = y[..., 1]
+
+ mu1_real = F.conv2d(x_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu1_imag = F.conv2d(x_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_real = F.conv2d(y_real, weight=kernel, stride=1, padding=0, groups=n_channels)
+ mu2_imag = F.conv2d(y_imag, weight=kernel, stride=1, padding=0, groups=n_channels)
+
+ mu1_sq = mu1_real.pow(2) + mu1_imag.pow(2)
+ mu2_sq = mu2_real.pow(2) + mu2_imag.pow(2)
+ mu1_mu2_real = mu1_real * mu2_real - mu1_imag * mu2_imag
+ mu1_mu2_imag = mu1_real * mu2_imag + mu1_imag * mu2_real
+
+ compensation = 1.0
+
+ x_sq = x_real.pow(2) + x_imag.pow(2)
+ y_sq = y_real.pow(2) + y_imag.pow(2)
+ x_y_real = x_real * y_real - x_imag * y_imag
+ x_y_imag = x_real * y_imag + x_imag * y_real
+
+ sigma1_sq = F.conv2d(x_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_sq
+ sigma2_sq = F.conv2d(y_sq, weight=kernel, stride=1, padding=0, groups=n_channels) - mu2_sq
+ sigma12_real = F.conv2d(x_y_real, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_real
+ sigma12_imag = F.conv2d(x_y_imag, weight=kernel, stride=1, padding=0, groups=n_channels) - mu1_mu2_imag
+ sigma12 = torch.stack((sigma12_imag, sigma12_real), dim=-1)
+ mu1_mu2 = torch.stack((mu1_mu2_real, mu1_mu2_imag), dim=-1)
+ # Set alpha = beta = gamma = 1.
+ cs_map = (sigma12 * 2 + c2 * compensation) / (sigma1_sq.unsqueeze(-1) + sigma2_sq.unsqueeze(-1) + c2 * compensation)
+ ssim_map = (mu1_mu2 * 2 + c1 * compensation) / (mu1_sq.unsqueeze(-1) + mu2_sq.unsqueeze(-1) + c1 * compensation)
+ ssim_map = ssim_map * cs_map
+
+ ssim_val = ssim_map.mean(dim=(-2, -3))
+ cs = cs_map.mean(dim=(-2, -3))
+
+ return ssim_val, cs
+
+
+def gaussian_filter(
+ kernel_size: int, sigma: float, device: Optional[str] = None, dtype: Optional[type] = None
+) -> torch.Tensor:
+ r"""Returns 2D Gaussian kernel N(0,`sigma`^2)
+ Args:
+ kernel_size: Size of the kernel
+ sigma: Std of the distribution
+ device: target device for kernel generation
+ dtype: target data type for kernel generation
+ Returns:
+ gaussian_kernel: Tensor with shape (1, kernel_size, kernel_size)
+ """
+ coords = torch.arange(kernel_size, dtype=dtype, device=device)
+ coords -= (kernel_size - 1) / 2.0
+
+ g = coords**2
+ g = (-(g.unsqueeze(0) + g.unsqueeze(1)) / (2 * sigma**2)).exp()
+
+ g /= g.sum()
+ return g.unsqueeze(0)
diff --git a/Meissonic/vidtok_cache/VidTok/vidtwin/README.md b/Meissonic/vidtok_cache/VidTok/vidtwin/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..e475c6eade763f9cef3b67a905dd738e1fb43fef
--- /dev/null
+++ b/Meissonic/vidtok_cache/VidTok/vidtwin/README.md
@@ -0,0 +1,211 @@
+
+
+
+ VidTwin: Video VAE with Decoupled Structure and Dynamics (CVPR 2025)
+
+
+
+
+
+
+
+[](https://arxiv.org/pdf/2412.17726) [](https://huggingface.co/microsoft/vidtwin) [](https://vidtwin.github.io/)
+
+
🔥 Check our Demo Page for enhanced visual experience.
+
+
+
+
+
+
+
+
Error: Server Error
\nThe server encountered a temporary error and could not complete your request.
Please try again in 30 seconds.\n
\n\n"}
diff --git a/Meissonic/wandb/run-20251207_162442-54o4hegd/logs/debug.log b/Meissonic/wandb/run-20251207_162442-54o4hegd/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8040adbfb5270528b2bbf220b0ec72df8d564d3e
--- /dev/null
+++ b/Meissonic/wandb/run-20251207_162442-54o4hegd/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_setup.py:_flush():80] Configure stats pid to 1347178
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251207_162442-54o4hegd/logs/debug.log
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251207_162442-54o4hegd/logs/debug-internal.log
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_init.py:init():841] calling init triggers
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-07 16:24:42,828 INFO MainThread:1347178 [wandb_init.py:init():889] starting backend
+2025-12-07 16:24:43,081 INFO MainThread:1347178 [wandb_init.py:init():892] sending inform_init request
+2025-12-07 16:24:43,085 INFO MainThread:1347178 [wandb_init.py:init():900] backend started and connected
+2025-12-07 16:24:43,087 INFO MainThread:1347178 [wandb_init.py:init():970] updated telemetry
+2025-12-07 16:24:43,091 INFO MainThread:1347178 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-07 16:24:43,521 INFO MainThread:1347178 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-07 16:24:43,630 INFO MainThread:1347178 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-07 16:24:43,630 INFO MainThread:1347178 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-07 16:24:43,630 INFO MainThread:1347178 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-07 16:24:43,630 INFO MainThread:1347178 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-07 16:24:43,633 INFO MainThread:1347178 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-07 16:24:43,634 INFO MainThread:1347178 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-base', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 8, 'video_height': 64, 'video_width': 112, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
diff --git a/Meissonic/wandb/run-20251207_162442-54o4hegd/run-54o4hegd.wandb b/Meissonic/wandb/run-20251207_162442-54o4hegd/run-54o4hegd.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a9e9dd08f569f54d894a3397dc044c9a0471ca43
--- /dev/null
+++ b/Meissonic/wandb/run-20251207_162442-54o4hegd/run-54o4hegd.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce67eacd158fd96dbf7ba1b3fd79a7fb83ceb723f95f1b23c5bcd0e0c2b31a3d
+size 9732096
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/config.yaml b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c232b95c5476b6772c165af7150a97749ce793ff
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/config.yaml
@@ -0,0 +1,288 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 4vc7e5b6c9w1c2lhq8r0tju8hogo2vvk:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --num_frames
+ - "8"
+ - --video_height
+ - "64"
+ - --video_width
+ - "112"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-1.0-Tokenizer-DV8x16x16
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "16"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_64x112_8f_16bs
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11813091573760"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T03:17:49.487702Z"
+ writerId: 4vc7e5b6c9w1c2lhq8r0tju8hogo2vvk
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 8
+output_dir:
+ value: ./output_64x112_8f_16bs
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 64
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 112
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/output.log b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d3cf105e64766791897201ff8fdfb3ce18cd132d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/output.log
@@ -0,0 +1,12 @@
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1350, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 459, in main
+ raise ValueError(f"For video training, text_encoder_architecture must be 'umt5-base' or 't5', got '{args.text_encoder_architecture}'")
+ValueError: For video training, text_encoder_architecture must be 'umt5-base' or 't5', got 'umt5-xxl'
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1350, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 459, in main
+[rank0]: raise ValueError(f"For video training, text_encoder_architecture must be 'umt5-base' or 't5', got '{args.text_encoder_architecture}'")
+[rank0]: ValueError: For video training, text_encoder_architecture must be 'umt5-base' or 't5', got 'umt5-xxl'
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/requirements.txt b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..62bae269aa82a40cac99c983da40ce917b2086d0
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T03:17:49.487702Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "8",
+ "--video_height",
+ "64",
+ "--video_width",
+ "112",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-1.0-Tokenizer-DV8x16x16",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "16",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_64x112_8f_16bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11813091573760"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "4vc7e5b6c9w1c2lhq8r0tju8hogo2vvk"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-summary.json b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0a620d0c1047a4dd8a400939b6da246ed8063a7
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-core.log b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..cd6c468851bfc0657488be0e325a6d4d13db691b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-08T03:17:49.837623537Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpn5to09j1/port-3352340.txt","pid":3352340,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T03:17:49.838956393Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3352340}
+{"time":"2025-12-08T03:17:49.839564548Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3352340-3352619-1046294805/socket","Net":"unix"}}
+{"time":"2025-12-08T03:17:50.019531422Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T03:17:50.032275187Z","level":"INFO","msg":"handleInformInit: received","streamId":"b01lmzcy","id":"1(@)"}
+{"time":"2025-12-08T03:17:50.208038227Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"b01lmzcy","id":"1(@)"}
+{"time":"2025-12-08T03:17:50.568307111Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-08T03:17:50.568347689Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-08T03:17:50.56834814Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-08T03:17:50.568413111Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3352340-3352619-1046294805/socket","Net":"unix"}}
+{"time":"2025-12-08T03:17:50.568432059Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-08T03:17:51.117622137Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-08T03:17:51.117639621Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-08T03:17:51.117648381Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-internal.log b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..af1f6c46c08a2d3b327eddbbf062fee8badbce1f
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-08T03:17:50.032379945Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T03:17:50.207002094Z","level":"INFO","msg":"stream: created new stream","id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.208028973Z","level":"INFO","msg":"stream: started","id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.208052916Z","level":"INFO","msg":"writer: started","stream_id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.208514733Z","level":"INFO","msg":"handler: started","stream_id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.208703909Z","level":"INFO","msg":"sender: started","stream_id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.568373135Z","level":"INFO","msg":"stream: closing","id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:50.986975935Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-08T03:17:51.103719495Z","level":"INFO","msg":"handler: closed","stream_id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:51.103789916Z","level":"INFO","msg":"sender: closed","stream_id":"b01lmzcy"}
+{"time":"2025-12-08T03:17:51.103797706Z","level":"INFO","msg":"stream: closed","id":"b01lmzcy"}
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug.log b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..3f76c45d429dff618018dc6a9881ec8fad9e71d1
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_setup.py:_flush():80] Configure stats pid to 3352340
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug.log
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_031749-b01lmzcy/logs/debug-internal.log
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_init.py:init():841] calling init triggers
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 03:17:49,490 INFO MainThread:3352340 [wandb_init.py:init():889] starting backend
+2025-12-08 03:17:50,019 INFO MainThread:3352340 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 03:17:50,026 INFO MainThread:3352340 [wandb_init.py:init():900] backend started and connected
+2025-12-08 03:17:50,029 INFO MainThread:3352340 [wandb_init.py:init():970] updated telemetry
+2025-12-08 03:17:50,034 INFO MainThread:3352340 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 03:17:50,441 INFO MainThread:3352340 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 03:17:50,556 INFO MainThread:3352340 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 03:17:50,556 INFO MainThread:3352340 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 03:17:50,556 INFO MainThread:3352340 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 03:17:50,556 INFO MainThread:3352340 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 03:17:50,561 INFO MainThread:3352340 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 03:17:50,562 INFO MainThread:3352340 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_64x112_8f_16bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 8, 'video_height': 64, 'video_width': 112, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
+2025-12-08 03:17:50,568 INFO wandb-AsyncioManager-main:3352340 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-08 03:17:50,568 INFO wandb-AsyncioManager-main:3352340 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251208_031749-b01lmzcy/run-b01lmzcy.wandb b/Meissonic/wandb/run-20251208_031749-b01lmzcy/run-b01lmzcy.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..d17247516273d99bca5236bc89ab45ec1711ee08
Binary files /dev/null and b/Meissonic/wandb/run-20251208_031749-b01lmzcy/run-b01lmzcy.wandb differ
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/config.yaml b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7de70d2aebda0c0075e2ec276373c62293037fbd
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/config.yaml
@@ -0,0 +1,288 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ scyk41pdbji4tjive4amqhjyjga3x3fn:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --num_frames
+ - "8"
+ - --video_height
+ - "64"
+ - --video_width
+ - "112"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-1.0-Tokenizer-DV8x16x16
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "16"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_64x112_8f_16bs
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11813091741696"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T03:20:21.949323Z"
+ writerId: scyk41pdbji4tjive4amqhjyjga3x3fn
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 8
+output_dir:
+ value: ./output_64x112_8f_16bs
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 64
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 112
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/output.log b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..448748f1ff63cfb7139b46a593876fccff8c3d6f
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/output.log
@@ -0,0 +1,47 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 70.28it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4988.98it/s]
+12/08/2025 03:20:28 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=4, W'=7
+12/08/2025 03:20:28 - INFO - __main__ - Theoretical dimensions: F'=1, H'=4, W'=7
+12/08/2025 03:20:28 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:20:31 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 03:20:47 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:20:47 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1363, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 727, in main
+ missing_keys, unexpected_keys = model.backbone.load_state_dict(wan_state_dict, strict=False)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2609, in load_state_dict
+ load(self, state_dict)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+ load(child, child_state_dict, child_prefix) # noqa: F821
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+ load(child, child_state_dict, child_prefix) # noqa: F821
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+ load(child, child_state_dict, child_prefix) # noqa: F821
+ [Previous line repeated 1 more time]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2580, in load
+ module._load_from_state_dict(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2487, in _load_from_state_dict
+ param.copy_(input_param)
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1363, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 727, in main
+[rank0]: missing_keys, unexpected_keys = model.backbone.load_state_dict(wan_state_dict, strict=False)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2609, in load_state_dict
+[rank0]: load(self, state_dict)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+[rank0]: load(child, child_state_dict, child_prefix) # noqa: F821
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+[rank0]: load(child, child_state_dict, child_prefix) # noqa: F821
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2597, in load
+[rank0]: load(child, child_state_dict, child_prefix) # noqa: F821
+[rank0]: [Previous line repeated 1 more time]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2580, in load
+[rank0]: module._load_from_state_dict(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 2487, in _load_from_state_dict
+[rank0]: param.copy_(input_param)
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/requirements.txt b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c1b14e19694c5a80362d3f96aeacd62177b111b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T03:20:21.949323Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "8",
+ "--video_height",
+ "64",
+ "--video_width",
+ "112",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-1.0-Tokenizer-DV8x16x16",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "16",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_64x112_8f_16bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11813091741696"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "scyk41pdbji4tjive4amqhjyjga3x3fn"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-summary.json b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e4dda75148b11087b7ca4d383b6747633b42a57
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":37},"_runtime":37}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-core.log b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..469470d366a736000f6ce838dbc6e686d3f9a069
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-08T03:20:22.018532764Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmprej4fezv/port-3354358.txt","pid":3354358,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T03:20:22.019109108Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3354358}
+{"time":"2025-12-08T03:20:22.01910427Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3354358-3354621-2754597729/socket","Net":"unix"}}
+{"time":"2025-12-08T03:20:22.203005927Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T03:20:22.208970474Z","level":"INFO","msg":"handleInformInit: received","streamId":"kq3w5uiu","id":"1(@)"}
+{"time":"2025-12-08T03:20:22.374155661Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"kq3w5uiu","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.090286328Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.090341092Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-08T03:21:00.090330468Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.090427144Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3354358-3354621-2754597729/socket","Net":"unix"}}
+{"time":"2025-12-08T03:21:00.09046078Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.489451665Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.489471453Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-08T03:21:00.489481117Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-internal.log b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..723993b298ae26734c8c5dfa46d09b1f5911367b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-08T03:20:22.209111428Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T03:20:22.373964604Z","level":"INFO","msg":"stream: created new stream","id":"kq3w5uiu"}
+{"time":"2025-12-08T03:20:22.374049571Z","level":"INFO","msg":"handler: started","stream_id":"kq3w5uiu"}
+{"time":"2025-12-08T03:20:22.374147086Z","level":"INFO","msg":"stream: started","id":"kq3w5uiu"}
+{"time":"2025-12-08T03:20:22.374162519Z","level":"INFO","msg":"writer: started","stream_id":"kq3w5uiu"}
+{"time":"2025-12-08T03:20:22.374162263Z","level":"INFO","msg":"sender: started","stream_id":"kq3w5uiu"}
+{"time":"2025-12-08T03:21:00.090348177Z","level":"INFO","msg":"stream: closing","id":"kq3w5uiu"}
+{"time":"2025-12-08T03:21:00.366237455Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-08T03:21:00.48656867Z","level":"INFO","msg":"handler: closed","stream_id":"kq3w5uiu"}
+{"time":"2025-12-08T03:21:00.486659461Z","level":"INFO","msg":"sender: closed","stream_id":"kq3w5uiu"}
+{"time":"2025-12-08T03:21:00.486667719Z","level":"INFO","msg":"stream: closed","id":"kq3w5uiu"}
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug.log b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..c647f07915d9a9c5f822690eee2e20e8fdb8f287
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_setup.py:_flush():80] Configure stats pid to 3354358
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug.log
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_032021-kq3w5uiu/logs/debug-internal.log
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_init.py:init():841] calling init triggers
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 03:20:21,952 INFO MainThread:3354358 [wandb_init.py:init():889] starting backend
+2025-12-08 03:20:22,203 INFO MainThread:3354358 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 03:20:22,207 INFO MainThread:3354358 [wandb_init.py:init():900] backend started and connected
+2025-12-08 03:20:22,208 INFO MainThread:3354358 [wandb_init.py:init():970] updated telemetry
+2025-12-08 03:20:22,212 INFO MainThread:3354358 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 03:20:22,747 INFO MainThread:3354358 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 03:20:22,855 INFO MainThread:3354358 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 03:20:22,855 INFO MainThread:3354358 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 03:20:22,855 INFO MainThread:3354358 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 03:20:22,856 INFO MainThread:3354358 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 03:20:22,858 INFO MainThread:3354358 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 03:20:22,859 INFO MainThread:3354358 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_64x112_8f_16bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 8, 'video_height': 64, 'video_width': 112, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
+2025-12-08 03:21:00,090 INFO wandb-AsyncioManager-main:3354358 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-08 03:21:00,090 INFO wandb-AsyncioManager-main:3354358 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251208_032021-kq3w5uiu/run-kq3w5uiu.wandb b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/run-kq3w5uiu.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..4fac154706520195a8d0616cdfd6958d9c04cf19
Binary files /dev/null and b/Meissonic/wandb/run-20251208_032021-kq3w5uiu/run-kq3w5uiu.wandb differ
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/config.yaml b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d4c4ef26c685d02aa0488f7e22179cb720133c48
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/config.yaml
@@ -0,0 +1,288 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ xtt0igej7usiniueah5c5vplpej2u7p3:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --num_frames
+ - "9"
+ - --video_height
+ - "64"
+ - --video_width
+ - "112"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-1.0-Tokenizer-DV8x16x16
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "16"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_64x112_8f_16bs
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11813091844096"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T03:21:17.884575Z"
+ writerId: xtt0igej7usiniueah5c5vplpej2u7p3
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 9
+output_dir:
+ value: ./output_64x112_8f_16bs
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 64
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 112
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/output.log b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..cafb69ae962b54d49a6b619869fc63bb07ab52f6
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/output.log
@@ -0,0 +1,39 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 69.36it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5493.01it/s]
+12/08/2025 03:21:22 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=4, W'=7
+12/08/2025 03:21:22 - INFO - __main__ - Theoretical dimensions: F'=1, H'=4, W'=7
+12/08/2025 03:21:22 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:21:23 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 03:21:38 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:21:38 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:22:05 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 03:22:08 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 03:22:15 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 03:22:15 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 03:22:15 - INFO - __main__ - Dataloader configuration:
+12/08/2025 03:22:15 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 03:22:15 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 03:22:15 - INFO - __main__ - - persistent_workers: True
+12/08/2025 03:22:15 - INFO - __main__ - - pin_memory: True
+12/08/2025 03:22:15 - INFO - __main__ - Preparing model, optimizer and dataloaders
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1363, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1033, in main
+ text_encoder, tokenize_prompt(tokenizer, "", args.text_encoder_architecture).to(accelerator.device, non_blocking=True), args.text_encoder_architecture
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/train/dataset_utils.py", line 69, in tokenize_prompt
+ raise ValueError(f"Unknown text_encoder_architecture: {text_encoder_architecture}")
+ValueError: Unknown text_encoder_architecture: umt5-xxl
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1363, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1033, in main
+[rank0]: text_encoder, tokenize_prompt(tokenizer, "", args.text_encoder_architecture).to(accelerator.device, non_blocking=True), args.text_encoder_architecture
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/train/dataset_utils.py", line 69, in tokenize_prompt
+[rank0]: raise ValueError(f"Unknown text_encoder_architecture: {text_encoder_architecture}")
+[rank0]: ValueError: Unknown text_encoder_architecture: umt5-xxl
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/requirements.txt b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..13db8eae4ea4367e9ba279b2e0f2107f82212614
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T03:21:17.884575Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "9",
+ "--video_height",
+ "64",
+ "--video_width",
+ "112",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-1.0-Tokenizer-DV8x16x16",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "16",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_64x112_8f_16bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11813091844096"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "xtt0igej7usiniueah5c5vplpej2u7p3"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-summary.json b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..f216acddd6fec52473bb964018993e25ebe701aa
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":232},"_runtime":232}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-core.log b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..1949a2955f6458833dad3423485a006497865521
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-08T03:21:17.951376965Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp11o8j3tm/port-3355453.txt","pid":3355453,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T03:21:17.951948637Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3355453}
+{"time":"2025-12-08T03:21:17.951960045Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3355453-3355707-2981470890/socket","Net":"unix"}}
+{"time":"2025-12-08T03:21:18.137503071Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T03:21:18.143619116Z","level":"INFO","msg":"handleInformInit: received","streamId":"84ktbwyp","id":"1(@)"}
+{"time":"2025-12-08T03:21:18.319484041Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"84ktbwyp","id":"1(@)"}
+{"time":"2025-12-08T03:25:10.729670716Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-08T03:25:10.729715992Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-08T03:25:10.729712192Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-08T03:25:10.729766051Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-08T03:25:10.729807052Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3355453-3355707-2981470890/socket","Net":"unix"}}
+{"time":"2025-12-08T03:25:11.116771304Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-08T03:25:11.116796954Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-08T03:25:11.116808113Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-internal.log b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..4ca9987714929550356a175e5b0fc2da35af7ea8
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-08T03:21:18.143764029Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T03:21:18.31920589Z","level":"INFO","msg":"stream: created new stream","id":"84ktbwyp"}
+{"time":"2025-12-08T03:21:18.319349929Z","level":"INFO","msg":"handler: started","stream_id":"84ktbwyp"}
+{"time":"2025-12-08T03:21:18.319476317Z","level":"INFO","msg":"stream: started","id":"84ktbwyp"}
+{"time":"2025-12-08T03:21:18.319498724Z","level":"INFO","msg":"writer: started","stream_id":"84ktbwyp"}
+{"time":"2025-12-08T03:21:18.319499946Z","level":"INFO","msg":"sender: started","stream_id":"84ktbwyp"}
+{"time":"2025-12-08T03:25:10.729723902Z","level":"INFO","msg":"stream: closing","id":"84ktbwyp"}
+{"time":"2025-12-08T03:25:10.979653379Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-08T03:25:11.113672154Z","level":"INFO","msg":"handler: closed","stream_id":"84ktbwyp"}
+{"time":"2025-12-08T03:25:11.113797561Z","level":"INFO","msg":"sender: closed","stream_id":"84ktbwyp"}
+{"time":"2025-12-08T03:25:11.113814607Z","level":"INFO","msg":"stream: closed","id":"84ktbwyp"}
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug.log b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..79953f52c15dc1ccccc0fa0c586299b6452c0bd6
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_setup.py:_flush():80] Configure stats pid to 3355453
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug.log
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_032117-84ktbwyp/logs/debug-internal.log
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_init.py:init():841] calling init triggers
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 03:21:17,887 INFO MainThread:3355453 [wandb_init.py:init():889] starting backend
+2025-12-08 03:21:18,137 INFO MainThread:3355453 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 03:21:18,142 INFO MainThread:3355453 [wandb_init.py:init():900] backend started and connected
+2025-12-08 03:21:18,143 INFO MainThread:3355453 [wandb_init.py:init():970] updated telemetry
+2025-12-08 03:21:18,147 INFO MainThread:3355453 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 03:21:18,482 INFO MainThread:3355453 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 03:21:18,596 INFO MainThread:3355453 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 03:21:18,596 INFO MainThread:3355453 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 03:21:18,596 INFO MainThread:3355453 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 03:21:18,596 INFO MainThread:3355453 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 03:21:18,599 INFO MainThread:3355453 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 03:21:18,600 INFO MainThread:3355453 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_64x112_8f_16bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 9, 'video_height': 64, 'video_width': 112, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
+2025-12-08 03:25:10,729 INFO wandb-AsyncioManager-main:3355453 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-08 03:25:10,730 INFO wandb-AsyncioManager-main:3355453 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251208_032117-84ktbwyp/run-84ktbwyp.wandb b/Meissonic/wandb/run-20251208_032117-84ktbwyp/run-84ktbwyp.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a5a2ebb1a6e29d22fd7c4da5fc61b6344dbf184a
Binary files /dev/null and b/Meissonic/wandb/run-20251208_032117-84ktbwyp/run-84ktbwyp.wandb differ
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/output.log b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..6923566ba663a19ad342f1f28a6d676994754e10
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/output.log
@@ -0,0 +1,56 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 67.76it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 03:29:58 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 7052.64it/s]
+12/08/2025 03:30:00 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=4, W'=7
+12/08/2025 03:30:00 - INFO - __main__ - Theoretical dimensions: F'=1, H'=4, W'=7
+12/08/2025 03:30:00 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:30:00 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 03:30:17 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:30:17 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 03:30:19 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 03:30:22 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 03:30:29 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 03:30:29 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 03:30:29 - INFO - __main__ - Dataloader configuration:
+12/08/2025 03:30:29 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 03:30:29 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 03:30:29 - INFO - __main__ - - persistent_workers: True
+12/08/2025 03:30:29 - INFO - __main__ - - pin_memory: True
+12/08/2025 03:30:29 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 03:30:47 - INFO - __main__ - ***** Running training *****
+12/08/2025 03:30:47 - INFO - __main__ - Num training steps = 10000
+12/08/2025 03:30:47 - INFO - __main__ - Instantaneous batch size per device = 16
+12/08/2025 03:30:47 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 128
+12/08/2025 03:30:47 - INFO - __main__ - Gradient Accumulation steps = 1
+12/08/2025 03:33:30 - INFO - __main__ - Step: 10 Loss: 11.1033 LR: 0.000300
+12/08/2025 03:34:08 - INFO - __main__ - Step: 20 Loss: 11.1034 LR: 0.000300
+12/08/2025 03:35:23 - INFO - __main__ - Step: 30 Loss: 11.1034 LR: 0.000300
+12/08/2025 03:36:15 - INFO - __main__ - Step: 40 Loss: 11.0977 LR: 0.000300
+12/08/2025 03:37:58 - INFO - __main__ - Step: 50 Loss: 11.0971 LR: 0.000300
+12/08/2025 03:38:36 - INFO - __main__ - Step: 60 Loss: 11.0946 LR: 0.000300
+12/08/2025 03:39:35 - INFO - __main__ - Step: 70 Loss: 11.0899 LR: 0.000300
+12/08/2025 03:40:55 - INFO - __main__ - Step: 80 Loss: 11.0908 LR: 0.000300
+12/08/2025 03:42:24 - INFO - __main__ - Step: 90 Loss: 11.0784 LR: 0.000300
+12/08/2025 03:43:06 - INFO - __main__ - Step: 100 Loss: 11.0728 LR: 0.000300
+12/08/2025 03:44:02 - INFO - __main__ - Step: 110 Loss: 11.0524 LR: 0.000300
+12/08/2025 03:45:19 - INFO - __main__ - Step: 120 Loss: 11.0345 LR: 0.000300
+12/08/2025 03:46:17 - INFO - __main__ - Step: 130 Loss: 11.0061 LR: 0.000300
+12/08/2025 03:47:33 - INFO - __main__ - Step: 140 Loss: 10.9730 LR: 0.000300
+12/08/2025 03:48:25 - INFO - __main__ - Step: 150 Loss: 10.9452 LR: 0.000300
+12/08/2025 03:49:34 - INFO - __main__ - Step: 160 Loss: 10.9156 LR: 0.000300
+12/08/2025 03:50:30 - INFO - __main__ - Step: 170 Loss: 10.8766 LR: 0.000300
+12/08/2025 03:51:28 - INFO - __main__ - Step: 180 Loss: 10.8338 LR: 0.000300
+12/08/2025 03:52:38 - INFO - __main__ - Step: 190 Loss: 10.8179 LR: 0.000300
+12/08/2025 03:53:55 - INFO - __main__ - Step: 200 Loss: 10.7630 LR: 0.000300
+12/08/2025 03:54:34 - INFO - __main__ - Step: 210 Loss: 10.7580 LR: 0.000300
+12/08/2025 03:56:04 - INFO - __main__ - Step: 220 Loss: 10.7254 LR: 0.000300
+12/08/2025 03:56:56 - INFO - __main__ - Step: 230 Loss: 10.6992 LR: 0.000300
+12/08/2025 03:58:10 - INFO - __main__ - Step: 240 Loss: 10.6697 LR: 0.000300
+12/08/2025 03:59:41 - INFO - __main__ - Step: 250 Loss: 10.6544 LR: 0.000300
+12/08/2025 04:00:36 - INFO - __main__ - Step: 260 Loss: 10.6203 LR: 0.000300
+12/08/2025 04:01:27 - INFO - __main__ - Step: 270 Loss: 10.6068 LR: 0.000300
+12/08/2025 04:02:38 - INFO - __main__ - Step: 280 Loss: 10.5423 LR: 0.000300
+12/08/2025 04:03:58 - INFO - __main__ - Step: 290 Loss: 10.5799 LR: 0.000300
+12/08/2025 04:04:48 - INFO - __main__ - Step: 300 Loss: 10.5523 LR: 0.000300
+12/08/2025 04:05:32 - INFO - __main__ - Step: 310 Loss: 10.5698 LR: 0.000300
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/requirements.txt b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..5a0f6f92c8e4228d6514ec50900dac905b812ee6
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T03:29:55.147354Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "9",
+ "--video_height",
+ "64",
+ "--video_width",
+ "112",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-1.0-Tokenizer-DV8x16x16",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "16",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_64x112_8f_16bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11813092118528"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "9ewipho8y9ufx2kaipiagam4kqn0wsgz"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-core.log b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..501d0ae2da501a7a12c508ef69d9ae22202576e2
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-08T03:29:55.218073996Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpi3vrch57/port-3361354.txt","pid":3361354,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T03:29:55.218523005Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3361354}
+{"time":"2025-12-08T03:29:55.218497187Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3361354-3361609-1392137282/socket","Net":"unix"}}
+{"time":"2025-12-08T03:29:55.40139843Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T03:29:55.407669546Z","level":"INFO","msg":"handleInformInit: received","streamId":"tl61pd0t","id":"1(@)"}
+{"time":"2025-12-08T03:29:55.578988829Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tl61pd0t","id":"1(@)"}
+{"time":"2025-12-08T04:05:43.832486673Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-internal.log b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..df455bf5d3b8c5d6633bb8d6ca582dbddb7dd988
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-08T03:29:55.407753699Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T03:29:55.578794326Z","level":"INFO","msg":"stream: created new stream","id":"tl61pd0t"}
+{"time":"2025-12-08T03:29:55.578885522Z","level":"INFO","msg":"handler: started","stream_id":"tl61pd0t"}
+{"time":"2025-12-08T03:29:55.578981351Z","level":"INFO","msg":"stream: started","id":"tl61pd0t"}
+{"time":"2025-12-08T03:29:55.578999079Z","level":"INFO","msg":"writer: started","stream_id":"tl61pd0t"}
+{"time":"2025-12-08T03:29:55.578999792Z","level":"INFO","msg":"sender: started","stream_id":"tl61pd0t"}
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug.log b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..747d6fe5211bbb35f431fea0b7ae1cf5cce11de0
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-08 03:29:55,149 INFO MainThread:3361354 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 03:29:55,149 INFO MainThread:3361354 [wandb_setup.py:_flush():80] Configure stats pid to 3361354
+2025-12-08 03:29:55,149 INFO MainThread:3361354 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug.log
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_032955-tl61pd0t/logs/debug-internal.log
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_init.py:init():841] calling init triggers
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 03:29:55,150 INFO MainThread:3361354 [wandb_init.py:init():889] starting backend
+2025-12-08 03:29:55,401 INFO MainThread:3361354 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 03:29:55,406 INFO MainThread:3361354 [wandb_init.py:init():900] backend started and connected
+2025-12-08 03:29:55,408 INFO MainThread:3361354 [wandb_init.py:init():970] updated telemetry
+2025-12-08 03:29:55,412 INFO MainThread:3361354 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 03:29:55,879 INFO MainThread:3361354 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 03:29:55,988 INFO MainThread:3361354 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 03:29:55,988 INFO MainThread:3361354 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 03:29:55,988 INFO MainThread:3361354 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 03:29:55,988 INFO MainThread:3361354 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 03:29:55,991 INFO MainThread:3361354 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 03:29:55,991 INFO MainThread:3361354 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_64x112_8f_16bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 9, 'video_height': 64, 'video_width': 112, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
diff --git a/Meissonic/wandb/run-20251208_032955-tl61pd0t/run-tl61pd0t.wandb b/Meissonic/wandb/run-20251208_032955-tl61pd0t/run-tl61pd0t.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..e0d555864914635b85a37110a9c99bfadac4cee8
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_032955-tl61pd0t/run-tl61pd0t.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e6df6c74d58c6d4c24a7aa545f9c7c8f5fe65622e55dfd93543de7a3613fbe2
+size 524288
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/output.log b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..8928b902d0409bf937037cf6a41de48ce630f38e
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/output.log
@@ -0,0 +1,239 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 65.76it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 04:06:09 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5144.58it/s]
+12/08/2025 04:06:11 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=16, W'=28
+12/08/2025 04:06:11 - INFO - __main__ - Theoretical dimensions: F'=1, H'=16, W'=28
+12/08/2025 04:06:11 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 04:06:11 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 04:06:28 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 04:06:28 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 04:06:30 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 04:06:32 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 04:06:39 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 04:06:39 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 04:06:39 - INFO - __main__ - Dataloader configuration:
+12/08/2025 04:06:39 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 04:06:39 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 04:06:39 - INFO - __main__ - - persistent_workers: True
+12/08/2025 04:06:39 - INFO - __main__ - - pin_memory: True
+12/08/2025 04:06:39 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 04:06:55 - INFO - __main__ - ***** Running training *****
+12/08/2025 04:06:55 - INFO - __main__ - Num training steps = 10000
+12/08/2025 04:06:55 - INFO - __main__ - Instantaneous batch size per device = 4
+12/08/2025 04:06:55 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 32
+12/08/2025 04:06:55 - INFO - __main__ - Gradient Accumulation steps = 1
+12/08/2025 04:07:29 - INFO - __main__ - Step: 10 Loss: 11.0996 LR: 0.000300
+12/08/2025 04:07:43 - INFO - __main__ - Step: 20 Loss: 11.0992 LR: 0.000300
+12/08/2025 04:07:56 - INFO - __main__ - Step: 30 Loss: 11.0959 LR: 0.000300
+12/08/2025 04:08:11 - INFO - __main__ - Step: 40 Loss: 11.0928 LR: 0.000300
+12/08/2025 04:08:25 - INFO - __main__ - Step: 50 Loss: 11.0939 LR: 0.000300
+12/08/2025 04:08:38 - INFO - __main__ - Step: 60 Loss: 11.0928 LR: 0.000300
+12/08/2025 04:08:53 - INFO - __main__ - Step: 70 Loss: 11.0855 LR: 0.000300
+12/08/2025 04:09:08 - INFO - __main__ - Step: 80 Loss: 11.0847 LR: 0.000300
+12/08/2025 04:09:23 - INFO - __main__ - Step: 90 Loss: 11.0807 LR: 0.000300
+12/08/2025 04:09:37 - INFO - __main__ - Step: 100 Loss: 11.0708 LR: 0.000300
+12/08/2025 04:09:51 - INFO - __main__ - Step: 110 Loss: 11.0565 LR: 0.000300
+12/08/2025 04:10:06 - INFO - __main__ - Step: 120 Loss: 11.0333 LR: 0.000300
+12/08/2025 04:10:20 - INFO - __main__ - Step: 130 Loss: 11.0024 LR: 0.000300
+12/08/2025 04:10:35 - INFO - __main__ - Step: 140 Loss: 10.9827 LR: 0.000300
+12/08/2025 04:10:49 - INFO - __main__ - Step: 150 Loss: 10.9469 LR: 0.000300
+12/08/2025 04:11:04 - INFO - __main__ - Step: 160 Loss: 10.9046 LR: 0.000300
+12/08/2025 04:11:18 - INFO - __main__ - Step: 170 Loss: 10.8506 LR: 0.000300
+12/08/2025 04:11:33 - INFO - __main__ - Step: 180 Loss: 10.8237 LR: 0.000300
+12/08/2025 04:11:47 - INFO - __main__ - Step: 190 Loss: 10.8008 LR: 0.000300
+12/08/2025 04:12:02 - INFO - __main__ - Step: 200 Loss: 10.7369 LR: 0.000300
+12/08/2025 04:12:17 - INFO - __main__ - Step: 210 Loss: 10.7677 LR: 0.000300
+12/08/2025 04:12:32 - INFO - __main__ - Step: 220 Loss: 10.7119 LR: 0.000300
+12/08/2025 04:12:47 - INFO - __main__ - Step: 230 Loss: 10.6432 LR: 0.000300
+12/08/2025 04:13:01 - INFO - __main__ - Step: 240 Loss: 10.6370 LR: 0.000300
+12/08/2025 04:13:15 - INFO - __main__ - Step: 250 Loss: 10.6359 LR: 0.000300
+12/08/2025 04:13:29 - INFO - __main__ - Step: 260 Loss: 10.5410 LR: 0.000300
+12/08/2025 04:13:43 - INFO - __main__ - Step: 270 Loss: 10.5677 LR: 0.000300
+12/08/2025 04:13:58 - INFO - __main__ - Step: 280 Loss: 10.5715 LR: 0.000300
+12/08/2025 04:14:13 - INFO - __main__ - Step: 290 Loss: 10.4722 LR: 0.000300
+12/08/2025 04:14:29 - INFO - __main__ - Step: 300 Loss: 10.5197 LR: 0.000300
+12/08/2025 04:14:43 - INFO - __main__ - Step: 310 Loss: 10.4737 LR: 0.000300
+12/08/2025 04:14:58 - INFO - __main__ - Step: 320 Loss: 10.4552 LR: 0.000300
+12/08/2025 04:15:13 - INFO - __main__ - Step: 330 Loss: 10.5440 LR: 0.000300
+12/08/2025 04:15:28 - INFO - __main__ - Step: 340 Loss: 10.4659 LR: 0.000300
+12/08/2025 04:15:44 - INFO - __main__ - Step: 350 Loss: 10.4662 LR: 0.000300
+12/08/2025 04:15:59 - INFO - __main__ - Step: 360 Loss: 10.4312 LR: 0.000300
+12/08/2025 04:16:14 - INFO - __main__ - Step: 370 Loss: 10.4508 LR: 0.000300
+12/08/2025 04:16:28 - INFO - __main__ - Step: 380 Loss: 10.4507 LR: 0.000300
+12/08/2025 04:16:43 - INFO - __main__ - Step: 390 Loss: 10.4685 LR: 0.000300
+12/08/2025 04:16:59 - INFO - __main__ - Step: 400 Loss: 10.3776 LR: 0.000300
+12/08/2025 04:17:14 - INFO - __main__ - Step: 410 Loss: 10.4464 LR: 0.000300
+12/08/2025 04:17:28 - INFO - __main__ - Step: 420 Loss: 10.3895 LR: 0.000300
+12/08/2025 04:17:44 - INFO - __main__ - Step: 430 Loss: 10.3926 LR: 0.000300
+12/08/2025 04:17:58 - INFO - __main__ - Step: 440 Loss: 10.4312 LR: 0.000300
+12/08/2025 04:18:14 - INFO - __main__ - Step: 450 Loss: 10.3412 LR: 0.000300
+12/08/2025 04:18:29 - INFO - __main__ - Step: 460 Loss: 10.2831 LR: 0.000300
+12/08/2025 04:18:44 - INFO - __main__ - Step: 470 Loss: 10.2265 LR: 0.000300
+12/08/2025 04:18:59 - INFO - __main__ - Step: 480 Loss: 10.3602 LR: 0.000300
+12/08/2025 04:19:15 - INFO - __main__ - Step: 490 Loss: 10.3724 LR: 0.000300
+12/08/2025 04:19:30 - INFO - __main__ - Step: 500 Loss: 10.3895 LR: 0.000300
+12/08/2025 04:19:30 - INFO - accelerate.accelerator - Saving current state to output_256x448_9f_4bs/checkpoint-500
+12/08/2025 04:19:41 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_9f_4bs/checkpoint-500/optimizer.bin
+12/08/2025 04:19:41 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_9f_4bs/checkpoint-500/scheduler.bin
+12/08/2025 04:19:41 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_9f_4bs/checkpoint-500/sampler.bin
+12/08/2025 04:19:41 - INFO - accelerate.checkpointing - Random states saved in output_256x448_9f_4bs/checkpoint-500/random_states_0.pkl
+12/08/2025 04:19:41 - INFO - __main__ - Saved state to output_256x448_9f_4bs/checkpoint-500
+12/08/2025 04:19:41 - INFO - __main__ - Generating videos for validation...
+12/08/2025 04:19:41 - INFO - __main__ - Generating videos for validation...
+12/08/2025 04:19:41 - ERROR - __main__ - Video validation failed: num_frames (9) must be divisible by temporal downsampling factor (8)
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1278, in main
+ videos = pipe(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/src/pipeline_video.py", line 541, in __call__
+ raise ValueError(
+ValueError: num_frames (9) must be divisible by temporal downsampling factor (8)
+12/08/2025 04:19:57 - INFO - __main__ - Step: 510 Loss: 10.3469 LR: 0.000300
+12/08/2025 04:20:12 - INFO - __main__ - Step: 520 Loss: 10.3415 LR: 0.000300
+12/08/2025 04:20:27 - INFO - __main__ - Step: 530 Loss: 10.4008 LR: 0.000300
+12/08/2025 04:20:42 - INFO - __main__ - Step: 540 Loss: 10.2568 LR: 0.000300
+12/08/2025 04:20:58 - INFO - __main__ - Step: 550 Loss: 10.3286 LR: 0.000300
+12/08/2025 04:21:13 - INFO - __main__ - Step: 560 Loss: 10.3954 LR: 0.000300
+12/08/2025 04:21:28 - INFO - __main__ - Step: 570 Loss: 10.3188 LR: 0.000300
+12/08/2025 04:21:42 - INFO - __main__ - Step: 580 Loss: 10.3820 LR: 0.000300
+12/08/2025 04:21:58 - INFO - __main__ - Step: 590 Loss: 10.3850 LR: 0.000300
+12/08/2025 04:22:12 - INFO - __main__ - Step: 600 Loss: 10.3458 LR: 0.000300
+12/08/2025 04:22:26 - INFO - __main__ - Step: 610 Loss: 10.3710 LR: 0.000300
+12/08/2025 04:22:41 - INFO - __main__ - Step: 620 Loss: 10.3231 LR: 0.000300
+12/08/2025 04:22:56 - INFO - __main__ - Step: 630 Loss: 10.3351 LR: 0.000300
+12/08/2025 04:23:10 - INFO - __main__ - Step: 640 Loss: 10.3568 LR: 0.000300
+12/08/2025 04:23:25 - INFO - __main__ - Step: 650 Loss: 10.2831 LR: 0.000300
+12/08/2025 04:23:39 - INFO - __main__ - Step: 660 Loss: 10.2838 LR: 0.000300
+12/08/2025 04:23:54 - INFO - __main__ - Step: 670 Loss: 10.2400 LR: 0.000300
+12/08/2025 04:24:08 - INFO - __main__ - Step: 680 Loss: 10.3153 LR: 0.000300
+12/08/2025 04:24:23 - INFO - __main__ - Step: 690 Loss: 10.3624 LR: 0.000300
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1171, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 359, in forward
+ k=rope_apply(k, grid_sizes, freqs),
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 251, in rope_apply
+ x_i = torch.view_as_real(x_i * freqs_i).flatten(2)
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1171, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 359, in forward
+[rank0]: k=rope_apply(k, grid_sizes, freqs),
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 251, in rope_apply
+[rank0]: x_i = torch.view_as_real(x_i * freqs_i).flatten(2)
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/requirements.txt b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..734c2a1d156069567b4cb1d81281a51800cb52dc
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T04:06:06.027369Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "9",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-1.0-Tokenizer-DV8x16x16",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "4",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_9f_4bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11813092904960"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "ye6zoy1wxwaa4ivuuvr0hgyyqpzf7o03"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-core.log b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..9b7270216a8559815b2a9ea42a7e7dc038786a44
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-08T04:06:06.095985831Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpw7aanfnn/port-290084.txt","pid":290084,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T04:06:06.096562812Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":290084}
+{"time":"2025-12-08T04:06:06.096546963Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-290084-290343-112641643/socket","Net":"unix"}}
+{"time":"2025-12-08T04:06:06.28262354Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T04:06:06.288971188Z","level":"INFO","msg":"handleInformInit: received","streamId":"2dcjc9k8","id":"1(@)"}
+{"time":"2025-12-08T04:06:06.457234565Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2dcjc9k8","id":"1(@)"}
+{"time":"2025-12-08T04:24:37.707615163Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-internal.log b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..e4e51aed8f34e9667f6424ba1e0ebd28921cfbc9
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-08T04:06:06.289151581Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T04:06:06.457022976Z","level":"INFO","msg":"stream: created new stream","id":"2dcjc9k8"}
+{"time":"2025-12-08T04:06:06.457113741Z","level":"INFO","msg":"handler: started","stream_id":"2dcjc9k8"}
+{"time":"2025-12-08T04:06:06.457227301Z","level":"INFO","msg":"stream: started","id":"2dcjc9k8"}
+{"time":"2025-12-08T04:06:06.457255259Z","level":"INFO","msg":"sender: started","stream_id":"2dcjc9k8"}
+{"time":"2025-12-08T04:06:06.457255856Z","level":"INFO","msg":"writer: started","stream_id":"2dcjc9k8"}
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug.log b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..adc7737841c2a8b057f943ffb2cfa61188b97667
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_setup.py:_flush():80] Configure stats pid to 290084
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug.log
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_040606-2dcjc9k8/logs/debug-internal.log
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_init.py:init():841] calling init triggers
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 04:06:06,030 INFO MainThread:290084 [wandb_init.py:init():889] starting backend
+2025-12-08 04:06:06,282 INFO MainThread:290084 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 04:06:06,287 INFO MainThread:290084 [wandb_init.py:init():900] backend started and connected
+2025-12-08 04:06:06,289 INFO MainThread:290084 [wandb_init.py:init():970] updated telemetry
+2025-12-08 04:06:06,294 INFO MainThread:290084 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 04:06:06,688 INFO MainThread:290084 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 04:06:06,801 INFO MainThread:290084 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 04:06:06,801 INFO MainThread:290084 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 04:06:06,801 INFO MainThread:290084 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 04:06:06,801 INFO MainThread:290084 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 04:06:06,804 INFO MainThread:290084 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 04:06:06,805 INFO MainThread:290084 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_9f_4bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 4, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 9, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
diff --git a/Meissonic/wandb/run-20251208_040606-2dcjc9k8/run-2dcjc9k8.wandb b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/run-2dcjc9k8.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..28c08b9445ef1c25f618a4d59c79deda77792616
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_040606-2dcjc9k8/run-2dcjc9k8.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7495b8eff0e7f0c259b20ef00d19656f0b2b1fc4e51d4cd5b0b128b6a8c336a1
+size 294912
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/config.yaml b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..72dcfe99ffb21617e9a63ee8bd61d04914b4f296
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/config.yaml
@@ -0,0 +1,288 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ i7ve4wwwnqexgpibqdpn8euqx2nyjccr:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --num_frames
+ - "8"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "4"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_9f_4bs
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11819720794112"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T06:24:56.224488Z"
+ writerId: i7ve4wwwnqexgpibqdpn8euqx2nyjccr
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 8
+output_dir:
+ value: ./output_256x448_9f_4bs
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 4
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/output.log b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..132972af43972ad88252665daaffd30c24756331
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/output.log
@@ -0,0 +1,53 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 69.55it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 06:24:59 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4788.02it/s]
+12/08/2025 06:25:01 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=3, H'=32, W'=56
+12/08/2025 06:25:01 - INFO - __main__ - Theoretical dimensions: F'=1, H'=16, W'=28
+12/08/2025 06:25:01 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:25:01 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 06:25:16 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:25:16 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:25:18 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 06:25:20 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 06:25:27 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 06:25:27 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 06:25:27 - INFO - __main__ - Dataloader configuration:
+12/08/2025 06:25:27 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 06:25:27 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 06:25:27 - INFO - __main__ - - persistent_workers: True
+12/08/2025 06:25:27 - INFO - __main__ - - pin_memory: True
+12/08/2025 06:25:27 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 06:25:45 - INFO - __main__ - ***** Running training *****
+12/08/2025 06:25:45 - INFO - __main__ - Num training steps = 10000
+12/08/2025 06:25:45 - INFO - __main__ - Instantaneous batch size per device = 4
+12/08/2025 06:25:45 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 32
+12/08/2025 06:25:45 - INFO - __main__ - Gradient Accumulation steps = 1
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1207, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 5.25 GiB. GPU 0 has a total capacity of 39.49 GiB of which 3.25 GiB is free. Process 988220 has 414.00 MiB memory in use. Process 988225 has 414.00 MiB memory in use. Process 988223 has 414.00 MiB memory in use. Process 988224 has 414.00 MiB memory in use. Process 988219 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 33.35 GiB memory in use. Process 988222 has 414.00 MiB memory in use. Process 988221 has 414.00 MiB memory in use. Of the allocated memory 31.91 GiB is allocated by PyTorch, and 349.89 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1207, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 5.25 GiB. GPU 0 has a total capacity of 39.49 GiB of which 3.25 GiB is free. Process 988220 has 414.00 MiB memory in use. Process 988225 has 414.00 MiB memory in use. Process 988223 has 414.00 MiB memory in use. Process 988224 has 414.00 MiB memory in use. Process 988219 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 33.35 GiB memory in use. Process 988222 has 414.00 MiB memory in use. Process 988221 has 414.00 MiB memory in use. Of the allocated memory 31.91 GiB is allocated by PyTorch, and 349.89 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/requirements.txt b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..25bb324183c438189f4c3ed7f5e4731d6cc736b4
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T06:24:56.224488Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "8",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "4",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_9f_4bs",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11819720794112"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "i7ve4wwwnqexgpibqdpn8euqx2nyjccr"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-summary.json b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6a58e23e676a0a08733884f4e00d6ed642aea39
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":112},"_runtime":112}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-core.log b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..9bac9286da012787fc6603ca767e04e16f7fc582
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-08T06:24:56.295049346Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp481wqo3r/port-988218.txt","pid":988218,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T06:24:56.295513671Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":988218}
+{"time":"2025-12-08T06:24:56.29552614Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-988218-988475-3388223444/socket","Net":"unix"}}
+{"time":"2025-12-08T06:24:56.480225837Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T06:24:56.48670425Z","level":"INFO","msg":"handleInformInit: received","streamId":"qz69oyz3","id":"1(@)"}
+{"time":"2025-12-08T06:24:56.652693188Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qz69oyz3","id":"1(@)"}
+{"time":"2025-12-08T06:26:48.94969808Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-08T06:26:48.949777226Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-08T06:26:48.949758796Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-08T06:26:48.949872901Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-08T06:26:48.949881001Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-988218-988475-3388223444/socket","Net":"unix"}}
+{"time":"2025-12-08T06:26:49.346344105Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-08T06:26:49.346375364Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-08T06:26:49.346388259Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-internal.log b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..3346f6c5ea0b042df17cc359bec3a89f2c99258e
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-08T06:24:56.486803829Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T06:24:56.652468229Z","level":"INFO","msg":"stream: created new stream","id":"qz69oyz3"}
+{"time":"2025-12-08T06:24:56.652567387Z","level":"INFO","msg":"handler: started","stream_id":"qz69oyz3"}
+{"time":"2025-12-08T06:24:56.652684577Z","level":"INFO","msg":"stream: started","id":"qz69oyz3"}
+{"time":"2025-12-08T06:24:56.65272407Z","level":"INFO","msg":"writer: started","stream_id":"qz69oyz3"}
+{"time":"2025-12-08T06:24:56.652727637Z","level":"INFO","msg":"sender: started","stream_id":"qz69oyz3"}
+{"time":"2025-12-08T06:26:48.949782152Z","level":"INFO","msg":"stream: closing","id":"qz69oyz3"}
+{"time":"2025-12-08T06:26:49.210637319Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-08T06:26:49.342287919Z","level":"INFO","msg":"handler: closed","stream_id":"qz69oyz3"}
+{"time":"2025-12-08T06:26:49.342363249Z","level":"INFO","msg":"sender: closed","stream_id":"qz69oyz3"}
+{"time":"2025-12-08T06:26:49.342374552Z","level":"INFO","msg":"stream: closed","id":"qz69oyz3"}
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug.log b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..0964d593925c1ffa8b437f17424ea8615fe235f4
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_setup.py:_flush():80] Configure stats pid to 988218
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug.log
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_062456-qz69oyz3/logs/debug-internal.log
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_init.py:init():841] calling init triggers
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 06:24:56,227 INFO MainThread:988218 [wandb_init.py:init():889] starting backend
+2025-12-08 06:24:56,480 INFO MainThread:988218 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 06:24:56,485 INFO MainThread:988218 [wandb_init.py:init():900] backend started and connected
+2025-12-08 06:24:56,486 INFO MainThread:988218 [wandb_init.py:init():970] updated telemetry
+2025-12-08 06:24:56,491 INFO MainThread:988218 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 06:24:56,892 INFO MainThread:988218 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 06:24:57,001 INFO MainThread:988218 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 06:24:57,001 INFO MainThread:988218 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 06:24:57,002 INFO MainThread:988218 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 06:24:57,002 INFO MainThread:988218 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 06:24:57,005 INFO MainThread:988218 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 06:24:57,006 INFO MainThread:988218 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_9f_4bs', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 4, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 8, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
+2025-12-08 06:26:48,949 INFO wandb-AsyncioManager-main:988218 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-08 06:26:48,949 INFO wandb-AsyncioManager-main:988218 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251208_062456-qz69oyz3/run-qz69oyz3.wandb b/Meissonic/wandb/run-20251208_062456-qz69oyz3/run-qz69oyz3.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..4e7dddf4b7353ea315af6e9868878bc05f583cdf
Binary files /dev/null and b/Meissonic/wandb/run-20251208_062456-qz69oyz3/run-qz69oyz3.wandb differ
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/files/output.log b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..f2670a600f41fa9a0b88eedef686a54cf3e84879
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/output.log
@@ -0,0 +1,167 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 67.43it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 06:27:44 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5715.42it/s]
+12/08/2025 06:27:46 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=4, W'=7
+12/08/2025 06:27:46 - INFO - __main__ - Theoretical dimensions: F'=0, H'=2, W'=3
+12/08/2025 06:27:46 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:27:46 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 06:28:03 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:28:03 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 06:28:05 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 06:28:07 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 06:28:14 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 06:28:14 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 06:28:14 - INFO - __main__ - Dataloader configuration:
+12/08/2025 06:28:14 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 06:28:14 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 06:28:14 - INFO - __main__ - - persistent_workers: True
+12/08/2025 06:28:14 - INFO - __main__ - - pin_memory: True
+12/08/2025 06:28:14 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 06:28:30 - INFO - __main__ - ***** Running training *****
+12/08/2025 06:28:30 - INFO - __main__ - Num training steps = 10000
+12/08/2025 06:28:30 - INFO - __main__ - Instantaneous batch size per device = 8
+12/08/2025 06:28:30 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/08/2025 06:28:30 - INFO - __main__ - Gradient Accumulation steps = 1
+12/08/2025 06:29:11 - INFO - __main__ - Step: 10 Loss: 11.0969 LR: 0.000300
+12/08/2025 06:29:32 - INFO - __main__ - Step: 20 Loss: 11.0993 LR: 0.000300
+12/08/2025 06:29:52 - INFO - __main__ - Step: 30 Loss: 11.1012 LR: 0.000300
+12/08/2025 06:30:13 - INFO - __main__ - Step: 40 Loss: 11.0992 LR: 0.000300
+12/08/2025 06:30:34 - INFO - __main__ - Step: 50 Loss: 11.1045 LR: 0.000300
+12/08/2025 06:30:56 - INFO - __main__ - Step: 60 Loss: 11.0981 LR: 0.000300
+12/08/2025 06:31:16 - INFO - __main__ - Step: 70 Loss: 11.0999 LR: 0.000300
+12/08/2025 06:31:37 - INFO - __main__ - Step: 80 Loss: 11.0901 LR: 0.000300
+12/08/2025 06:31:59 - INFO - __main__ - Step: 90 Loss: 11.0786 LR: 0.000300
+12/08/2025 06:32:20 - INFO - __main__ - Step: 100 Loss: 11.0815 LR: 0.000300
+12/08/2025 06:32:41 - INFO - __main__ - Step: 110 Loss: 11.0670 LR: 0.000300
+12/08/2025 06:33:02 - INFO - __main__ - Step: 120 Loss: 11.0464 LR: 0.000300
+12/08/2025 06:33:23 - INFO - __main__ - Step: 130 Loss: 11.0319 LR: 0.000300
+12/08/2025 06:33:45 - INFO - __main__ - Step: 140 Loss: 11.0280 LR: 0.000300
+12/08/2025 06:34:06 - INFO - __main__ - Step: 150 Loss: 11.0030 LR: 0.000300
+12/08/2025 06:34:27 - INFO - __main__ - Step: 160 Loss: 10.9769 LR: 0.000300
+12/08/2025 06:34:49 - INFO - __main__ - Step: 170 Loss: 10.9461 LR: 0.000300
+12/08/2025 06:35:10 - INFO - __main__ - Step: 180 Loss: 10.9273 LR: 0.000300
+12/08/2025 06:35:31 - INFO - __main__ - Step: 190 Loss: 10.9210 LR: 0.000300
+12/08/2025 06:35:53 - INFO - __main__ - Step: 200 Loss: 10.8968 LR: 0.000300
+12/08/2025 06:36:15 - INFO - __main__ - Step: 210 Loss: 10.8888 LR: 0.000300
+12/08/2025 06:36:37 - INFO - __main__ - Step: 220 Loss: 10.8603 LR: 0.000300
+12/08/2025 06:36:58 - INFO - __main__ - Step: 230 Loss: 10.8578 LR: 0.000300
+12/08/2025 06:37:20 - INFO - __main__ - Step: 240 Loss: 10.8413 LR: 0.000300
+12/08/2025 06:37:42 - INFO - __main__ - Step: 250 Loss: 10.8050 LR: 0.000300
+12/08/2025 06:38:04 - INFO - __main__ - Step: 260 Loss: 10.8143 LR: 0.000300
+12/08/2025 06:38:25 - INFO - __main__ - Step: 270 Loss: 10.7800 LR: 0.000300
+12/08/2025 06:38:47 - INFO - __main__ - Step: 280 Loss: 10.7418 LR: 0.000300
+12/08/2025 06:39:08 - INFO - __main__ - Step: 290 Loss: 10.7516 LR: 0.000300
+12/08/2025 06:39:31 - INFO - __main__ - Step: 300 Loss: 10.7369 LR: 0.000300
+12/08/2025 06:39:52 - INFO - __main__ - Step: 310 Loss: 10.7122 LR: 0.000300
+12/08/2025 06:40:14 - INFO - __main__ - Step: 320 Loss: 10.7289 LR: 0.000300
+12/08/2025 06:40:36 - INFO - __main__ - Step: 330 Loss: 10.7455 LR: 0.000300
+12/08/2025 06:40:58 - INFO - __main__ - Step: 340 Loss: 10.7406 LR: 0.000300
+12/08/2025 06:41:20 - INFO - __main__ - Step: 350 Loss: 10.6698 LR: 0.000300
+12/08/2025 06:41:42 - INFO - __main__ - Step: 360 Loss: 10.6939 LR: 0.000300
+12/08/2025 06:42:04 - INFO - __main__ - Step: 370 Loss: 10.6785 LR: 0.000300
+12/08/2025 06:42:27 - INFO - __main__ - Step: 380 Loss: 10.6714 LR: 0.000300
+12/08/2025 06:42:48 - INFO - __main__ - Step: 390 Loss: 10.6647 LR: 0.000300
+12/08/2025 06:43:10 - INFO - __main__ - Step: 400 Loss: 10.6492 LR: 0.000300
+12/08/2025 06:43:32 - INFO - __main__ - Step: 410 Loss: 10.6488 LR: 0.000300
+12/08/2025 06:43:53 - INFO - __main__ - Step: 420 Loss: 10.6739 LR: 0.000300
+12/08/2025 06:44:15 - INFO - __main__ - Step: 430 Loss: 10.6344 LR: 0.000300
+12/08/2025 06:44:37 - INFO - __main__ - Step: 440 Loss: 10.6503 LR: 0.000300
+12/08/2025 06:44:59 - INFO - __main__ - Step: 450 Loss: 10.6358 LR: 0.000300
+12/08/2025 06:45:22 - INFO - __main__ - Step: 460 Loss: 10.6386 LR: 0.000300
+12/08/2025 06:45:44 - INFO - __main__ - Step: 470 Loss: 10.6178 LR: 0.000300
+12/08/2025 06:46:06 - INFO - __main__ - Step: 480 Loss: 10.6220 LR: 0.000300
+12/08/2025 06:46:28 - INFO - __main__ - Step: 490 Loss: 10.6692 LR: 0.000300
+12/08/2025 06:46:50 - INFO - __main__ - Step: 500 Loss: 10.6238 LR: 0.000300
+12/08/2025 06:46:50 - INFO - accelerate.accelerator - Saving current state to output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500
+12/08/2025 06:47:00 - INFO - accelerate.checkpointing - Optimizer state saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500/optimizer.bin
+12/08/2025 06:47:00 - INFO - accelerate.checkpointing - Scheduler state saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500/scheduler.bin
+12/08/2025 06:47:00 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500/sampler.bin
+12/08/2025 06:47:00 - INFO - accelerate.checkpointing - Random states saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500/random_states_0.pkl
+12/08/2025 06:47:00 - INFO - __main__ - Saved state to output_32x56_4f_8bs_4*8*8vqvae/checkpoint-500
+12/08/2025 06:47:00 - INFO - __main__ - Generating videos for validation...
+12/08/2025 06:47:00 - INFO - __main__ - Generating videos for validation...
+12/08/2025 06:47:00 - ERROR - __main__ - Video validation failed: num_frames (4) must be divisible by temporal downsampling factor (8)
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1278, in main
+ videos = pipe(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/src/pipeline_video.py", line 541, in __call__
+ raise ValueError(
+ValueError: num_frames (4) must be divisible by temporal downsampling factor (8)
+12/08/2025 06:47:22 - INFO - __main__ - Step: 510 Loss: 10.5937 LR: 0.000300
+12/08/2025 06:47:43 - INFO - __main__ - Step: 520 Loss: 10.5638 LR: 0.000300
+12/08/2025 06:48:06 - INFO - __main__ - Step: 530 Loss: 10.6022 LR: 0.000300
+12/08/2025 06:48:28 - INFO - __main__ - Step: 540 Loss: 10.6146 LR: 0.000300
+12/08/2025 06:48:50 - INFO - __main__ - Step: 550 Loss: 10.6357 LR: 0.000300
+12/08/2025 06:49:12 - INFO - __main__ - Step: 560 Loss: 10.5949 LR: 0.000300
+12/08/2025 06:49:35 - INFO - __main__ - Step: 570 Loss: 10.6044 LR: 0.000300
+12/08/2025 06:49:57 - INFO - __main__ - Step: 580 Loss: 10.6040 LR: 0.000300
+12/08/2025 06:50:19 - INFO - __main__ - Step: 590 Loss: 10.5649 LR: 0.000300
+12/08/2025 06:50:41 - INFO - __main__ - Step: 600 Loss: 10.5936 LR: 0.000300
+12/08/2025 06:51:03 - INFO - __main__ - Step: 610 Loss: 10.6108 LR: 0.000300
+12/08/2025 06:51:25 - INFO - __main__ - Step: 620 Loss: 10.5532 LR: 0.000300
+12/08/2025 06:51:47 - INFO - __main__ - Step: 630 Loss: 10.5550 LR: 0.000300
+12/08/2025 06:52:09 - INFO - __main__ - Step: 640 Loss: 10.6156 LR: 0.000300
+12/08/2025 06:52:30 - INFO - __main__ - Step: 650 Loss: 10.6020 LR: 0.000300
+12/08/2025 06:53:25 - INFO - __main__ - Step: 660 Loss: 10.5624 LR: 0.000300
+12/08/2025 06:53:47 - INFO - __main__ - Step: 670 Loss: 10.5785 LR: 0.000300
+12/08/2025 06:54:22 - INFO - __main__ - Step: 680 Loss: 10.5761 LR: 0.000300
+12/08/2025 06:54:47 - INFO - __main__ - Step: 690 Loss: 10.5741 LR: 0.000300
+12/08/2025 06:55:31 - INFO - __main__ - Step: 700 Loss: 10.5785 LR: 0.000300
+12/08/2025 06:55:54 - INFO - __main__ - Step: 710 Loss: 10.5945 LR: 0.000300
+12/08/2025 06:56:43 - INFO - __main__ - Step: 720 Loss: 10.5611 LR: 0.000300
+12/08/2025 06:57:18 - INFO - __main__ - Step: 730 Loss: 10.5659 LR: 0.000300
+12/08/2025 06:57:39 - INFO - __main__ - Step: 740 Loss: 10.5558 LR: 0.000300
+12/08/2025 06:58:11 - INFO - __main__ - Step: 750 Loss: 10.5648 LR: 0.000300
+12/08/2025 06:58:43 - INFO - __main__ - Step: 760 Loss: 10.5705 LR: 0.000300
+12/08/2025 06:59:28 - INFO - __main__ - Step: 770 Loss: 10.5315 LR: 0.000300
+12/08/2025 06:59:50 - INFO - __main__ - Step: 780 Loss: 10.5513 LR: 0.000300
+12/08/2025 07:00:39 - INFO - __main__ - Step: 790 Loss: 10.5674 LR: 0.000300
+12/08/2025 07:01:00 - INFO - __main__ - Step: 800 Loss: 10.5174 LR: 0.000300
+12/08/2025 07:01:39 - INFO - __main__ - Step: 810 Loss: 10.5911 LR: 0.000300
+12/08/2025 07:02:02 - INFO - __main__ - Step: 820 Loss: 10.5389 LR: 0.000300
+12/08/2025 07:02:52 - INFO - __main__ - Step: 830 Loss: 10.5487 LR: 0.000300
+12/08/2025 07:03:13 - INFO - __main__ - Step: 840 Loss: 10.5537 LR: 0.000300
+12/08/2025 07:03:48 - INFO - __main__ - Step: 850 Loss: 10.5564 LR: 0.000300
+12/08/2025 07:04:21 - INFO - __main__ - Step: 860 Loss: 10.5350 LR: 0.000300
+12/08/2025 07:04:52 - INFO - __main__ - Step: 870 Loss: 10.5221 LR: 0.000300
+12/08/2025 07:05:35 - INFO - __main__ - Step: 880 Loss: 10.4884 LR: 0.000300
+12/08/2025 07:06:01 - INFO - __main__ - Step: 890 Loss: 10.5384 LR: 0.000300
+12/08/2025 07:06:45 - INFO - __main__ - Step: 900 Loss: 10.5458 LR: 0.000300
+12/08/2025 07:07:06 - INFO - __main__ - Step: 910 Loss: 10.5227 LR: 0.000300
+12/08/2025 07:08:01 - INFO - __main__ - Step: 920 Loss: 10.5706 LR: 0.000300
+12/08/2025 07:08:21 - INFO - __main__ - Step: 930 Loss: 10.5531 LR: 0.000300
+12/08/2025 07:08:43 - INFO - __main__ - Step: 940 Loss: 10.5201 LR: 0.000300
+12/08/2025 07:09:33 - INFO - __main__ - Step: 950 Loss: 10.5325 LR: 0.000300
+12/08/2025 07:09:59 - INFO - __main__ - Step: 960 Loss: 10.5191 LR: 0.000300
+12/08/2025 07:10:42 - INFO - __main__ - Step: 970 Loss: 10.5592 LR: 0.000300
+12/08/2025 07:11:12 - INFO - __main__ - Step: 980 Loss: 10.5364 LR: 0.000300
+12/08/2025 07:11:56 - INFO - __main__ - Step: 990 Loss: 10.5596 LR: 0.000300
+12/08/2025 07:12:18 - INFO - __main__ - Step: 1000 Loss: 10.4784 LR: 0.000300
+12/08/2025 07:12:18 - INFO - accelerate.accelerator - Saving current state to output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000
+12/08/2025 07:12:27 - INFO - accelerate.checkpointing - Optimizer state saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000/optimizer.bin
+12/08/2025 07:12:27 - INFO - accelerate.checkpointing - Scheduler state saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000/scheduler.bin
+12/08/2025 07:12:27 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000/sampler.bin
+12/08/2025 07:12:27 - INFO - accelerate.checkpointing - Random states saved in output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000/random_states_0.pkl
+12/08/2025 07:12:27 - INFO - __main__ - Saved state to output_32x56_4f_8bs_4*8*8vqvae/checkpoint-1000
+12/08/2025 07:12:27 - INFO - __main__ - Generating videos for validation...
+12/08/2025 07:12:27 - INFO - __main__ - Generating videos for validation...
+12/08/2025 07:12:27 - ERROR - __main__ - Video validation failed: num_frames (4) must be divisible by temporal downsampling factor (8)
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1278, in main
+ videos = pipe(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/src/pipeline_video.py", line 541, in __call__
+ # raise ValueError(
+ValueError: num_frames (4) must be divisible by temporal downsampling factor (8)
+12/08/2025 07:13:05 - INFO - __main__ - Step: 1010 Loss: 10.5068 LR: 0.000300
+12/08/2025 07:13:26 - INFO - __main__ - Step: 1020 Loss: 10.5221 LR: 0.000300
+12/08/2025 07:14:23 - INFO - __main__ - Step: 1030 Loss: 10.5297 LR: 0.000300
+12/08/2025 07:14:44 - INFO - __main__ - Step: 1040 Loss: 10.5535 LR: 0.000300
+12/08/2025 07:15:22 - INFO - __main__ - Step: 1050 Loss: 10.4940 LR: 0.000300
+12/08/2025 07:16:09 - INFO - __main__ - Step: 1060 Loss: 10.5193 LR: 0.000300
+12/08/2025 07:16:29 - INFO - __main__ - Step: 1070 Loss: 10.5210 LR: 0.000300
+12/08/2025 07:17:11 - INFO - __main__ - Step: 1080 Loss: 10.5040 LR: 0.000300
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/files/requirements.txt b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0ec402a259213b8efe30829beccb256a9922e76
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T06:27:41.588262Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "32",
+ "--video_width",
+ "56",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_32x56_4f_8bs_4*8*8vqvae",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11819721035776"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "k0gn0kc3ownpjwqi8jnk80jh4wuclza1"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-core.log b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..2e34670a5d26421792ba54893afade248a1fb4b9
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-08T06:27:41.660179644Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpwsrt0puo/port-1006985.txt","pid":1006985,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T06:27:41.660621135Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1006985}
+{"time":"2025-12-08T06:27:41.660633944Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1006985-1007245-2704984213/socket","Net":"unix"}}
+{"time":"2025-12-08T06:27:41.847376456Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T06:27:41.853178851Z","level":"INFO","msg":"handleInformInit: received","streamId":"qalkbn80","id":"1(@)"}
+{"time":"2025-12-08T06:27:42.023589568Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qalkbn80","id":"1(@)"}
+{"time":"2025-12-08T07:17:14.111168672Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-internal.log b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..b82da0c7d7cd4bdf482bdc98a1decdc61ffa65fc
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-08T06:27:41.853277789Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T06:27:42.023420232Z","level":"INFO","msg":"stream: created new stream","id":"qalkbn80"}
+{"time":"2025-12-08T06:27:42.023501295Z","level":"INFO","msg":"handler: started","stream_id":"qalkbn80"}
+{"time":"2025-12-08T06:27:42.023581598Z","level":"INFO","msg":"stream: started","id":"qalkbn80"}
+{"time":"2025-12-08T06:27:42.023599784Z","level":"INFO","msg":"sender: started","stream_id":"qalkbn80"}
+{"time":"2025-12-08T06:27:42.023603396Z","level":"INFO","msg":"writer: started","stream_id":"qalkbn80"}
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug.log b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..28abeefcf99357fec2c4d10f8b2646a4bf87ac3d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_setup.py:_flush():80] Configure stats pid to 1006985
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug.log
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_062741-qalkbn80/logs/debug-internal.log
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_init.py:init():841] calling init triggers
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 06:27:41,591 INFO MainThread:1006985 [wandb_init.py:init():889] starting backend
+2025-12-08 06:27:41,847 INFO MainThread:1006985 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 06:27:41,851 INFO MainThread:1006985 [wandb_init.py:init():900] backend started and connected
+2025-12-08 06:27:41,852 INFO MainThread:1006985 [wandb_init.py:init():970] updated telemetry
+2025-12-08 06:27:41,857 INFO MainThread:1006985 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 06:27:42,221 INFO MainThread:1006985 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 06:27:42,330 INFO MainThread:1006985 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 06:27:42,330 INFO MainThread:1006985 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 06:27:42,330 INFO MainThread:1006985 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 06:27:42,330 INFO MainThread:1006985 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 06:27:42,333 INFO MainThread:1006985 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 06:27:42,334 INFO MainThread:1006985 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_32x56_4f_8bs_4*8*8vqvae', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 32, 'video_width': 56, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
diff --git a/Meissonic/wandb/run-20251208_062741-qalkbn80/run-qalkbn80.wandb b/Meissonic/wandb/run-20251208_062741-qalkbn80/run-qalkbn80.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..30c19bb3b99bad76cdcd6557e4e665bc32502fbf
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_062741-qalkbn80/run-qalkbn80.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36015ce0a4a235532e09c78c419a4c7f2c496cb34e5c654be87efd8db78d2f9e
+size 753664
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_5fba5b3b9257a6ea4286.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_5fba5b3b9257a6ea4286.png
new file mode 100644
index 0000000000000000000000000000000000000000..ec5038cf126880965951c4cd6b3c7e782541a01d
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_5fba5b3b9257a6ea4286.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_76389031b22f5747f9d1.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_76389031b22f5747f9d1.png
new file mode 100644
index 0000000000000000000000000000000000000000..696a127bb2b8baf83f042ca6a79e2c8847c12b4c
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1000_76389031b22f5747f9d1.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_53281bd82574163ad2ab.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_53281bd82574163ad2ab.png
new file mode 100644
index 0000000000000000000000000000000000000000..d62bca398a77b34ae3370abb06c9a589c619e8d1
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_53281bd82574163ad2ab.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_9d4f74d0f61cf2b121fe.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_9d4f74d0f61cf2b121fe.png
new file mode 100644
index 0000000000000000000000000000000000000000..9d515824cd840ec4cc777ea421db55149f10f9e2
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_1500_9d4f74d0f61cf2b121fe.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_884f228ba161275a3639.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_884f228ba161275a3639.png
new file mode 100644
index 0000000000000000000000000000000000000000..c949d7f93efb8581eb380eaecaaa4cb13b5e9a64
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_884f228ba161275a3639.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_a607f990a79bec5f1066.png b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_a607f990a79bec5f1066.png
new file mode 100644
index 0000000000000000000000000000000000000000..df56735949c1ee4dd76678a022b0c2fff4428dce
Binary files /dev/null and b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/media/images/generated_videos_first_frame_500_a607f990a79bec5f1066.png differ
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/output.log b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d3290a9f72e5ce4d08a5f97e74e0d7fb158eba01
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/output.log
@@ -0,0 +1,359 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 68.78it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 07:18:26 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 3671.39it/s]
+12/08/2025 07:18:28 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=16, W'=28
+12/08/2025 07:18:28 - INFO - __main__ - Theoretical dimensions: F'=1, H'=16, W'=28
+12/08/2025 07:18:28 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 07:18:28 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 07:18:43 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 07:18:43 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 07:18:45 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 07:18:47 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 07:18:56 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 07:18:56 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 07:18:56 - INFO - __main__ - Dataloader configuration:
+12/08/2025 07:18:56 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 07:18:56 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 07:18:56 - INFO - __main__ - - persistent_workers: True
+12/08/2025 07:18:56 - INFO - __main__ - - pin_memory: True
+12/08/2025 07:18:56 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 07:19:12 - INFO - __main__ - ***** Running training *****
+12/08/2025 07:19:12 - INFO - __main__ - Num training steps = 10000
+12/08/2025 07:19:12 - INFO - __main__ - Instantaneous batch size per device = 2
+12/08/2025 07:19:12 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 16
+12/08/2025 07:19:12 - INFO - __main__ - Gradient Accumulation steps = 1
+12/08/2025 07:19:41 - INFO - __main__ - Step: 10 Loss: 11.0791 LR: 0.000300
+12/08/2025 07:19:51 - INFO - __main__ - Step: 20 Loss: 11.0750 LR: 0.000300
+12/08/2025 07:20:01 - INFO - __main__ - Step: 30 Loss: 11.0739 LR: 0.000300
+12/08/2025 07:20:12 - INFO - __main__ - Step: 40 Loss: 11.0735 LR: 0.000300
+12/08/2025 07:20:22 - INFO - __main__ - Step: 50 Loss: 11.0703 LR: 0.000300
+12/08/2025 07:20:33 - INFO - __main__ - Step: 60 Loss: 11.0699 LR: 0.000300
+12/08/2025 07:20:43 - INFO - __main__ - Step: 70 Loss: 11.0723 LR: 0.000300
+12/08/2025 07:20:54 - INFO - __main__ - Step: 80 Loss: 11.0692 LR: 0.000300
+12/08/2025 07:21:04 - INFO - __main__ - Step: 90 Loss: 11.0679 LR: 0.000300
+12/08/2025 07:21:14 - INFO - __main__ - Step: 100 Loss: 11.0647 LR: 0.000300
+12/08/2025 07:21:25 - INFO - __main__ - Step: 110 Loss: 11.0511 LR: 0.000300
+12/08/2025 07:21:36 - INFO - __main__ - Step: 120 Loss: 11.0460 LR: 0.000300
+12/08/2025 07:21:46 - INFO - __main__ - Step: 130 Loss: 11.0441 LR: 0.000300
+12/08/2025 07:21:57 - INFO - __main__ - Step: 140 Loss: 11.0292 LR: 0.000300
+12/08/2025 07:22:07 - INFO - __main__ - Step: 150 Loss: 11.0091 LR: 0.000300
+12/08/2025 07:22:16 - INFO - __main__ - Step: 160 Loss: 10.9959 LR: 0.000300
+12/08/2025 07:22:27 - INFO - __main__ - Step: 170 Loss: 10.9785 LR: 0.000300
+12/08/2025 07:22:37 - INFO - __main__ - Step: 180 Loss: 10.9652 LR: 0.000300
+12/08/2025 07:22:48 - INFO - __main__ - Step: 190 Loss: 10.9404 LR: 0.000300
+12/08/2025 07:22:58 - INFO - __main__ - Step: 200 Loss: 10.9111 LR: 0.000300
+12/08/2025 07:23:09 - INFO - __main__ - Step: 210 Loss: 10.8844 LR: 0.000300
+12/08/2025 07:23:19 - INFO - __main__ - Step: 220 Loss: 10.8804 LR: 0.000300
+12/08/2025 07:23:29 - INFO - __main__ - Step: 230 Loss: 10.8483 LR: 0.000300
+12/08/2025 07:23:39 - INFO - __main__ - Step: 240 Loss: 10.8448 LR: 0.000300
+12/08/2025 07:23:50 - INFO - __main__ - Step: 250 Loss: 10.8387 LR: 0.000300
+12/08/2025 07:24:01 - INFO - __main__ - Step: 260 Loss: 10.8275 LR: 0.000300
+12/08/2025 07:24:11 - INFO - __main__ - Step: 270 Loss: 10.7833 LR: 0.000300
+12/08/2025 07:24:22 - INFO - __main__ - Step: 280 Loss: 10.8187 LR: 0.000300
+12/08/2025 07:24:33 - INFO - __main__ - Step: 290 Loss: 10.7622 LR: 0.000300
+12/08/2025 07:24:43 - INFO - __main__ - Step: 300 Loss: 10.7791 LR: 0.000300
+12/08/2025 07:24:54 - INFO - __main__ - Step: 310 Loss: 10.7556 LR: 0.000300
+12/08/2025 07:25:04 - INFO - __main__ - Step: 320 Loss: 10.7484 LR: 0.000300
+12/08/2025 07:25:14 - INFO - __main__ - Step: 330 Loss: 10.7761 LR: 0.000300
+12/08/2025 07:25:24 - INFO - __main__ - Step: 340 Loss: 10.7330 LR: 0.000300
+12/08/2025 07:25:35 - INFO - __main__ - Step: 350 Loss: 10.7359 LR: 0.000300
+12/08/2025 07:25:45 - INFO - __main__ - Step: 360 Loss: 10.7433 LR: 0.000300
+12/08/2025 07:25:55 - INFO - __main__ - Step: 370 Loss: 10.7281 LR: 0.000300
+12/08/2025 07:26:06 - INFO - __main__ - Step: 380 Loss: 10.7423 LR: 0.000300
+12/08/2025 07:26:16 - INFO - __main__ - Step: 390 Loss: 10.7052 LR: 0.000300
+12/08/2025 07:26:26 - INFO - __main__ - Step: 400 Loss: 10.6671 LR: 0.000300
+12/08/2025 07:26:37 - INFO - __main__ - Step: 410 Loss: 10.7009 LR: 0.000300
+12/08/2025 07:26:47 - INFO - __main__ - Step: 420 Loss: 10.6547 LR: 0.000300
+12/08/2025 07:26:57 - INFO - __main__ - Step: 430 Loss: 10.6868 LR: 0.000300
+12/08/2025 07:27:08 - INFO - __main__ - Step: 440 Loss: 10.6214 LR: 0.000300
+12/08/2025 07:27:18 - INFO - __main__ - Step: 450 Loss: 10.6331 LR: 0.000300
+12/08/2025 07:27:28 - INFO - __main__ - Step: 460 Loss: 10.6731 LR: 0.000300
+12/08/2025 07:27:39 - INFO - __main__ - Step: 470 Loss: 10.6490 LR: 0.000300
+12/08/2025 07:27:50 - INFO - __main__ - Step: 480 Loss: 10.6500 LR: 0.000300
+12/08/2025 07:28:00 - INFO - __main__ - Step: 490 Loss: 10.6372 LR: 0.000300
+12/08/2025 07:28:10 - INFO - __main__ - Step: 500 Loss: 10.5730 LR: 0.000300
+12/08/2025 07:28:10 - INFO - accelerate.accelerator - Saving current state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500
+12/08/2025 07:28:18 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500/optimizer.bin
+12/08/2025 07:28:18 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500/scheduler.bin
+12/08/2025 07:28:18 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500/sampler.bin
+12/08/2025 07:28:18 - INFO - accelerate.checkpointing - Random states saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500/random_states_0.pkl
+12/08/2025 07:28:18 - INFO - __main__ - Saved state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-500
+12/08/2025 07:28:18 - INFO - __main__ - Generating videos for validation...
+12/08/2025 07:28:18 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.28it/s]
+12/08/2025 07:28:25 - INFO - __main__ - Validation videos saved to ./output_128x224_4f_2bs_4*8*8vqvae
+12/08/2025 07:28:35 - INFO - __main__ - Step: 510 Loss: 10.6406 LR: 0.000300
+12/08/2025 07:28:46 - INFO - __main__ - Step: 520 Loss: 10.6642 LR: 0.000300
+12/08/2025 07:28:56 - INFO - __main__ - Step: 530 Loss: 10.6562 LR: 0.000300
+12/08/2025 07:29:07 - INFO - __main__ - Step: 540 Loss: 10.6532 LR: 0.000300
+12/08/2025 07:29:18 - INFO - __main__ - Step: 550 Loss: 10.6268 LR: 0.000300
+12/08/2025 07:29:28 - INFO - __main__ - Step: 560 Loss: 10.5279 LR: 0.000300
+12/08/2025 07:29:39 - INFO - __main__ - Step: 570 Loss: 10.6712 LR: 0.000300
+12/08/2025 07:29:49 - INFO - __main__ - Step: 580 Loss: 10.6163 LR: 0.000300
+12/08/2025 07:30:00 - INFO - __main__ - Step: 590 Loss: 10.5797 LR: 0.000300
+12/08/2025 07:30:12 - INFO - __main__ - Step: 600 Loss: 10.5672 LR: 0.000300
+12/08/2025 07:30:23 - INFO - __main__ - Step: 610 Loss: 10.6494 LR: 0.000300
+12/08/2025 07:30:35 - INFO - __main__ - Step: 620 Loss: 10.5624 LR: 0.000300
+12/08/2025 07:30:45 - INFO - __main__ - Step: 630 Loss: 10.6043 LR: 0.000300
+12/08/2025 07:30:55 - INFO - __main__ - Step: 640 Loss: 10.5373 LR: 0.000300
+12/08/2025 07:31:06 - INFO - __main__ - Step: 650 Loss: 10.5340 LR: 0.000300
+12/08/2025 07:31:17 - INFO - __main__ - Step: 660 Loss: 10.5840 LR: 0.000300
+12/08/2025 07:31:28 - INFO - __main__ - Step: 670 Loss: 10.5831 LR: 0.000300
+12/08/2025 07:31:39 - INFO - __main__ - Step: 680 Loss: 10.6162 LR: 0.000300
+12/08/2025 07:31:49 - INFO - __main__ - Step: 690 Loss: 10.5944 LR: 0.000300
+12/08/2025 07:31:59 - INFO - __main__ - Step: 700 Loss: 10.5457 LR: 0.000300
+12/08/2025 07:32:10 - INFO - __main__ - Step: 710 Loss: 10.5504 LR: 0.000300
+12/08/2025 07:32:21 - INFO - __main__ - Step: 720 Loss: 10.5841 LR: 0.000300
+12/08/2025 07:32:31 - INFO - __main__ - Step: 730 Loss: 10.5708 LR: 0.000300
+12/08/2025 07:32:42 - INFO - __main__ - Step: 740 Loss: 10.5878 LR: 0.000300
+12/08/2025 07:32:53 - INFO - __main__ - Step: 750 Loss: 10.4668 LR: 0.000300
+12/08/2025 07:33:04 - INFO - __main__ - Step: 760 Loss: 10.5180 LR: 0.000300
+12/08/2025 07:33:14 - INFO - __main__ - Step: 770 Loss: 10.5158 LR: 0.000300
+12/08/2025 07:33:25 - INFO - __main__ - Step: 780 Loss: 10.5711 LR: 0.000300
+12/08/2025 07:33:35 - INFO - __main__ - Step: 790 Loss: 10.4961 LR: 0.000300
+12/08/2025 07:33:47 - INFO - __main__ - Step: 800 Loss: 10.5915 LR: 0.000300
+12/08/2025 07:33:58 - INFO - __main__ - Step: 810 Loss: 10.5373 LR: 0.000300
+12/08/2025 07:34:10 - INFO - __main__ - Step: 820 Loss: 10.4458 LR: 0.000300
+12/08/2025 07:34:20 - INFO - __main__ - Step: 830 Loss: 10.5727 LR: 0.000300
+12/08/2025 07:34:30 - INFO - __main__ - Step: 840 Loss: 10.5203 LR: 0.000300
+12/08/2025 07:34:41 - INFO - __main__ - Step: 850 Loss: 10.5065 LR: 0.000300
+12/08/2025 07:34:53 - INFO - __main__ - Step: 860 Loss: 10.5346 LR: 0.000300
+12/08/2025 07:35:05 - INFO - __main__ - Step: 870 Loss: 10.4913 LR: 0.000300
+12/08/2025 07:35:16 - INFO - __main__ - Step: 880 Loss: 10.4633 LR: 0.000300
+12/08/2025 07:35:27 - INFO - __main__ - Step: 890 Loss: 10.5258 LR: 0.000300
+12/08/2025 07:35:37 - INFO - __main__ - Step: 900 Loss: 10.4886 LR: 0.000300
+12/08/2025 07:35:47 - INFO - __main__ - Step: 910 Loss: 10.4839 LR: 0.000300
+12/08/2025 07:35:58 - INFO - __main__ - Step: 920 Loss: 10.5210 LR: 0.000300
+12/08/2025 07:36:10 - INFO - __main__ - Step: 930 Loss: 10.4808 LR: 0.000300
+12/08/2025 07:36:21 - INFO - __main__ - Step: 940 Loss: 10.4740 LR: 0.000300
+12/08/2025 07:36:32 - INFO - __main__ - Step: 950 Loss: 10.5103 LR: 0.000300
+12/08/2025 07:36:43 - INFO - __main__ - Step: 960 Loss: 10.5238 LR: 0.000300
+12/08/2025 07:36:54 - INFO - __main__ - Step: 970 Loss: 10.5924 LR: 0.000300
+12/08/2025 07:37:04 - INFO - __main__ - Step: 980 Loss: 10.5212 LR: 0.000300
+12/08/2025 07:37:16 - INFO - __main__ - Step: 990 Loss: 10.4879 LR: 0.000300
+12/08/2025 07:37:27 - INFO - __main__ - Step: 1000 Loss: 10.6040 LR: 0.000300
+12/08/2025 07:37:27 - INFO - accelerate.accelerator - Saving current state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000
+12/08/2025 07:37:34 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000/optimizer.bin
+12/08/2025 07:37:34 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000/scheduler.bin
+12/08/2025 07:37:34 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000/sampler.bin
+12/08/2025 07:37:34 - INFO - accelerate.checkpointing - Random states saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000/random_states_0.pkl
+12/08/2025 07:37:34 - INFO - __main__ - Saved state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1000
+12/08/2025 07:37:34 - INFO - __main__ - Generating videos for validation...
+12/08/2025 07:37:34 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:07<00:00, 6.59it/s]
+12/08/2025 07:37:43 - INFO - __main__ - Validation videos saved to ./output_128x224_4f_2bs_4*8*8vqvae
+12/08/2025 07:37:55 - INFO - __main__ - Step: 1010 Loss: 10.5551 LR: 0.000300
+12/08/2025 07:38:06 - INFO - __main__ - Step: 1020 Loss: 10.5330 LR: 0.000300
+12/08/2025 07:38:18 - INFO - __main__ - Step: 1030 Loss: 10.5111 LR: 0.000300
+12/08/2025 07:38:29 - INFO - __main__ - Step: 1040 Loss: 10.4348 LR: 0.000300
+12/08/2025 07:38:41 - INFO - __main__ - Step: 1050 Loss: 10.4561 LR: 0.000300
+12/08/2025 07:38:52 - INFO - __main__ - Step: 1060 Loss: 10.5186 LR: 0.000300
+12/08/2025 07:39:03 - INFO - __main__ - Step: 1070 Loss: 10.5010 LR: 0.000300
+12/08/2025 07:39:15 - INFO - __main__ - Step: 1080 Loss: 10.4697 LR: 0.000300
+12/08/2025 07:39:26 - INFO - __main__ - Step: 1090 Loss: 10.4187 LR: 0.000300
+12/08/2025 07:39:38 - INFO - __main__ - Step: 1100 Loss: 10.4345 LR: 0.000300
+12/08/2025 07:39:50 - INFO - __main__ - Step: 1110 Loss: 10.4531 LR: 0.000300
+12/08/2025 07:40:01 - INFO - __main__ - Step: 1120 Loss: 10.4995 LR: 0.000300
+12/08/2025 07:40:12 - INFO - __main__ - Step: 1130 Loss: 10.5177 LR: 0.000300
+12/08/2025 07:40:23 - INFO - __main__ - Step: 1140 Loss: 10.4628 LR: 0.000300
+12/08/2025 07:40:34 - INFO - __main__ - Step: 1150 Loss: 10.5039 LR: 0.000300
+12/08/2025 07:40:45 - INFO - __main__ - Step: 1160 Loss: 10.5378 LR: 0.000300
+12/08/2025 07:40:56 - INFO - __main__ - Step: 1170 Loss: 10.4866 LR: 0.000300
+12/08/2025 07:41:07 - INFO - __main__ - Step: 1180 Loss: 10.4979 LR: 0.000300
+12/08/2025 07:41:19 - INFO - __main__ - Step: 1190 Loss: 10.4341 LR: 0.000300
+12/08/2025 07:41:30 - INFO - __main__ - Step: 1200 Loss: 10.4345 LR: 0.000300
+12/08/2025 07:41:42 - INFO - __main__ - Step: 1210 Loss: 10.5064 LR: 0.000300
+12/08/2025 07:41:53 - INFO - __main__ - Step: 1220 Loss: 10.5068 LR: 0.000300
+12/08/2025 07:42:04 - INFO - __main__ - Step: 1230 Loss: 10.5067 LR: 0.000300
+12/08/2025 07:42:16 - INFO - __main__ - Step: 1240 Loss: 10.5162 LR: 0.000300
+12/08/2025 07:42:27 - INFO - __main__ - Step: 1250 Loss: 10.5313 LR: 0.000300
+12/08/2025 07:42:38 - INFO - __main__ - Step: 1260 Loss: 10.4982 LR: 0.000300
+12/08/2025 07:42:50 - INFO - __main__ - Step: 1270 Loss: 10.4725 LR: 0.000300
+12/08/2025 07:43:01 - INFO - __main__ - Step: 1280 Loss: 10.5044 LR: 0.000300
+12/08/2025 07:43:12 - INFO - __main__ - Step: 1290 Loss: 10.4312 LR: 0.000300
+12/08/2025 07:43:24 - INFO - __main__ - Step: 1300 Loss: 10.4405 LR: 0.000300
+12/08/2025 07:43:35 - INFO - __main__ - Step: 1310 Loss: 10.4837 LR: 0.000300
+12/08/2025 07:43:45 - INFO - __main__ - Step: 1320 Loss: 10.4423 LR: 0.000300
+12/08/2025 07:43:57 - INFO - __main__ - Step: 1330 Loss: 10.4131 LR: 0.000300
+12/08/2025 07:44:08 - INFO - __main__ - Step: 1340 Loss: 10.4227 LR: 0.000300
+12/08/2025 07:44:20 - INFO - __main__ - Step: 1350 Loss: 10.5178 LR: 0.000300
+12/08/2025 07:44:31 - INFO - __main__ - Step: 1360 Loss: 10.4928 LR: 0.000300
+12/08/2025 07:44:41 - INFO - __main__ - Step: 1370 Loss: 10.5283 LR: 0.000300
+12/08/2025 07:44:52 - INFO - __main__ - Step: 1380 Loss: 10.4598 LR: 0.000300
+12/08/2025 07:45:03 - INFO - __main__ - Step: 1390 Loss: 10.5008 LR: 0.000300
+12/08/2025 07:45:15 - INFO - __main__ - Step: 1400 Loss: 10.5150 LR: 0.000300
+12/08/2025 07:45:26 - INFO - __main__ - Step: 1410 Loss: 10.4277 LR: 0.000300
+12/08/2025 07:45:37 - INFO - __main__ - Step: 1420 Loss: 10.5060 LR: 0.000300
+12/08/2025 07:45:48 - INFO - __main__ - Step: 1430 Loss: 10.4265 LR: 0.000300
+12/08/2025 07:46:00 - INFO - __main__ - Step: 1440 Loss: 10.3760 LR: 0.000300
+12/08/2025 07:46:11 - INFO - __main__ - Step: 1450 Loss: 10.4575 LR: 0.000300
+12/08/2025 07:46:22 - INFO - __main__ - Step: 1460 Loss: 10.4462 LR: 0.000300
+12/08/2025 07:46:34 - INFO - __main__ - Step: 1470 Loss: 10.4534 LR: 0.000300
+12/08/2025 07:46:45 - INFO - __main__ - Step: 1480 Loss: 10.4444 LR: 0.000300
+12/08/2025 07:46:55 - INFO - __main__ - Step: 1490 Loss: 10.4124 LR: 0.000300
+12/08/2025 07:47:06 - INFO - __main__ - Step: 1500 Loss: 10.5423 LR: 0.000300
+12/08/2025 07:47:06 - INFO - accelerate.accelerator - Saving current state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500
+12/08/2025 07:47:15 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500/optimizer.bin
+12/08/2025 07:47:15 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500/scheduler.bin
+12/08/2025 07:47:15 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500/sampler.bin
+12/08/2025 07:47:15 - INFO - accelerate.checkpointing - Random states saved in output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500/random_states_0.pkl
+12/08/2025 07:47:15 - INFO - __main__ - Saved state to output_128x224_4f_2bs_4*8*8vqvae/checkpoint-1500
+12/08/2025 07:47:15 - INFO - __main__ - Generating videos for validation...
+12/08/2025 07:47:15 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.10it/s]
+12/08/2025 07:47:21 - INFO - __main__ - Validation videos saved to ./output_128x224_4f_2bs_4*8*8vqvae
+12/08/2025 07:47:32 - INFO - __main__ - Step: 1510 Loss: 10.4962 LR: 0.000300
+12/08/2025 07:47:44 - INFO - __main__ - Step: 1520 Loss: 10.4884 LR: 0.000300
+12/08/2025 07:47:55 - INFO - __main__ - Step: 1530 Loss: 10.4585 LR: 0.000300
+12/08/2025 07:48:07 - INFO - __main__ - Step: 1540 Loss: 10.5206 LR: 0.000300
+12/08/2025 07:48:18 - INFO - __main__ - Step: 1550 Loss: 10.5191 LR: 0.000300
+12/08/2025 07:48:29 - INFO - __main__ - Step: 1560 Loss: 10.4605 LR: 0.000300
+12/08/2025 07:48:41 - INFO - __main__ - Step: 1570 Loss: 10.4379 LR: 0.000300
+12/08/2025 07:48:52 - INFO - __main__ - Step: 1580 Loss: 10.3727 LR: 0.000300
+12/08/2025 07:49:04 - INFO - __main__ - Step: 1590 Loss: 10.4478 LR: 0.000300
+12/08/2025 07:49:15 - INFO - __main__ - Step: 1600 Loss: 10.4360 LR: 0.000300
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1171, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+ x = cross_attn_ffn(x, context, context_lens, e)
+ File "/mnt/Meissonic/src/transformer_video.py", line 477, in cross_attn_ffn
+ x = x + self.cross_attn(self.norm3(x), context, context_lens)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 388, in forward
+ q = self.norm_q(self.q(x)).view(b, -1, n, d)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 281, in forward
+ return self._norm(x.float()).type_as(x) * self.weight.type_as(x)
+ File "/mnt/Meissonic/src/transformer_video.py", line 284, in _norm
+ return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1129, in pack_hook
+ def pack_hook(x):
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1359, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1171, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+[rank0]: x = cross_attn_ffn(x, context, context_lens, e)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 477, in cross_attn_ffn
+[rank0]: x = x + self.cross_attn(self.norm3(x), context, context_lens)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 388, in forward
+[rank0]: q = self.norm_q(self.q(x)).view(b, -1, n, d)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 281, in forward
+[rank0]: return self._norm(x.float()).type_as(x) * self.weight.type_as(x)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 284, in _norm
+[rank0]: return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1129, in pack_hook
+[rank0]: def pack_hook(x):
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/requirements.txt b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a8c4148adaf6c6672260af35dbc4f306eaeb16c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/requirements.txt
@@ -0,0 +1,139 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+aiohttp==3.13.2
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+pandas==2.3.3
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..32bd4f42d87816ec7051f233779e9c11846b44d4
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T07:18:23.137418Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "128",
+ "--video_width",
+ "224",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x224_4f_2bs_4*8*8vqvae",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11831191023616"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "455p1nc4ntuffrug5i0ormilp4qait21"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-core.log b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..bfd08d20e5678a4258e6fec10ffdc657bf955cf8
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-08T07:18:23.203099372Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpseyl58k4/port-2490558.txt","pid":2490558,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T07:18:23.20364499Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2490558}
+{"time":"2025-12-08T07:18:23.203657229Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2490558-2490821-350546645/socket","Net":"unix"}}
+{"time":"2025-12-08T07:18:23.390280295Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T07:18:23.396482982Z","level":"INFO","msg":"handleInformInit: received","streamId":"0hjx73rw","id":"1(@)"}
+{"time":"2025-12-08T07:18:23.561433511Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0hjx73rw","id":"1(@)"}
+{"time":"2025-12-08T07:49:20.399918343Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-internal.log b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..803928fa69963b998da035c419954c265e90bf30
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-08T07:18:23.396700819Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T07:18:23.561211348Z","level":"INFO","msg":"stream: created new stream","id":"0hjx73rw"}
+{"time":"2025-12-08T07:18:23.561299615Z","level":"INFO","msg":"handler: started","stream_id":"0hjx73rw"}
+{"time":"2025-12-08T07:18:23.561424841Z","level":"INFO","msg":"stream: started","id":"0hjx73rw"}
+{"time":"2025-12-08T07:18:23.561440104Z","level":"INFO","msg":"writer: started","stream_id":"0hjx73rw"}
+{"time":"2025-12-08T07:18:23.561440819Z","level":"INFO","msg":"sender: started","stream_id":"0hjx73rw"}
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug.log b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ab27e1c2a274b132415946312b79ea7294b323b7
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_setup.py:_flush():80] Configure stats pid to 2490558
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug.log
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_071823-0hjx73rw/logs/debug-internal.log
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_init.py:init():841] calling init triggers
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 07:18:23,140 INFO MainThread:2490558 [wandb_init.py:init():889] starting backend
+2025-12-08 07:18:23,390 INFO MainThread:2490558 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 07:18:23,394 INFO MainThread:2490558 [wandb_init.py:init():900] backend started and connected
+2025-12-08 07:18:23,396 INFO MainThread:2490558 [wandb_init.py:init():970] updated telemetry
+2025-12-08 07:18:23,400 INFO MainThread:2490558 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 07:18:23,747 INFO MainThread:2490558 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 07:18:23,856 INFO MainThread:2490558 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 07:18:23,856 INFO MainThread:2490558 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 07:18:23,856 INFO MainThread:2490558 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 07:18:23,856 INFO MainThread:2490558 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 07:18:23,859 INFO MainThread:2490558 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 07:18:23,860 INFO MainThread:2490558 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x224_4f_2bs_4*8*8vqvae', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 128, 'video_width': 224, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B'}
diff --git a/Meissonic/wandb/run-20251208_071823-0hjx73rw/run-0hjx73rw.wandb b/Meissonic/wandb/run-20251208_071823-0hjx73rw/run-0hjx73rw.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..1bdc181f2cfaf9097e586aac8f4f61a1507374b2
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_071823-0hjx73rw/run-0hjx73rw.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e444d4e446935f8afc716a8dc9c6a719d6314cbc073ed86d425452e2cb0cccd
+size 589824
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/files/config.yaml b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c8e9d7d65cbb5a1098dfdf3904cc051180267235
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/config.yaml
@@ -0,0 +1,296 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 0vt3ubbl1cfi1nqfghobvagoflxcjvzc:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11900304912384"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T15:56:07.007993Z"
+ writerId: 0vt3ubbl1cfi1nqfghobvagoflxcjvzc
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 0.1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/files/output.log b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..cc0f9d565473a86e4fdcb6f24b6d0b3473233043
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/output.log
@@ -0,0 +1,164 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 68.46it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 15:56:10 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4299.96it/s]
+12/08/2025 15:56:12 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/08/2025 15:56:12 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/08/2025 15:56:12 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 15:56:12 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 15:56:27 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 15:56:27 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 15:56:29 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 15:56:31 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/08/2025 15:56:31 - INFO - __main__ - Wan backbone lr = 0.000030 (base_lr * 0.1)
+12/08/2025 15:56:31 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/08/2025 15:56:31 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 15:56:38 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 15:56:38 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 15:56:38 - INFO - __main__ - Dataloader configuration:
+12/08/2025 15:56:38 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 15:56:38 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 15:56:38 - INFO - __main__ - - persistent_workers: True
+12/08/2025 15:56:38 - INFO - __main__ - - pin_memory: True
+12/08/2025 15:56:38 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 15:56:57 - INFO - __main__ - ***** Running training *****
+12/08/2025 15:56:57 - INFO - __main__ - Num training steps = 10000
+12/08/2025 15:56:57 - INFO - __main__ - Instantaneous batch size per device = 2
+12/08/2025 15:56:57 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 16
+12/08/2025 15:56:57 - INFO - __main__ - Gradient Accumulation steps = 1
+12/08/2025 15:57:19 - INFO - __main__ - Step: 10 Loss: 11.0874 LR: 0.000030
+12/08/2025 15:57:30 - INFO - __main__ - Step: 20 Loss: 11.0766 LR: 0.000030
+12/08/2025 15:57:41 - INFO - __main__ - Step: 30 Loss: 11.0784 LR: 0.000030
+12/08/2025 15:57:52 - INFO - __main__ - Step: 40 Loss: 11.0731 LR: 0.000030
+12/08/2025 15:58:04 - INFO - __main__ - Step: 50 Loss: 11.0742 LR: 0.000030
+12/08/2025 15:58:15 - INFO - __main__ - Step: 60 Loss: 11.0691 LR: 0.000030
+12/08/2025 15:58:27 - INFO - __main__ - Step: 70 Loss: 11.0676 LR: 0.000030
+12/08/2025 15:58:38 - INFO - __main__ - Step: 80 Loss: 11.0666 LR: 0.000030
+12/08/2025 15:58:49 - INFO - __main__ - Step: 90 Loss: 11.0633 LR: 0.000030
+12/08/2025 15:59:00 - INFO - __main__ - Step: 100 Loss: 11.0617 LR: 0.000030
+12/08/2025 15:59:11 - INFO - __main__ - Step: 110 Loss: 11.0550 LR: 0.000030
+12/08/2025 15:59:23 - INFO - __main__ - Step: 120 Loss: 11.0503 LR: 0.000030
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1435, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1247, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 357, in forward
+ x = flash_attention(
+ File "/mnt/Meissonic/src/transformer_video.py", line 77, in flash_attention
+ q_lens = torch.tensor(
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1435, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1247, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 357, in forward
+[rank0]: x = flash_attention(
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 77, in flash_attention
+[rank0]: q_lens = torch.tensor(
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/files/requirements.txt b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ca92f178ce0933595fef14b0bd95b3f51a2cf30
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T15:56:07.007993Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11900304912384"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "0vt3ubbl1cfi1nqfghobvagoflxcjvzc"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-summary.json b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ad32f50c2245505a6cd51fa77e9b45c2c086edc
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/files/wandb-summary.json
@@ -0,0 +1 @@
+{"avg_masking_rate":0.5684051513671875,"_timestamp":1.765209563009791e+09,"_step":120,"_runtime":197.270936192,"_wandb":{"runtime":197},"step_loss":11.050317764282227,"lr":2.9999999999999997e-05}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-core.log b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..3f002bed3d617a55d4dbc55514cd568540a2a670
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-08T15:56:07.075882669Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpii1iaw4p/port-3410532.txt","pid":3410532,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T15:56:07.076335169Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3410532}
+{"time":"2025-12-08T15:56:07.076347658Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3410532-3410856-3631560017/socket","Net":"unix"}}
+{"time":"2025-12-08T15:56:07.262421503Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T15:56:07.268400421Z","level":"INFO","msg":"handleInformInit: received","streamId":"ja5993e9","id":"1(@)"}
+{"time":"2025-12-08T15:56:07.431953902Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ja5993e9","id":"1(@)"}
+{"time":"2025-12-08T15:59:24.939887729Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-08T15:59:24.939946483Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-08T15:59:24.93997108Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-08T15:59:24.940000281Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-08T15:59:24.94006594Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3410532-3410856-3631560017/socket","Net":"unix"}}
+{"time":"2025-12-08T15:59:25.355246261Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-08T15:59:25.355288673Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-08T15:59:25.355306619Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-internal.log b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d91ea0ded5a3a0bdfd584666f3ed5d1de86f4dd5
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-08T15:56:07.26854712Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T15:56:07.431715392Z","level":"INFO","msg":"stream: created new stream","id":"ja5993e9"}
+{"time":"2025-12-08T15:56:07.43179556Z","level":"INFO","msg":"handler: started","stream_id":"ja5993e9"}
+{"time":"2025-12-08T15:56:07.43194582Z","level":"INFO","msg":"stream: started","id":"ja5993e9"}
+{"time":"2025-12-08T15:56:07.43196348Z","level":"INFO","msg":"writer: started","stream_id":"ja5993e9"}
+{"time":"2025-12-08T15:56:07.43196722Z","level":"INFO","msg":"sender: started","stream_id":"ja5993e9"}
+{"time":"2025-12-08T15:59:24.939960682Z","level":"INFO","msg":"stream: closing","id":"ja5993e9"}
+{"time":"2025-12-08T15:59:25.193247831Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-08T15:59:25.351322144Z","level":"INFO","msg":"handler: closed","stream_id":"ja5993e9"}
+{"time":"2025-12-08T15:59:25.351682568Z","level":"INFO","msg":"sender: closed","stream_id":"ja5993e9"}
+{"time":"2025-12-08T15:59:25.351702093Z","level":"INFO","msg":"stream: closed","id":"ja5993e9"}
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug.log b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..a9898d7551095a29733d3db45f5b68b7ccef3965
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_setup.py:_flush():80] Configure stats pid to 3410532
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug.log
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_155607-ja5993e9/logs/debug-internal.log
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_init.py:init():841] calling init triggers
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 15:56:07,010 INFO MainThread:3410532 [wandb_init.py:init():889] starting backend
+2025-12-08 15:56:07,262 INFO MainThread:3410532 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 15:56:07,266 INFO MainThread:3410532 [wandb_init.py:init():900] backend started and connected
+2025-12-08 15:56:07,268 INFO MainThread:3410532 [wandb_init.py:init():970] updated telemetry
+2025-12-08 15:56:07,272 INFO MainThread:3410532 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 15:56:07,668 INFO MainThread:3410532 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 15:56:07,791 INFO MainThread:3410532 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 15:56:07,791 INFO MainThread:3410532 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 15:56:07,792 INFO MainThread:3410532 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 15:56:07,792 INFO MainThread:3410532 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 15:56:07,794 INFO MainThread:3410532 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 15:56:07,795 INFO MainThread:3410532 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.1}
+2025-12-08 15:59:24,940 INFO wandb-AsyncioManager-main:3410532 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-08 15:59:24,940 INFO wandb-AsyncioManager-main:3410532 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251208_155607-ja5993e9/run-ja5993e9.wandb b/Meissonic/wandb/run-20251208_155607-ja5993e9/run-ja5993e9.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..532536289e0b74179b6410f48d12106677fc38cb
Binary files /dev/null and b/Meissonic/wandb/run-20251208_155607-ja5993e9/run-ja5993e9.wandb differ
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/config.yaml b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..99fdc429ea0225debb67b29c7ae42790fc323bcf
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/config.yaml
@@ -0,0 +1,297 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ f0bt4siuu2gjzmv16hc4nme2mcfrsfsi:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "11900305113088"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-08T15:59:43.669474Z"
+ writerId: f0bt4siuu2gjzmv16hc4nme2mcfrsfsi
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 2
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 0.1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_8328d2d0556a95ff2759.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_8328d2d0556a95ff2759.png
new file mode 100644
index 0000000000000000000000000000000000000000..c2aa59ddfc654ec9f82d4c9abff61590cc837f39
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_8328d2d0556a95ff2759.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8328d2d0556a95ff2759ddf86ad4dcf2e293d935c74841fcc2a3b4a6bcafa2a5
+size 143482
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_980ee3261a5cf9cce942.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_980ee3261a5cf9cce942.png
new file mode 100644
index 0000000000000000000000000000000000000000..1a5c6ceb734f615b46258dee56e70de2cef8ee42
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_10000_980ee3261a5cf9cce942.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:980ee3261a5cf9cce942d12e4e4729bf254c67eef014ef3b6cf421a272ab480c
+size 157765
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_8fd26361f0705a90a632.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_8fd26361f0705a90a632.png
new file mode 100644
index 0000000000000000000000000000000000000000..d71a3c7a84b2a61722a420daeeb652249dd5c59f
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_8fd26361f0705a90a632.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fd26361f0705a90a6329c8a9cd00e16eae894edbeb49608d76a9da33ccdb3f5
+size 171306
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_cec203cb5c36d2873217.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_cec203cb5c36d2873217.png
new file mode 100644
index 0000000000000000000000000000000000000000..4c0a3241e815e1072e83a7c1930add5724c2e6e1
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1000_cec203cb5c36d2873217.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cec203cb5c36d2873217dce75eb242633b0d4fd0aa6d1b1a5ae91203cc511aa7
+size 154952
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_c061c65a6ce343b1660e.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_c061c65a6ce343b1660e.png
new file mode 100644
index 0000000000000000000000000000000000000000..bcc8d3c0fbb18c2aed90fb5ab881a79a74a90a6c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_c061c65a6ce343b1660e.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c061c65a6ce343b1660e7e716906c76632561aff9ccd459c06e2f77a6c6ce023
+size 151206
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_f047fb97b642dc30b33c.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_f047fb97b642dc30b33c.png
new file mode 100644
index 0000000000000000000000000000000000000000..3be830fede518a8205d018f3422730b27bc68f08
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_1500_f047fb97b642dc30b33c.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f047fb97b642dc30b33c713788d30200b71afdcfb75e30c504fdf6c6a207f390
+size 163793
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_2805ac51dfa6ef4de083.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_2805ac51dfa6ef4de083.png
new file mode 100644
index 0000000000000000000000000000000000000000..71284bdfb848395c693268f4a58f7c484b3ac992
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_2805ac51dfa6ef4de083.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2805ac51dfa6ef4de083f749ae468d53d2e04e1fe1f53862f0e92a1b77c77c0f
+size 154531
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_e98ce360ce92d75f9a36.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_e98ce360ce92d75f9a36.png
new file mode 100644
index 0000000000000000000000000000000000000000..96d8972d2c2014040d9269500a147c781820c929
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2000_e98ce360ce92d75f9a36.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98ce360ce92d75f9a36beceaa6a8e65c74d6190aa53a74c43f7bd80ee9135e9
+size 160429
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_430592107b01c838d952.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_430592107b01c838d952.png
new file mode 100644
index 0000000000000000000000000000000000000000..7b859d41e223d88cfcc30a0d94543156bd92097d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_430592107b01c838d952.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430592107b01c838d95230455d6237687f51489c736abc6f4a54a3a888b2d9d4
+size 151819
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_ecca4db815beca263f13.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_ecca4db815beca263f13.png
new file mode 100644
index 0000000000000000000000000000000000000000..527bff166840b69be09c32f54b33a8afa7349421
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_2500_ecca4db815beca263f13.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecca4db815beca263f13207354f4e3e1b9cc6944e4a79a1a08edaefebfa40c19
+size 164005
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_52422bc6ab7caedd5b8c.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_52422bc6ab7caedd5b8c.png
new file mode 100644
index 0000000000000000000000000000000000000000..7804b818e67765833102006e8205c3f9a2753821
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_52422bc6ab7caedd5b8c.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52422bc6ab7caedd5b8c29f1057d92a2c74d6c38fcd27ede46ebec43ff8cfe59
+size 106641
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_8e35b9b7d6b7a0806553.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_8e35b9b7d6b7a0806553.png
new file mode 100644
index 0000000000000000000000000000000000000000..a8bbcdfcc00d28eb2ad200c7ca671522dc63446b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3000_8e35b9b7d6b7a0806553.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e35b9b7d6b7a08065532353cfef040cacdaa06a696822639c2349c5ae229bbc
+size 128321
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_227067a6cd64b7cdced4.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_227067a6cd64b7cdced4.png
new file mode 100644
index 0000000000000000000000000000000000000000..73ca44051bb59bfad72ceb656c79c9743ff4b68c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_227067a6cd64b7cdced4.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:227067a6cd64b7cdced40c483f0da283047ad4d0df910d1d6deeeb57272326f4
+size 154964
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_55ba9221da0bf3c49190.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_55ba9221da0bf3c49190.png
new file mode 100644
index 0000000000000000000000000000000000000000..a7ee239599090e349a38a61feeee19ca1e7d5600
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_3500_55ba9221da0bf3c49190.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55ba9221da0bf3c4919028ee3c41da6922b3b52403bdedcf340600bdbd33f512
+size 155417
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_9a50d3903fd31767c616.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_9a50d3903fd31767c616.png
new file mode 100644
index 0000000000000000000000000000000000000000..c8652ab59f8567c725658b38071a10399cfcba7d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_9a50d3903fd31767c616.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a50d3903fd31767c61693491fe8690adbe9040c7f7f963a469cda72c36ac636
+size 116216
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_ffacfcca81b53cb27319.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_ffacfcca81b53cb27319.png
new file mode 100644
index 0000000000000000000000000000000000000000..ea92180801509d5a83faf01f800be416c6a17141
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4000_ffacfcca81b53cb27319.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffacfcca81b53cb27319a5287eebacece7bb2bb70ecf2770cbf90c249834bf4b
+size 137536
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_935711ba29b3ab613691.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_935711ba29b3ab613691.png
new file mode 100644
index 0000000000000000000000000000000000000000..60077216d24148c8123a137ea44ec45536c6b8a9
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_935711ba29b3ab613691.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:935711ba29b3ab6136917a68834005a2c21d880bbcb78d211043cfead73f3db1
+size 134205
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_bf885e1339d92cc386d1.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_bf885e1339d92cc386d1.png
new file mode 100644
index 0000000000000000000000000000000000000000..8f68427d9557ffbe515f9e696aac833d8769bf7c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_4500_bf885e1339d92cc386d1.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf885e1339d92cc386d12ff933a6b042a9f83815a2f473435cf3337ae3623e11
+size 152408
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_bdd3a8c8c0c8a7a7d4dd.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_bdd3a8c8c0c8a7a7d4dd.png
new file mode 100644
index 0000000000000000000000000000000000000000..96007f6d41f66705495c18834570ebc055c0c490
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_bdd3a8c8c0c8a7a7d4dd.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdd3a8c8c0c8a7a7d4dd9c5151f6f7649ed6527208d08f9f8255652e8b034f16
+size 123734
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_c8333d970fbc70e45c64.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_c8333d970fbc70e45c64.png
new file mode 100644
index 0000000000000000000000000000000000000000..8e5214a6dc17030c736e893a0547d59bed4f452c
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5000_c8333d970fbc70e45c64.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8333d970fbc70e45c642cb23a3e7fd7438ecce32d0be670b3a387f953daecf1
+size 167486
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_3d483725c07baf8663d3.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_3d483725c07baf8663d3.png
new file mode 100644
index 0000000000000000000000000000000000000000..00d6645819dfdd4a9693792021c6d5ff74a0ad29
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_3d483725c07baf8663d3.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d483725c07baf8663d3b6db52b3269fd8031f2ebebafda06080754ba5683d7f
+size 144108
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_b0f06ea56e9a9c08850c.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_b0f06ea56e9a9c08850c.png
new file mode 100644
index 0000000000000000000000000000000000000000..b0df40b09ef1cda6b9cf9273f58db521b78572a8
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_500_b0f06ea56e9a9c08850c.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0f06ea56e9a9c08850cac9ed8053164b762fd6419e22952eb7b1a36ad1da224
+size 148943
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_60d433cf43a3cb8d1412.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_60d433cf43a3cb8d1412.png
new file mode 100644
index 0000000000000000000000000000000000000000..3d134aad94d1ba4ca786fcb26757af5f22e0b782
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_60d433cf43a3cb8d1412.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60d433cf43a3cb8d141289e76d86186829d37e199ad24b3cd9f314d064683b81
+size 179518
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_7cd8c962e4d1b79b5dcc.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_7cd8c962e4d1b79b5dcc.png
new file mode 100644
index 0000000000000000000000000000000000000000..508e841842459242c459c798515a257d3fc8702d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_5500_7cd8c962e4d1b79b5dcc.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cd8c962e4d1b79b5dcc9a9fffa54e2c2c3fc7704df985575327c2ad146acead
+size 175060
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_41402987f48490139945.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_41402987f48490139945.png
new file mode 100644
index 0000000000000000000000000000000000000000..a80e2aa0aa0e3679562e074024cec89b3c3a9454
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_41402987f48490139945.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41402987f48490139945c8630ce10fcfa3044835d82af955b434a4f01cbb0be4
+size 132213
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_c6c41d57fcadc12fd69b.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_c6c41d57fcadc12fd69b.png
new file mode 100644
index 0000000000000000000000000000000000000000..28bdefd5622b7e455c278960cc77ad2d4bc7001f
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6000_c6c41d57fcadc12fd69b.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c41d57fcadc12fd69bff3e72eec586a0132a0fee1b3e93dc6c6f1b5d2c41d1
+size 164647
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_2d21e8a2ea1688bffb9d.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_2d21e8a2ea1688bffb9d.png
new file mode 100644
index 0000000000000000000000000000000000000000..0643baa4f180bfb355192421abae8ea5a0429f31
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_2d21e8a2ea1688bffb9d.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d21e8a2ea1688bffb9dcda19294c42221521cf3c627020900931bdae73005d7
+size 176092
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_a609810c96cec2279a46.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_a609810c96cec2279a46.png
new file mode 100644
index 0000000000000000000000000000000000000000..051836d358de492344ae4844304ac614265f51b4
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_6500_a609810c96cec2279a46.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a609810c96cec2279a467c884847027bef702479b858c86a3e752d4823f68f25
+size 126319
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_4a1fe2fe98784f7b8841.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_4a1fe2fe98784f7b8841.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc30282c0d2ecb0f9c80794ee74c8d8c605a4bf1
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_4a1fe2fe98784f7b8841.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a1fe2fe98784f7b88413b766592c5a488a59591b64999a55556dcfa562776f2
+size 147009
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_6119b9f39242430c319b.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_6119b9f39242430c319b.png
new file mode 100644
index 0000000000000000000000000000000000000000..98ced0718ed5d975db5b40e88bb1324df7e18cd2
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7000_6119b9f39242430c319b.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6119b9f39242430c319b0daceaec879ae9ad9083e344212bdef0e1cd151decc6
+size 136868
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_7e0ee18074e9b8d85c45.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_7e0ee18074e9b8d85c45.png
new file mode 100644
index 0000000000000000000000000000000000000000..d9f84038f5553861fefff104f431fa13a67fb55a
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_7e0ee18074e9b8d85c45.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e0ee18074e9b8d85c455bb9361748e3609fa839ec598d0ba23b72c54cbbd32f
+size 161518
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_b01a808f5a897296f898.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_b01a808f5a897296f898.png
new file mode 100644
index 0000000000000000000000000000000000000000..352dfd00189ac53d2a35c2f541a7d11ed76db53d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_7500_b01a808f5a897296f898.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b01a808f5a897296f898e99594f2fbe5296e949754f2c18ab4c841fbfb3180c1
+size 166963
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_15555bb3e2ce8b16ddcf.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_15555bb3e2ce8b16ddcf.png
new file mode 100644
index 0000000000000000000000000000000000000000..203875ac6f102c9e2fa58ad8be4e7af0cd73898e
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_15555bb3e2ce8b16ddcf.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15555bb3e2ce8b16ddcfa20cef56fe272f4cdba3ee6f67eeb67a180bf2eaba00
+size 138955
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_9652b904aa757dce7aeb.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_9652b904aa757dce7aeb.png
new file mode 100644
index 0000000000000000000000000000000000000000..f7f4dde7d9bbe7acf63ef12ccf92771deb4d73ab
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8000_9652b904aa757dce7aeb.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9652b904aa757dce7aeb7b829d3941c14c3be1223c9043c20ecbfa2adeb780c8
+size 155441
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_51b1e18deefe31476638.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_51b1e18deefe31476638.png
new file mode 100644
index 0000000000000000000000000000000000000000..01ca7c32cd50930ca9b996de5f6a3e8d36c9db2d
Binary files /dev/null and b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_51b1e18deefe31476638.png differ
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_c2d1b91c197ca101b350.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_c2d1b91c197ca101b350.png
new file mode 100644
index 0000000000000000000000000000000000000000..2d73374aa67fe4facf3a3a89ad1aa7063c54a85f
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_8500_c2d1b91c197ca101b350.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2d1b91c197ca101b35012f0071df85cf87e090310cbb4ab1867e35020df33c5
+size 143284
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_03ba3747205343bd9935.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_03ba3747205343bd9935.png
new file mode 100644
index 0000000000000000000000000000000000000000..65a846e6f728e7d27743b9f683fd40cff01f8f1b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_03ba3747205343bd9935.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ba3747205343bd9935e61ab2385fe52d07526e752022a0229a955d695ac70f
+size 176256
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_cc8a6153b15016f58ad3.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_cc8a6153b15016f58ad3.png
new file mode 100644
index 0000000000000000000000000000000000000000..46fb24357fdfa794e12e73fae70f4eef89c56781
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9000_cc8a6153b15016f58ad3.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8a6153b15016f58ad3b04fbbafbb6d504df8a5bd258ccc404656e33977d223
+size 137044
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_342589ce9380e8bb866b.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_342589ce9380e8bb866b.png
new file mode 100644
index 0000000000000000000000000000000000000000..bb62e002af8d68983844dad322e0201440d38faa
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_342589ce9380e8bb866b.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:342589ce9380e8bb866b85d1c53a018795105fe91a580b442f2bb385178278ed
+size 166363
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_753da26bf467a9c41611.png b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_753da26bf467a9c41611.png
new file mode 100644
index 0000000000000000000000000000000000000000..febe374134f9679f855543c80236f716c1901c15
Binary files /dev/null and b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/media/images/generated_videos_first_frame_9500_753da26bf467a9c41611.png differ
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/output.log b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..b5f09f5474afa0167497f0a3e1684ef06d6962ad
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/output.log
@@ -0,0 +1,1234 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 68.66it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/08/2025 15:59:47 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6064.89it/s]
+12/08/2025 15:59:48 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/08/2025 15:59:48 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/08/2025 15:59:48 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 15:59:49 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/08/2025 16:00:04 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 16:00:04 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/08/2025 16:00:06 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/08/2025 16:00:08 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/08/2025 16:00:08 - INFO - __main__ - Wan backbone lr = 0.000030 (base_lr * 0.1)
+12/08/2025 16:00:08 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/08/2025 16:00:08 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/08/2025 16:00:15 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/08/2025 16:00:15 - INFO - train.dataset_utils - Using decord for video loading
+12/08/2025 16:00:15 - INFO - __main__ - Dataloader configuration:
+12/08/2025 16:00:15 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/08/2025 16:00:15 - INFO - __main__ - - prefetch_factor: 2
+12/08/2025 16:00:15 - INFO - __main__ - - persistent_workers: True
+12/08/2025 16:00:15 - INFO - __main__ - - pin_memory: True
+12/08/2025 16:00:15 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/08/2025 16:00:32 - INFO - __main__ - ***** Running training *****
+12/08/2025 16:00:32 - INFO - __main__ - Num training steps = 10000
+12/08/2025 16:00:32 - INFO - __main__ - Instantaneous batch size per device = 2
+12/08/2025 16:00:32 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/08/2025 16:00:32 - INFO - __main__ - Gradient Accumulation steps = 4
+12/08/2025 16:01:18 - INFO - __main__ - Step: 10 Loss: 11.0765 LR: 0.000030
+12/08/2025 16:01:56 - INFO - __main__ - Step: 20 Loss: 11.0747 LR: 0.000030
+12/08/2025 16:02:34 - INFO - __main__ - Step: 30 Loss: 11.0743 LR: 0.000030
+12/08/2025 16:03:12 - INFO - __main__ - Step: 40 Loss: 11.0727 LR: 0.000030
+12/08/2025 16:03:51 - INFO - __main__ - Step: 50 Loss: 11.0682 LR: 0.000030
+12/08/2025 16:04:29 - INFO - __main__ - Step: 60 Loss: 11.0666 LR: 0.000030
+12/08/2025 16:05:07 - INFO - __main__ - Step: 70 Loss: 11.0632 LR: 0.000030
+12/08/2025 16:05:45 - INFO - __main__ - Step: 80 Loss: 11.0599 LR: 0.000030
+12/08/2025 16:06:22 - INFO - __main__ - Step: 90 Loss: 11.0537 LR: 0.000030
+12/08/2025 16:07:00 - INFO - __main__ - Step: 100 Loss: 11.0397 LR: 0.000030
+12/08/2025 16:07:37 - INFO - __main__ - Step: 110 Loss: 11.0273 LR: 0.000030
+12/08/2025 16:08:15 - INFO - __main__ - Step: 120 Loss: 11.0027 LR: 0.000030
+12/08/2025 16:08:52 - INFO - __main__ - Step: 130 Loss: 10.9829 LR: 0.000030
+12/08/2025 16:09:30 - INFO - __main__ - Step: 140 Loss: 10.9128 LR: 0.000030
+12/08/2025 16:10:08 - INFO - __main__ - Step: 150 Loss: 10.8865 LR: 0.000030
+12/08/2025 16:10:46 - INFO - __main__ - Step: 160 Loss: 10.8413 LR: 0.000030
+12/08/2025 16:11:24 - INFO - __main__ - Step: 170 Loss: 10.8309 LR: 0.000030
+12/08/2025 16:12:02 - INFO - __main__ - Step: 180 Loss: 10.7915 LR: 0.000030
+12/08/2025 16:12:40 - INFO - __main__ - Step: 190 Loss: 10.7034 LR: 0.000030
+12/08/2025 16:13:17 - INFO - __main__ - Step: 200 Loss: 10.7173 LR: 0.000030
+12/08/2025 16:13:56 - INFO - __main__ - Step: 210 Loss: 10.6464 LR: 0.000030
+12/08/2025 16:14:35 - INFO - __main__ - Step: 220 Loss: 10.5748 LR: 0.000030
+12/08/2025 16:15:13 - INFO - __main__ - Step: 230 Loss: 10.6039 LR: 0.000030
+12/08/2025 16:15:51 - INFO - __main__ - Step: 240 Loss: 10.5468 LR: 0.000030
+12/08/2025 16:16:29 - INFO - __main__ - Step: 250 Loss: 10.5881 LR: 0.000030
+12/08/2025 16:17:07 - INFO - __main__ - Step: 260 Loss: 10.4530 LR: 0.000030
+12/08/2025 16:17:46 - INFO - __main__ - Step: 270 Loss: 10.4748 LR: 0.000030
+12/08/2025 16:18:23 - INFO - __main__ - Step: 280 Loss: 10.4648 LR: 0.000030
+12/08/2025 16:19:00 - INFO - __main__ - Step: 290 Loss: 10.5356 LR: 0.000030
+12/08/2025 16:19:38 - INFO - __main__ - Step: 300 Loss: 10.4129 LR: 0.000030
+12/08/2025 16:20:16 - INFO - __main__ - Step: 310 Loss: 10.5077 LR: 0.000030
+12/08/2025 16:20:55 - INFO - __main__ - Step: 320 Loss: 10.5066 LR: 0.000030
+12/08/2025 16:21:32 - INFO - __main__ - Step: 330 Loss: 10.4071 LR: 0.000030
+12/08/2025 16:22:12 - INFO - __main__ - Step: 340 Loss: 10.4521 LR: 0.000030
+12/08/2025 16:22:49 - INFO - __main__ - Step: 350 Loss: 10.4234 LR: 0.000030
+12/08/2025 16:23:25 - INFO - __main__ - Step: 360 Loss: 10.2828 LR: 0.000030
+12/08/2025 16:24:02 - INFO - __main__ - Step: 370 Loss: 10.4057 LR: 0.000030
+12/08/2025 16:24:39 - INFO - __main__ - Step: 380 Loss: 10.4306 LR: 0.000030
+12/08/2025 16:25:17 - INFO - __main__ - Step: 390 Loss: 10.4251 LR: 0.000030
+12/08/2025 16:25:54 - INFO - __main__ - Step: 400 Loss: 10.3598 LR: 0.000030
+12/08/2025 16:26:31 - INFO - __main__ - Step: 410 Loss: 10.4499 LR: 0.000030
+12/08/2025 16:27:08 - INFO - __main__ - Step: 420 Loss: 10.3914 LR: 0.000030
+12/08/2025 16:27:47 - INFO - __main__ - Step: 430 Loss: 10.4325 LR: 0.000030
+12/08/2025 16:28:24 - INFO - __main__ - Step: 440 Loss: 10.4138 LR: 0.000030
+12/08/2025 16:29:01 - INFO - __main__ - Step: 450 Loss: 10.4066 LR: 0.000030
+12/08/2025 16:29:39 - INFO - __main__ - Step: 460 Loss: 10.4038 LR: 0.000030
+12/08/2025 16:30:17 - INFO - __main__ - Step: 470 Loss: 10.3804 LR: 0.000030
+12/08/2025 16:30:56 - INFO - __main__ - Step: 480 Loss: 10.4564 LR: 0.000030
+12/08/2025 16:31:35 - INFO - __main__ - Step: 490 Loss: 10.4131 LR: 0.000030
+12/08/2025 16:32:12 - INFO - __main__ - Step: 500 Loss: 10.4588 LR: 0.000030
+12/08/2025 16:32:12 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500
+12/08/2025 16:32:20 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500/optimizer.bin
+12/08/2025 16:32:20 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500/scheduler.bin
+12/08/2025 16:32:20 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500/sampler.bin
+12/08/2025 16:32:20 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500/random_states_0.pkl
+12/08/2025 16:32:20 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-500
+12/08/2025 16:32:20 - INFO - __main__ - Generating videos for validation...
+12/08/2025 16:32:20 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.67it/s]
+12/08/2025 16:32:28 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 16:33:06 - INFO - __main__ - Step: 510 Loss: 10.2678 LR: 0.000030
+12/08/2025 16:33:48 - INFO - __main__ - Step: 520 Loss: 10.3771 LR: 0.000030
+12/08/2025 16:34:26 - INFO - __main__ - Step: 530 Loss: 10.2721 LR: 0.000030
+12/08/2025 16:35:03 - INFO - __main__ - Step: 540 Loss: 10.3496 LR: 0.000030
+12/08/2025 16:35:41 - INFO - __main__ - Step: 550 Loss: 10.3773 LR: 0.000030
+12/08/2025 16:36:19 - INFO - __main__ - Step: 560 Loss: 10.4052 LR: 0.000030
+12/08/2025 16:36:59 - INFO - __main__ - Step: 570 Loss: 10.4429 LR: 0.000030
+12/08/2025 16:37:36 - INFO - __main__ - Step: 580 Loss: 10.3743 LR: 0.000030
+12/08/2025 16:38:14 - INFO - __main__ - Step: 590 Loss: 10.4892 LR: 0.000030
+12/08/2025 16:38:50 - INFO - __main__ - Step: 600 Loss: 10.4520 LR: 0.000030
+12/08/2025 16:39:27 - INFO - __main__ - Step: 610 Loss: 10.3981 LR: 0.000030
+12/08/2025 16:40:05 - INFO - __main__ - Step: 620 Loss: 10.3463 LR: 0.000030
+12/08/2025 16:40:42 - INFO - __main__ - Step: 630 Loss: 10.4125 LR: 0.000030
+12/08/2025 16:41:21 - INFO - __main__ - Step: 640 Loss: 10.4231 LR: 0.000030
+12/08/2025 16:41:58 - INFO - __main__ - Step: 650 Loss: 10.3965 LR: 0.000030
+12/08/2025 16:42:37 - INFO - __main__ - Step: 660 Loss: 10.3814 LR: 0.000030
+12/08/2025 16:43:15 - INFO - __main__ - Step: 670 Loss: 10.3961 LR: 0.000030
+12/08/2025 16:43:52 - INFO - __main__ - Step: 680 Loss: 10.4251 LR: 0.000030
+12/08/2025 16:44:29 - INFO - __main__ - Step: 690 Loss: 10.4327 LR: 0.000030
+12/08/2025 16:45:07 - INFO - __main__ - Step: 700 Loss: 10.2706 LR: 0.000030
+12/08/2025 16:45:44 - INFO - __main__ - Step: 710 Loss: 10.3643 LR: 0.000030
+12/08/2025 16:46:24 - INFO - __main__ - Step: 720 Loss: 10.4459 LR: 0.000030
+12/08/2025 16:47:02 - INFO - __main__ - Step: 730 Loss: 10.4146 LR: 0.000030
+12/08/2025 16:47:40 - INFO - __main__ - Step: 740 Loss: 10.3954 LR: 0.000030
+12/08/2025 16:48:20 - INFO - __main__ - Step: 750 Loss: 10.3323 LR: 0.000030
+12/08/2025 16:48:59 - INFO - __main__ - Step: 760 Loss: 10.3137 LR: 0.000030
+12/08/2025 16:49:37 - INFO - __main__ - Step: 770 Loss: 10.3287 LR: 0.000030
+12/08/2025 16:50:15 - INFO - __main__ - Step: 780 Loss: 10.3346 LR: 0.000030
+12/08/2025 16:50:52 - INFO - __main__ - Step: 790 Loss: 10.3639 LR: 0.000030
+12/08/2025 16:51:29 - INFO - __main__ - Step: 800 Loss: 10.2505 LR: 0.000030
+12/08/2025 16:52:07 - INFO - __main__ - Step: 810 Loss: 10.4047 LR: 0.000030
+12/08/2025 16:52:45 - INFO - __main__ - Step: 820 Loss: 10.4114 LR: 0.000030
+12/08/2025 16:53:23 - INFO - __main__ - Step: 830 Loss: 10.3459 LR: 0.000030
+12/08/2025 16:54:01 - INFO - __main__ - Step: 840 Loss: 10.2657 LR: 0.000030
+12/08/2025 16:54:38 - INFO - __main__ - Step: 850 Loss: 10.4799 LR: 0.000030
+12/08/2025 16:55:16 - INFO - __main__ - Step: 860 Loss: 10.3322 LR: 0.000030
+12/08/2025 16:55:57 - INFO - __main__ - Step: 870 Loss: 10.2937 LR: 0.000030
+12/08/2025 16:56:34 - INFO - __main__ - Step: 880 Loss: 10.4723 LR: 0.000030
+12/08/2025 16:57:11 - INFO - __main__ - Step: 890 Loss: 10.2729 LR: 0.000030
+12/08/2025 16:57:47 - INFO - __main__ - Step: 900 Loss: 10.3868 LR: 0.000030
+12/08/2025 16:58:25 - INFO - __main__ - Step: 910 Loss: 10.4276 LR: 0.000030
+12/08/2025 16:59:01 - INFO - __main__ - Step: 920 Loss: 10.3418 LR: 0.000030
+12/08/2025 16:59:39 - INFO - __main__ - Step: 930 Loss: 10.4002 LR: 0.000030
+12/08/2025 17:00:18 - INFO - __main__ - Step: 940 Loss: 10.3860 LR: 0.000030
+12/08/2025 17:00:55 - INFO - __main__ - Step: 950 Loss: 10.3090 LR: 0.000030
+12/08/2025 17:01:31 - INFO - __main__ - Step: 960 Loss: 10.3547 LR: 0.000030
+12/08/2025 17:02:10 - INFO - __main__ - Step: 970 Loss: 10.4002 LR: 0.000030
+12/08/2025 17:02:47 - INFO - __main__ - Step: 980 Loss: 10.3720 LR: 0.000030
+12/08/2025 17:03:23 - INFO - __main__ - Step: 990 Loss: 10.3952 LR: 0.000030
+12/08/2025 17:04:00 - INFO - __main__ - Step: 1000 Loss: 10.3774 LR: 0.000030
+12/08/2025 17:04:00 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000
+12/08/2025 17:04:08 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000/optimizer.bin
+12/08/2025 17:04:08 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000/scheduler.bin
+12/08/2025 17:04:08 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000/sampler.bin
+12/08/2025 17:04:08 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000/random_states_0.pkl
+12/08/2025 17:04:08 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1000
+12/08/2025 17:04:08 - INFO - __main__ - Generating videos for validation...
+12/08/2025 17:04:08 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.31it/s]
+12/08/2025 17:04:16 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 17:04:56 - INFO - __main__ - Step: 1010 Loss: 10.3488 LR: 0.000030
+12/08/2025 17:05:38 - INFO - __main__ - Step: 1020 Loss: 10.4207 LR: 0.000030
+12/08/2025 17:06:15 - INFO - __main__ - Step: 1030 Loss: 10.2589 LR: 0.000030
+12/08/2025 17:06:54 - INFO - __main__ - Step: 1040 Loss: 10.4372 LR: 0.000030
+12/08/2025 17:07:30 - INFO - __main__ - Step: 1050 Loss: 10.3223 LR: 0.000030
+12/08/2025 17:08:10 - INFO - __main__ - Step: 1060 Loss: 10.3129 LR: 0.000030
+12/08/2025 17:08:57 - INFO - __main__ - Step: 1070 Loss: 10.3165 LR: 0.000030
+12/08/2025 17:09:34 - INFO - __main__ - Step: 1080 Loss: 10.4149 LR: 0.000030
+12/08/2025 17:10:10 - INFO - __main__ - Step: 1090 Loss: 10.2954 LR: 0.000030
+12/08/2025 17:10:48 - INFO - __main__ - Step: 1100 Loss: 10.3628 LR: 0.000030
+12/08/2025 17:11:25 - INFO - __main__ - Step: 1110 Loss: 10.3230 LR: 0.000030
+12/08/2025 17:12:01 - INFO - __main__ - Step: 1120 Loss: 10.3802 LR: 0.000030
+12/08/2025 17:12:38 - INFO - __main__ - Step: 1130 Loss: 10.4042 LR: 0.000030
+12/08/2025 17:13:15 - INFO - __main__ - Step: 1140 Loss: 10.3732 LR: 0.000030
+12/08/2025 17:13:52 - INFO - __main__ - Step: 1150 Loss: 10.2436 LR: 0.000030
+12/08/2025 17:14:31 - INFO - __main__ - Step: 1160 Loss: 10.3452 LR: 0.000030
+12/08/2025 17:15:09 - INFO - __main__ - Step: 1170 Loss: 10.3288 LR: 0.000030
+12/08/2025 17:15:47 - INFO - __main__ - Step: 1180 Loss: 10.3345 LR: 0.000030
+12/08/2025 17:16:25 - INFO - __main__ - Step: 1190 Loss: 10.3835 LR: 0.000030
+12/08/2025 17:17:02 - INFO - __main__ - Step: 1200 Loss: 10.2665 LR: 0.000030
+12/08/2025 17:17:41 - INFO - __main__ - Step: 1210 Loss: 10.3739 LR: 0.000030
+12/08/2025 17:18:19 - INFO - __main__ - Step: 1220 Loss: 10.4482 LR: 0.000030
+12/08/2025 17:19:02 - INFO - __main__ - Step: 1230 Loss: 10.1238 LR: 0.000030
+12/08/2025 17:19:39 - INFO - __main__ - Step: 1240 Loss: 10.3839 LR: 0.000030
+12/08/2025 17:20:16 - INFO - __main__ - Step: 1250 Loss: 10.3929 LR: 0.000030
+12/08/2025 17:20:52 - INFO - __main__ - Step: 1260 Loss: 10.2856 LR: 0.000030
+12/08/2025 17:21:29 - INFO - __main__ - Step: 1270 Loss: 10.3958 LR: 0.000030
+12/08/2025 17:22:07 - INFO - __main__ - Step: 1280 Loss: 10.4225 LR: 0.000030
+12/08/2025 17:22:44 - INFO - __main__ - Step: 1290 Loss: 10.3787 LR: 0.000030
+12/08/2025 17:23:21 - INFO - __main__ - Step: 1300 Loss: 10.2556 LR: 0.000030
+12/08/2025 17:23:58 - INFO - __main__ - Step: 1310 Loss: 10.3971 LR: 0.000030
+12/08/2025 17:24:34 - INFO - __main__ - Step: 1320 Loss: 10.1756 LR: 0.000030
+12/08/2025 17:25:14 - INFO - __main__ - Step: 1330 Loss: 10.3488 LR: 0.000030
+12/08/2025 17:25:50 - INFO - __main__ - Step: 1340 Loss: 10.3324 LR: 0.000030
+12/08/2025 17:26:27 - INFO - __main__ - Step: 1350 Loss: 10.3611 LR: 0.000030
+12/08/2025 17:27:04 - INFO - __main__ - Step: 1360 Loss: 10.3655 LR: 0.000030
+12/08/2025 17:27:41 - INFO - __main__ - Step: 1370 Loss: 10.1179 LR: 0.000030
+12/08/2025 17:28:17 - INFO - __main__ - Step: 1380 Loss: 10.3356 LR: 0.000030
+12/08/2025 17:28:54 - INFO - __main__ - Step: 1390 Loss: 10.3898 LR: 0.000030
+12/08/2025 17:29:31 - INFO - __main__ - Step: 1400 Loss: 10.3390 LR: 0.000030
+12/08/2025 17:30:11 - INFO - __main__ - Step: 1410 Loss: 10.1545 LR: 0.000030
+12/08/2025 17:30:49 - INFO - __main__ - Step: 1420 Loss: 10.4155 LR: 0.000030
+12/08/2025 17:31:25 - INFO - __main__ - Step: 1430 Loss: 10.3101 LR: 0.000030
+12/08/2025 17:32:02 - INFO - __main__ - Step: 1440 Loss: 10.1570 LR: 0.000030
+12/08/2025 17:32:38 - INFO - __main__ - Step: 1450 Loss: 10.3250 LR: 0.000030
+12/08/2025 17:33:15 - INFO - __main__ - Step: 1460 Loss: 10.3853 LR: 0.000030
+12/08/2025 17:33:51 - INFO - __main__ - Step: 1470 Loss: 10.3949 LR: 0.000030
+12/08/2025 17:34:28 - INFO - __main__ - Step: 1480 Loss: 10.3797 LR: 0.000030
+12/08/2025 17:35:05 - INFO - __main__ - Step: 1490 Loss: 10.3865 LR: 0.000030
+12/08/2025 17:35:41 - INFO - __main__ - Step: 1500 Loss: 10.3832 LR: 0.000030
+12/08/2025 17:35:41 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500
+12/08/2025 17:35:51 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500/optimizer.bin
+12/08/2025 17:35:51 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500/scheduler.bin
+12/08/2025 17:35:51 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500/sampler.bin
+12/08/2025 17:35:51 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500/random_states_0.pkl
+12/08/2025 17:35:51 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-1500
+12/08/2025 17:35:51 - INFO - __main__ - Generating videos for validation...
+12/08/2025 17:35:51 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.66it/s]
+12/08/2025 17:35:58 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 17:36:36 - INFO - __main__ - Step: 1510 Loss: 10.2973 LR: 0.000030
+12/08/2025 17:37:13 - INFO - __main__ - Step: 1520 Loss: 10.1411 LR: 0.000030
+12/08/2025 17:37:53 - INFO - __main__ - Step: 1530 Loss: 10.2939 LR: 0.000030
+12/08/2025 17:38:30 - INFO - __main__ - Step: 1540 Loss: 10.4230 LR: 0.000030
+12/08/2025 17:39:07 - INFO - __main__ - Step: 1550 Loss: 10.4209 LR: 0.000030
+12/08/2025 17:39:46 - INFO - __main__ - Step: 1560 Loss: 10.2173 LR: 0.000030
+12/08/2025 17:40:23 - INFO - __main__ - Step: 1570 Loss: 10.3776 LR: 0.000030
+12/08/2025 17:41:00 - INFO - __main__ - Step: 1580 Loss: 10.2596 LR: 0.000030
+12/08/2025 17:41:39 - INFO - __main__ - Step: 1590 Loss: 10.3701 LR: 0.000030
+12/08/2025 17:42:17 - INFO - __main__ - Step: 1600 Loss: 10.3020 LR: 0.000030
+12/08/2025 17:42:55 - INFO - __main__ - Step: 1610 Loss: 10.3507 LR: 0.000030
+12/08/2025 17:43:34 - INFO - __main__ - Step: 1620 Loss: 10.4234 LR: 0.000030
+12/08/2025 17:44:12 - INFO - __main__ - Step: 1630 Loss: 10.3653 LR: 0.000030
+12/08/2025 17:44:51 - INFO - __main__ - Step: 1640 Loss: 10.3003 LR: 0.000030
+12/08/2025 17:45:30 - INFO - __main__ - Step: 1650 Loss: 10.2873 LR: 0.000030
+12/08/2025 17:46:09 - INFO - __main__ - Step: 1660 Loss: 10.2909 LR: 0.000030
+12/08/2025 17:46:46 - INFO - __main__ - Step: 1670 Loss: 10.3515 LR: 0.000030
+12/08/2025 17:47:25 - INFO - __main__ - Step: 1680 Loss: 10.0193 LR: 0.000030
+12/08/2025 17:48:05 - INFO - __main__ - Step: 1690 Loss: 10.4187 LR: 0.000030
+12/08/2025 17:48:44 - INFO - __main__ - Step: 1700 Loss: 10.4093 LR: 0.000030
+12/08/2025 17:49:23 - INFO - __main__ - Step: 1710 Loss: 10.2773 LR: 0.000030
+12/08/2025 17:50:03 - INFO - __main__ - Step: 1720 Loss: 10.3371 LR: 0.000030
+12/08/2025 17:50:43 - INFO - __main__ - Step: 1730 Loss: 10.3882 LR: 0.000030
+12/08/2025 17:51:22 - INFO - __main__ - Step: 1740 Loss: 10.3619 LR: 0.000030
+12/08/2025 17:52:02 - INFO - __main__ - Step: 1750 Loss: 10.3271 LR: 0.000030
+12/08/2025 17:52:41 - INFO - __main__ - Step: 1760 Loss: 10.4465 LR: 0.000030
+12/08/2025 17:53:18 - INFO - __main__ - Step: 1770 Loss: 10.3722 LR: 0.000030
+12/08/2025 17:53:57 - INFO - __main__ - Step: 1780 Loss: 10.2072 LR: 0.000030
+12/08/2025 17:54:35 - INFO - __main__ - Step: 1790 Loss: 10.4272 LR: 0.000030
+12/08/2025 17:55:14 - INFO - __main__ - Step: 1800 Loss: 10.3846 LR: 0.000030
+12/08/2025 17:55:53 - INFO - __main__ - Step: 1810 Loss: 10.4292 LR: 0.000030
+12/08/2025 17:56:32 - INFO - __main__ - Step: 1820 Loss: 10.3997 LR: 0.000030
+12/08/2025 17:57:12 - INFO - __main__ - Step: 1830 Loss: 10.3803 LR: 0.000030
+12/08/2025 17:57:50 - INFO - __main__ - Step: 1840 Loss: 10.1679 LR: 0.000030
+12/08/2025 17:58:30 - INFO - __main__ - Step: 1850 Loss: 10.4325 LR: 0.000030
+12/08/2025 17:59:08 - INFO - __main__ - Step: 1860 Loss: 10.3337 LR: 0.000030
+12/08/2025 17:59:46 - INFO - __main__ - Step: 1870 Loss: 10.3278 LR: 0.000030
+12/08/2025 18:00:28 - INFO - __main__ - Step: 1880 Loss: 10.2863 LR: 0.000030
+12/08/2025 18:01:06 - INFO - __main__ - Step: 1890 Loss: 10.2458 LR: 0.000030
+12/08/2025 18:01:46 - INFO - __main__ - Step: 1900 Loss: 10.3747 LR: 0.000030
+12/08/2025 18:02:24 - INFO - __main__ - Step: 1910 Loss: 10.4138 LR: 0.000030
+12/08/2025 18:03:03 - INFO - __main__ - Step: 1920 Loss: 10.4281 LR: 0.000030
+12/08/2025 18:03:42 - INFO - __main__ - Step: 1930 Loss: 10.3979 LR: 0.000030
+12/08/2025 18:04:20 - INFO - __main__ - Step: 1940 Loss: 10.3887 LR: 0.000030
+12/08/2025 18:05:00 - INFO - __main__ - Step: 1950 Loss: 10.2837 LR: 0.000030
+12/08/2025 18:05:39 - INFO - __main__ - Step: 1960 Loss: 10.4395 LR: 0.000030
+12/08/2025 18:06:19 - INFO - __main__ - Step: 1970 Loss: 10.4266 LR: 0.000030
+12/08/2025 18:06:59 - INFO - __main__ - Step: 1980 Loss: 10.3031 LR: 0.000030
+12/08/2025 18:07:38 - INFO - __main__ - Step: 1990 Loss: 10.4140 LR: 0.000030
+12/08/2025 18:08:19 - INFO - __main__ - Step: 2000 Loss: 10.4107 LR: 0.000030
+12/08/2025 18:08:19 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000
+12/08/2025 18:08:27 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000/optimizer.bin
+12/08/2025 18:08:27 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000/scheduler.bin
+12/08/2025 18:08:27 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000/sampler.bin
+12/08/2025 18:08:27 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000/random_states_0.pkl
+12/08/2025 18:08:27 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2000
+12/08/2025 18:08:27 - INFO - __main__ - Generating videos for validation...
+12/08/2025 18:08:27 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.82it/s]
+12/08/2025 18:08:33 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 18:09:12 - INFO - __main__ - Step: 2010 Loss: 10.3039 LR: 0.000030
+12/08/2025 18:09:54 - INFO - __main__ - Step: 2020 Loss: 10.3751 LR: 0.000030
+12/08/2025 18:10:33 - INFO - __main__ - Step: 2030 Loss: 10.3716 LR: 0.000030
+12/08/2025 18:11:12 - INFO - __main__ - Step: 2040 Loss: 10.3147 LR: 0.000030
+12/08/2025 18:11:50 - INFO - __main__ - Step: 2050 Loss: 10.3584 LR: 0.000030
+12/08/2025 18:12:29 - INFO - __main__ - Step: 2060 Loss: 10.3668 LR: 0.000030
+12/08/2025 18:13:08 - INFO - __main__ - Step: 2070 Loss: 10.2852 LR: 0.000030
+12/08/2025 18:13:47 - INFO - __main__ - Step: 2080 Loss: 10.3371 LR: 0.000030
+12/08/2025 18:14:27 - INFO - __main__ - Step: 2090 Loss: 10.4287 LR: 0.000030
+12/08/2025 18:15:06 - INFO - __main__ - Step: 2100 Loss: 10.3314 LR: 0.000030
+12/08/2025 18:15:44 - INFO - __main__ - Step: 2110 Loss: 10.3791 LR: 0.000030
+12/08/2025 18:16:23 - INFO - __main__ - Step: 2120 Loss: 10.3936 LR: 0.000030
+12/08/2025 18:17:02 - INFO - __main__ - Step: 2130 Loss: 10.3446 LR: 0.000030
+12/08/2025 18:17:41 - INFO - __main__ - Step: 2140 Loss: 10.3441 LR: 0.000030
+12/08/2025 18:18:18 - INFO - __main__ - Step: 2150 Loss: 10.2765 LR: 0.000030
+12/08/2025 18:18:57 - INFO - __main__ - Step: 2160 Loss: 10.2760 LR: 0.000030
+12/08/2025 18:19:35 - INFO - __main__ - Step: 2170 Loss: 10.4445 LR: 0.000030
+12/08/2025 18:20:14 - INFO - __main__ - Step: 2180 Loss: 10.4068 LR: 0.000030
+12/08/2025 18:20:54 - INFO - __main__ - Step: 2190 Loss: 10.3239 LR: 0.000030
+12/08/2025 18:21:32 - INFO - __main__ - Step: 2200 Loss: 10.3521 LR: 0.000030
+12/08/2025 18:22:11 - INFO - __main__ - Step: 2210 Loss: 10.3568 LR: 0.000030
+12/08/2025 18:22:50 - INFO - __main__ - Step: 2220 Loss: 10.1969 LR: 0.000030
+12/08/2025 18:23:29 - INFO - __main__ - Step: 2230 Loss: 10.3199 LR: 0.000030
+12/08/2025 18:24:07 - INFO - __main__ - Step: 2240 Loss: 10.2889 LR: 0.000030
+12/08/2025 18:24:45 - INFO - __main__ - Step: 2250 Loss: 10.3757 LR: 0.000030
+12/08/2025 18:25:25 - INFO - __main__ - Step: 2260 Loss: 10.3597 LR: 0.000030
+12/08/2025 18:26:03 - INFO - __main__ - Step: 2270 Loss: 10.3497 LR: 0.000030
+12/08/2025 18:26:42 - INFO - __main__ - Step: 2280 Loss: 10.3869 LR: 0.000030
+12/08/2025 18:27:20 - INFO - __main__ - Step: 2290 Loss: 10.4222 LR: 0.000030
+12/08/2025 18:27:56 - INFO - __main__ - Step: 2300 Loss: 10.3973 LR: 0.000030
+12/08/2025 18:28:35 - INFO - __main__ - Step: 2310 Loss: 10.2908 LR: 0.000030
+12/08/2025 18:29:14 - INFO - __main__ - Step: 2320 Loss: 10.4200 LR: 0.000030
+12/08/2025 18:29:52 - INFO - __main__ - Step: 2330 Loss: 10.1784 LR: 0.000030
+12/08/2025 18:30:31 - INFO - __main__ - Step: 2340 Loss: 10.2928 LR: 0.000030
+12/08/2025 18:31:09 - INFO - __main__ - Step: 2350 Loss: 10.1785 LR: 0.000030
+12/08/2025 18:31:47 - INFO - __main__ - Step: 2360 Loss: 10.0671 LR: 0.000030
+12/08/2025 18:32:25 - INFO - __main__ - Step: 2370 Loss: 10.3262 LR: 0.000030
+12/08/2025 18:33:02 - INFO - __main__ - Step: 2380 Loss: 10.3750 LR: 0.000030
+12/08/2025 18:33:41 - INFO - __main__ - Step: 2390 Loss: 10.3780 LR: 0.000030
+12/08/2025 18:34:22 - INFO - __main__ - Step: 2400 Loss: 10.3725 LR: 0.000030
+12/08/2025 18:35:01 - INFO - __main__ - Step: 2410 Loss: 10.3987 LR: 0.000030
+12/08/2025 18:35:41 - INFO - __main__ - Step: 2420 Loss: 10.3628 LR: 0.000030
+12/08/2025 18:36:20 - INFO - __main__ - Step: 2430 Loss: 10.2904 LR: 0.000030
+12/08/2025 18:36:57 - INFO - __main__ - Step: 2440 Loss: 10.1454 LR: 0.000030
+12/08/2025 18:37:36 - INFO - __main__ - Step: 2450 Loss: 10.3886 LR: 0.000030
+12/08/2025 18:38:14 - INFO - __main__ - Step: 2460 Loss: 10.3994 LR: 0.000030
+12/08/2025 18:38:55 - INFO - __main__ - Step: 2470 Loss: 10.3844 LR: 0.000030
+12/08/2025 18:39:34 - INFO - __main__ - Step: 2480 Loss: 10.3436 LR: 0.000030
+12/08/2025 18:40:12 - INFO - __main__ - Step: 2490 Loss: 10.2371 LR: 0.000030
+12/08/2025 18:40:51 - INFO - __main__ - Step: 2500 Loss: 10.3769 LR: 0.000030
+12/08/2025 18:40:51 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500
+12/08/2025 18:40:59 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500/optimizer.bin
+12/08/2025 18:40:59 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500/scheduler.bin
+12/08/2025 18:40:59 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500/sampler.bin
+12/08/2025 18:40:59 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500/random_states_0.pkl
+12/08/2025 18:40:59 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-2500
+12/08/2025 18:40:59 - INFO - __main__ - Generating videos for validation...
+12/08/2025 18:40:59 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.74it/s]
+12/08/2025 18:41:05 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 18:41:44 - INFO - __main__ - Step: 2510 Loss: 10.4104 LR: 0.000030
+12/08/2025 18:42:23 - INFO - __main__ - Step: 2520 Loss: 10.3128 LR: 0.000030
+12/08/2025 18:43:01 - INFO - __main__ - Step: 2530 Loss: 10.3471 LR: 0.000030
+12/08/2025 18:43:41 - INFO - __main__ - Step: 2540 Loss: 10.2331 LR: 0.000030
+12/08/2025 18:44:20 - INFO - __main__ - Step: 2550 Loss: 10.4053 LR: 0.000030
+12/08/2025 18:45:00 - INFO - __main__ - Step: 2560 Loss: 10.3362 LR: 0.000030
+12/08/2025 18:45:39 - INFO - __main__ - Step: 2570 Loss: 10.1949 LR: 0.000030
+12/08/2025 18:46:18 - INFO - __main__ - Step: 2580 Loss: 10.3445 LR: 0.000030
+12/08/2025 18:46:58 - INFO - __main__ - Step: 2590 Loss: 10.2847 LR: 0.000030
+12/08/2025 18:47:38 - INFO - __main__ - Step: 2600 Loss: 10.3255 LR: 0.000030
+12/08/2025 18:48:17 - INFO - __main__ - Step: 2610 Loss: 10.3907 LR: 0.000030
+12/08/2025 18:48:56 - INFO - __main__ - Step: 2620 Loss: 10.2480 LR: 0.000030
+12/08/2025 18:49:35 - INFO - __main__ - Step: 2630 Loss: 10.3263 LR: 0.000030
+12/08/2025 18:50:15 - INFO - __main__ - Step: 2640 Loss: 10.1264 LR: 0.000030
+12/08/2025 18:50:55 - INFO - __main__ - Step: 2650 Loss: 10.4146 LR: 0.000030
+12/08/2025 18:51:35 - INFO - __main__ - Step: 2660 Loss: 10.3681 LR: 0.000030
+12/08/2025 18:52:13 - INFO - __main__ - Step: 2670 Loss: 10.2980 LR: 0.000030
+12/08/2025 18:52:53 - INFO - __main__ - Step: 2680 Loss: 10.3355 LR: 0.000030
+12/08/2025 18:53:32 - INFO - __main__ - Step: 2690 Loss: 10.2069 LR: 0.000030
+12/08/2025 18:54:12 - INFO - __main__ - Step: 2700 Loss: 10.3311 LR: 0.000030
+12/08/2025 18:54:52 - INFO - __main__ - Step: 2710 Loss: 10.3150 LR: 0.000030
+12/08/2025 18:55:31 - INFO - __main__ - Step: 2720 Loss: 10.2567 LR: 0.000030
+12/08/2025 18:56:11 - INFO - __main__ - Step: 2730 Loss: 10.2787 LR: 0.000030
+12/08/2025 18:56:50 - INFO - __main__ - Step: 2740 Loss: 10.3526 LR: 0.000030
+12/08/2025 18:57:29 - INFO - __main__ - Step: 2750 Loss: 10.3628 LR: 0.000030
+12/08/2025 18:58:08 - INFO - __main__ - Step: 2760 Loss: 10.2796 LR: 0.000030
+12/08/2025 18:58:47 - INFO - __main__ - Step: 2770 Loss: 10.2616 LR: 0.000030
+12/08/2025 18:59:27 - INFO - __main__ - Step: 2780 Loss: 10.3035 LR: 0.000030
+12/08/2025 19:00:05 - INFO - __main__ - Step: 2790 Loss: 10.3651 LR: 0.000030
+12/08/2025 19:00:45 - INFO - __main__ - Step: 2800 Loss: 10.3469 LR: 0.000030
+12/08/2025 19:01:25 - INFO - __main__ - Step: 2810 Loss: 10.3005 LR: 0.000030
+12/08/2025 19:02:05 - INFO - __main__ - Step: 2820 Loss: 10.2700 LR: 0.000030
+12/08/2025 19:02:44 - INFO - __main__ - Step: 2830 Loss: 10.3371 LR: 0.000030
+12/08/2025 19:03:24 - INFO - __main__ - Step: 2840 Loss: 10.3848 LR: 0.000030
+12/08/2025 19:04:04 - INFO - __main__ - Step: 2850 Loss: 10.3789 LR: 0.000030
+12/08/2025 19:04:42 - INFO - __main__ - Step: 2860 Loss: 10.3298 LR: 0.000030
+12/08/2025 19:05:22 - INFO - __main__ - Step: 2870 Loss: 10.3286 LR: 0.000030
+12/08/2025 19:06:00 - INFO - __main__ - Step: 2880 Loss: 10.2525 LR: 0.000030
+12/08/2025 19:06:39 - INFO - __main__ - Step: 2890 Loss: 10.4446 LR: 0.000030
+12/08/2025 19:07:19 - INFO - __main__ - Step: 2900 Loss: 10.3086 LR: 0.000030
+12/08/2025 19:07:59 - INFO - __main__ - Step: 2910 Loss: 10.3149 LR: 0.000030
+12/08/2025 19:08:39 - INFO - __main__ - Step: 2920 Loss: 10.3575 LR: 0.000030
+12/08/2025 19:09:18 - INFO - __main__ - Step: 2930 Loss: 10.2278 LR: 0.000030
+12/08/2025 19:09:57 - INFO - __main__ - Step: 2940 Loss: 10.2349 LR: 0.000030
+12/08/2025 19:10:36 - INFO - __main__ - Step: 2950 Loss: 10.4023 LR: 0.000030
+12/08/2025 19:11:15 - INFO - __main__ - Step: 2960 Loss: 10.2979 LR: 0.000030
+12/08/2025 19:11:54 - INFO - __main__ - Step: 2970 Loss: 10.3096 LR: 0.000030
+12/08/2025 19:12:35 - INFO - __main__ - Step: 2980 Loss: 10.3134 LR: 0.000030
+12/08/2025 19:13:13 - INFO - __main__ - Step: 2990 Loss: 10.3931 LR: 0.000030
+12/08/2025 19:13:54 - INFO - __main__ - Step: 3000 Loss: 10.3829 LR: 0.000030
+12/08/2025 19:13:54 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000
+12/08/2025 19:14:02 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000/optimizer.bin
+12/08/2025 19:14:02 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000/scheduler.bin
+12/08/2025 19:14:02 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000/sampler.bin
+12/08/2025 19:14:02 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000/random_states_0.pkl
+12/08/2025 19:14:02 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3000
+12/08/2025 19:14:02 - INFO - __main__ - Generating videos for validation...
+12/08/2025 19:14:02 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.79it/s]
+12/08/2025 19:14:08 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 19:14:49 - INFO - __main__ - Step: 3010 Loss: 10.2929 LR: 0.000030
+12/08/2025 19:15:32 - INFO - __main__ - Step: 3020 Loss: 10.4008 LR: 0.000030
+12/08/2025 19:16:12 - INFO - __main__ - Step: 3030 Loss: 10.3054 LR: 0.000030
+12/08/2025 19:16:52 - INFO - __main__ - Step: 3040 Loss: 10.3365 LR: 0.000030
+12/08/2025 19:17:36 - INFO - __main__ - Step: 3050 Loss: 10.3967 LR: 0.000030
+12/08/2025 19:18:16 - INFO - __main__ - Step: 3060 Loss: 10.2645 LR: 0.000030
+12/08/2025 19:18:55 - INFO - __main__ - Step: 3070 Loss: 10.1784 LR: 0.000030
+12/08/2025 19:19:35 - INFO - __main__ - Step: 3080 Loss: 9.6810 LR: 0.000030
+12/08/2025 19:20:15 - INFO - __main__ - Step: 3090 Loss: 10.3777 LR: 0.000030
+12/08/2025 19:20:55 - INFO - __main__ - Step: 3100 Loss: 10.3118 LR: 0.000030
+12/08/2025 19:21:36 - INFO - __main__ - Step: 3110 Loss: 10.3528 LR: 0.000030
+12/08/2025 19:22:21 - INFO - __main__ - Step: 3120 Loss: 10.2164 LR: 0.000030
+12/08/2025 19:23:01 - INFO - __main__ - Step: 3130 Loss: 10.3696 LR: 0.000030
+12/08/2025 19:23:42 - INFO - __main__ - Step: 3140 Loss: 10.2756 LR: 0.000030
+12/08/2025 19:24:21 - INFO - __main__ - Step: 3150 Loss: 10.2743 LR: 0.000030
+12/08/2025 19:25:02 - INFO - __main__ - Step: 3160 Loss: 10.4775 LR: 0.000030
+12/08/2025 19:25:42 - INFO - __main__ - Step: 3170 Loss: 10.3352 LR: 0.000030
+12/08/2025 19:26:23 - INFO - __main__ - Step: 3180 Loss: 10.2950 LR: 0.000030
+12/08/2025 19:27:04 - INFO - __main__ - Step: 3190 Loss: 10.3197 LR: 0.000030
+12/08/2025 19:27:44 - INFO - __main__ - Step: 3200 Loss: 10.4200 LR: 0.000030
+12/08/2025 19:28:25 - INFO - __main__ - Step: 3210 Loss: 10.4490 LR: 0.000030
+12/08/2025 19:29:05 - INFO - __main__ - Step: 3220 Loss: 10.2776 LR: 0.000030
+12/08/2025 19:29:44 - INFO - __main__ - Step: 3230 Loss: 10.3765 LR: 0.000030
+12/08/2025 19:30:25 - INFO - __main__ - Step: 3240 Loss: 10.4211 LR: 0.000030
+12/08/2025 19:31:04 - INFO - __main__ - Step: 3250 Loss: 10.1657 LR: 0.000030
+12/08/2025 19:31:44 - INFO - __main__ - Step: 3260 Loss: 10.3544 LR: 0.000030
+12/08/2025 19:32:25 - INFO - __main__ - Step: 3270 Loss: 10.3862 LR: 0.000030
+12/08/2025 19:33:06 - INFO - __main__ - Step: 3280 Loss: 10.3992 LR: 0.000030
+12/08/2025 19:33:47 - INFO - __main__ - Step: 3290 Loss: 10.3372 LR: 0.000030
+12/08/2025 19:34:28 - INFO - __main__ - Step: 3300 Loss: 10.2070 LR: 0.000030
+12/08/2025 19:35:08 - INFO - __main__ - Step: 3310 Loss: 10.2486 LR: 0.000030
+12/08/2025 19:35:50 - INFO - __main__ - Step: 3320 Loss: 10.4331 LR: 0.000030
+12/08/2025 19:36:31 - INFO - __main__ - Step: 3330 Loss: 10.2179 LR: 0.000030
+12/08/2025 19:37:11 - INFO - __main__ - Step: 3340 Loss: 10.3287 LR: 0.000030
+12/08/2025 19:37:52 - INFO - __main__ - Step: 3350 Loss: 10.2189 LR: 0.000030
+12/08/2025 19:38:31 - INFO - __main__ - Step: 3360 Loss: 10.3541 LR: 0.000030
+12/08/2025 19:39:11 - INFO - __main__ - Step: 3370 Loss: 10.1355 LR: 0.000030
+12/08/2025 19:39:51 - INFO - __main__ - Step: 3380 Loss: 10.1508 LR: 0.000030
+12/08/2025 19:40:31 - INFO - __main__ - Step: 3390 Loss: 10.2778 LR: 0.000030
+12/08/2025 19:41:12 - INFO - __main__ - Step: 3400 Loss: 10.2547 LR: 0.000030
+12/08/2025 19:41:52 - INFO - __main__ - Step: 3410 Loss: 10.2936 LR: 0.000030
+12/08/2025 19:42:34 - INFO - __main__ - Step: 3420 Loss: 10.2440 LR: 0.000030
+12/08/2025 19:43:14 - INFO - __main__ - Step: 3430 Loss: 10.2664 LR: 0.000030
+12/08/2025 19:43:54 - INFO - __main__ - Step: 3440 Loss: 10.2321 LR: 0.000030
+12/08/2025 19:44:34 - INFO - __main__ - Step: 3450 Loss: 10.2659 LR: 0.000030
+12/08/2025 19:45:14 - INFO - __main__ - Step: 3460 Loss: 10.4566 LR: 0.000030
+12/08/2025 19:45:55 - INFO - __main__ - Step: 3470 Loss: 10.2818 LR: 0.000030
+12/08/2025 19:46:36 - INFO - __main__ - Step: 3480 Loss: 10.3455 LR: 0.000030
+12/08/2025 19:47:17 - INFO - __main__ - Step: 3490 Loss: 10.3028 LR: 0.000030
+12/08/2025 19:47:56 - INFO - __main__ - Step: 3500 Loss: 10.3447 LR: 0.000030
+12/08/2025 19:47:56 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500
+12/08/2025 19:48:05 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500/optimizer.bin
+12/08/2025 19:48:05 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500/scheduler.bin
+12/08/2025 19:48:05 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500/sampler.bin
+12/08/2025 19:48:05 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500/random_states_0.pkl
+12/08/2025 19:48:05 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-3500
+12/08/2025 19:48:05 - INFO - __main__ - Generating videos for validation...
+12/08/2025 19:48:05 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.86it/s]
+12/08/2025 19:48:11 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 19:48:51 - INFO - __main__ - Step: 3510 Loss: 10.4375 LR: 0.000030
+12/08/2025 19:49:30 - INFO - __main__ - Step: 3520 Loss: 10.2045 LR: 0.000030
+12/08/2025 19:50:11 - INFO - __main__ - Step: 3530 Loss: 10.3081 LR: 0.000030
+12/08/2025 19:50:52 - INFO - __main__ - Step: 3540 Loss: 10.3131 LR: 0.000030
+12/08/2025 19:51:32 - INFO - __main__ - Step: 3550 Loss: 10.3914 LR: 0.000030
+12/08/2025 19:52:13 - INFO - __main__ - Step: 3560 Loss: 10.2767 LR: 0.000030
+12/08/2025 19:52:54 - INFO - __main__ - Step: 3570 Loss: 10.3079 LR: 0.000030
+12/08/2025 19:53:34 - INFO - __main__ - Step: 3580 Loss: 10.3233 LR: 0.000030
+12/08/2025 19:54:16 - INFO - __main__ - Step: 3590 Loss: 10.3012 LR: 0.000030
+12/08/2025 19:54:57 - INFO - __main__ - Step: 3600 Loss: 10.3112 LR: 0.000030
+12/08/2025 19:55:38 - INFO - __main__ - Step: 3610 Loss: 10.2488 LR: 0.000030
+12/08/2025 19:56:21 - INFO - __main__ - Step: 3620 Loss: 10.3730 LR: 0.000030
+12/08/2025 19:57:02 - INFO - __main__ - Step: 3630 Loss: 10.3894 LR: 0.000030
+12/08/2025 19:57:42 - INFO - __main__ - Step: 3640 Loss: 10.1371 LR: 0.000030
+12/08/2025 19:58:24 - INFO - __main__ - Step: 3650 Loss: 10.3148 LR: 0.000030
+12/08/2025 19:59:06 - INFO - __main__ - Step: 3660 Loss: 10.2854 LR: 0.000030
+12/08/2025 19:59:47 - INFO - __main__ - Step: 3670 Loss: 10.2385 LR: 0.000030
+12/08/2025 20:00:29 - INFO - __main__ - Step: 3680 Loss: 10.0399 LR: 0.000030
+12/08/2025 20:01:10 - INFO - __main__ - Step: 3690 Loss: 10.1957 LR: 0.000030
+12/08/2025 20:01:52 - INFO - __main__ - Step: 3700 Loss: 10.3439 LR: 0.000030
+12/08/2025 20:02:32 - INFO - __main__ - Step: 3710 Loss: 10.3401 LR: 0.000030
+12/08/2025 20:03:14 - INFO - __main__ - Step: 3720 Loss: 10.2707 LR: 0.000030
+12/08/2025 20:03:55 - INFO - __main__ - Step: 3730 Loss: 9.9035 LR: 0.000030
+12/08/2025 20:04:38 - INFO - __main__ - Step: 3740 Loss: 10.2220 LR: 0.000030
+12/08/2025 20:05:19 - INFO - __main__ - Step: 3750 Loss: 10.2304 LR: 0.000030
+12/08/2025 20:06:01 - INFO - __main__ - Step: 3760 Loss: 10.3545 LR: 0.000030
+12/08/2025 20:06:43 - INFO - __main__ - Step: 3770 Loss: 10.2927 LR: 0.000030
+12/08/2025 20:07:24 - INFO - __main__ - Step: 3780 Loss: 10.2880 LR: 0.000030
+12/08/2025 20:08:06 - INFO - __main__ - Step: 3790 Loss: 10.2844 LR: 0.000030
+12/08/2025 20:08:48 - INFO - __main__ - Step: 3800 Loss: 10.4498 LR: 0.000030
+12/08/2025 20:09:31 - INFO - __main__ - Step: 3810 Loss: 10.2957 LR: 0.000030
+12/08/2025 20:10:13 - INFO - __main__ - Step: 3820 Loss: 10.2790 LR: 0.000030
+12/08/2025 20:10:54 - INFO - __main__ - Step: 3830 Loss: 10.2985 LR: 0.000030
+12/08/2025 20:11:34 - INFO - __main__ - Step: 3840 Loss: 10.2712 LR: 0.000030
+12/08/2025 20:12:16 - INFO - __main__ - Step: 3850 Loss: 10.2378 LR: 0.000030
+12/08/2025 20:12:57 - INFO - __main__ - Step: 3860 Loss: 10.3652 LR: 0.000030
+12/08/2025 20:13:39 - INFO - __main__ - Step: 3870 Loss: 10.2111 LR: 0.000030
+12/08/2025 20:14:19 - INFO - __main__ - Step: 3880 Loss: 10.3703 LR: 0.000030
+12/08/2025 20:15:02 - INFO - __main__ - Step: 3890 Loss: 10.0977 LR: 0.000030
+12/08/2025 20:15:44 - INFO - __main__ - Step: 3900 Loss: 10.3042 LR: 0.000030
+12/08/2025 20:16:26 - INFO - __main__ - Step: 3910 Loss: 10.3480 LR: 0.000030
+12/08/2025 20:17:08 - INFO - __main__ - Step: 3920 Loss: 10.2822 LR: 0.000030
+12/08/2025 20:17:49 - INFO - __main__ - Step: 3930 Loss: 10.3450 LR: 0.000030
+12/08/2025 20:18:31 - INFO - __main__ - Step: 3940 Loss: 10.0804 LR: 0.000030
+12/08/2025 20:19:12 - INFO - __main__ - Step: 3950 Loss: 10.3248 LR: 0.000030
+12/08/2025 20:19:53 - INFO - __main__ - Step: 3960 Loss: 10.1971 LR: 0.000030
+12/08/2025 20:20:34 - INFO - __main__ - Step: 3970 Loss: 10.2639 LR: 0.000030
+12/08/2025 20:21:14 - INFO - __main__ - Step: 3980 Loss: 10.3137 LR: 0.000030
+12/08/2025 20:21:55 - INFO - __main__ - Step: 3990 Loss: 10.2407 LR: 0.000030
+12/08/2025 20:22:36 - INFO - __main__ - Step: 4000 Loss: 10.2451 LR: 0.000030
+12/08/2025 20:22:36 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000
+12/08/2025 20:22:44 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000/optimizer.bin
+12/08/2025 20:22:44 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000/scheduler.bin
+12/08/2025 20:22:44 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000/sampler.bin
+12/08/2025 20:22:44 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000/random_states_0.pkl
+12/08/2025 20:22:44 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4000
+12/08/2025 20:22:44 - INFO - __main__ - Generating videos for validation...
+12/08/2025 20:22:44 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.75it/s]
+12/08/2025 20:22:50 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 20:23:31 - INFO - __main__ - Step: 4010 Loss: 10.2899 LR: 0.000030
+12/08/2025 20:24:11 - INFO - __main__ - Step: 4020 Loss: 10.3379 LR: 0.000030
+12/08/2025 20:24:51 - INFO - __main__ - Step: 4030 Loss: 10.2633 LR: 0.000030
+12/08/2025 20:25:34 - INFO - __main__ - Step: 4040 Loss: 10.3699 LR: 0.000030
+12/08/2025 20:26:13 - INFO - __main__ - Step: 4050 Loss: 10.2436 LR: 0.000030
+12/08/2025 20:26:54 - INFO - __main__ - Step: 4060 Loss: 10.1246 LR: 0.000030
+12/08/2025 20:27:37 - INFO - __main__ - Step: 4070 Loss: 10.3255 LR: 0.000030
+12/08/2025 20:28:18 - INFO - __main__ - Step: 4080 Loss: 10.3316 LR: 0.000030
+12/08/2025 20:28:57 - INFO - __main__ - Step: 4090 Loss: 10.4003 LR: 0.000030
+12/08/2025 20:29:38 - INFO - __main__ - Step: 4100 Loss: 10.1379 LR: 0.000030
+12/08/2025 20:30:20 - INFO - __main__ - Step: 4110 Loss: 10.1628 LR: 0.000030
+12/08/2025 20:31:02 - INFO - __main__ - Step: 4120 Loss: 10.2809 LR: 0.000030
+12/08/2025 20:31:42 - INFO - __main__ - Step: 4130 Loss: 10.0747 LR: 0.000030
+12/08/2025 20:32:22 - INFO - __main__ - Step: 4140 Loss: 10.1733 LR: 0.000030
+12/08/2025 20:33:02 - INFO - __main__ - Step: 4150 Loss: 10.1063 LR: 0.000030
+12/08/2025 20:33:43 - INFO - __main__ - Step: 4160 Loss: 10.3310 LR: 0.000030
+12/08/2025 20:34:23 - INFO - __main__ - Step: 4170 Loss: 10.3512 LR: 0.000030
+12/08/2025 20:35:03 - INFO - __main__ - Step: 4180 Loss: 10.2131 LR: 0.000030
+12/08/2025 20:35:44 - INFO - __main__ - Step: 4190 Loss: 10.3586 LR: 0.000030
+12/08/2025 20:36:27 - INFO - __main__ - Step: 4200 Loss: 10.3207 LR: 0.000030
+12/08/2025 20:37:07 - INFO - __main__ - Step: 4210 Loss: 10.3537 LR: 0.000030
+12/08/2025 20:37:47 - INFO - __main__ - Step: 4220 Loss: 10.1798 LR: 0.000030
+12/08/2025 20:38:29 - INFO - __main__ - Step: 4230 Loss: 10.2969 LR: 0.000030
+12/08/2025 20:39:09 - INFO - __main__ - Step: 4240 Loss: 10.1916 LR: 0.000030
+12/08/2025 20:39:50 - INFO - __main__ - Step: 4250 Loss: 10.2351 LR: 0.000030
+12/08/2025 20:40:30 - INFO - __main__ - Step: 4260 Loss: 10.4348 LR: 0.000030
+12/08/2025 20:41:11 - INFO - __main__ - Step: 4270 Loss: 10.2058 LR: 0.000030
+12/08/2025 20:41:52 - INFO - __main__ - Step: 4280 Loss: 10.4054 LR: 0.000030
+12/08/2025 20:42:33 - INFO - __main__ - Step: 4290 Loss: 10.2221 LR: 0.000030
+12/08/2025 20:43:13 - INFO - __main__ - Step: 4300 Loss: 10.3184 LR: 0.000030
+12/08/2025 20:43:56 - INFO - __main__ - Step: 4310 Loss: 10.0380 LR: 0.000030
+12/08/2025 20:44:37 - INFO - __main__ - Step: 4320 Loss: 10.1752 LR: 0.000030
+12/08/2025 20:45:18 - INFO - __main__ - Step: 4330 Loss: 9.9107 LR: 0.000030
+12/08/2025 20:45:59 - INFO - __main__ - Step: 4340 Loss: 10.2164 LR: 0.000030
+12/08/2025 20:46:42 - INFO - __main__ - Step: 4350 Loss: 10.2119 LR: 0.000030
+12/08/2025 20:47:22 - INFO - __main__ - Step: 4360 Loss: 10.1768 LR: 0.000030
+12/08/2025 20:48:05 - INFO - __main__ - Step: 4370 Loss: 10.2246 LR: 0.000030
+12/08/2025 20:48:45 - INFO - __main__ - Step: 4380 Loss: 10.3193 LR: 0.000030
+12/08/2025 20:49:28 - INFO - __main__ - Step: 4390 Loss: 10.2580 LR: 0.000030
+12/08/2025 20:50:10 - INFO - __main__ - Step: 4400 Loss: 10.1080 LR: 0.000030
+12/08/2025 20:50:52 - INFO - __main__ - Step: 4410 Loss: 10.3309 LR: 0.000030
+12/08/2025 20:51:35 - INFO - __main__ - Step: 4420 Loss: 10.2459 LR: 0.000030
+12/08/2025 20:52:17 - INFO - __main__ - Step: 4430 Loss: 10.3830 LR: 0.000030
+12/08/2025 20:52:58 - INFO - __main__ - Step: 4440 Loss: 10.2223 LR: 0.000030
+12/08/2025 20:53:40 - INFO - __main__ - Step: 4450 Loss: 10.2005 LR: 0.000030
+12/08/2025 20:54:23 - INFO - __main__ - Step: 4460 Loss: 10.2870 LR: 0.000030
+12/08/2025 20:55:07 - INFO - __main__ - Step: 4470 Loss: 10.2336 LR: 0.000030
+12/08/2025 20:55:48 - INFO - __main__ - Step: 4480 Loss: 10.1000 LR: 0.000030
+12/08/2025 20:56:31 - INFO - __main__ - Step: 4490 Loss: 10.2858 LR: 0.000030
+12/08/2025 20:57:13 - INFO - __main__ - Step: 4500 Loss: 10.4108 LR: 0.000030
+12/08/2025 20:57:13 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500
+12/08/2025 20:57:22 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500/optimizer.bin
+12/08/2025 20:57:22 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500/scheduler.bin
+12/08/2025 20:57:22 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500/sampler.bin
+12/08/2025 20:57:22 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500/random_states_0.pkl
+12/08/2025 20:57:22 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-4500
+12/08/2025 20:57:22 - INFO - __main__ - Generating videos for validation...
+12/08/2025 20:57:22 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.80it/s]
+12/08/2025 20:57:29 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 20:58:09 - INFO - __main__ - Step: 4510 Loss: 10.2823 LR: 0.000030
+12/08/2025 20:58:51 - INFO - __main__ - Step: 4520 Loss: 10.1449 LR: 0.000030
+12/08/2025 20:59:32 - INFO - __main__ - Step: 4530 Loss: 10.3825 LR: 0.000030
+12/08/2025 21:00:13 - INFO - __main__ - Step: 4540 Loss: 10.0908 LR: 0.000030
+12/08/2025 21:00:56 - INFO - __main__ - Step: 4550 Loss: 10.2845 LR: 0.000030
+12/08/2025 21:01:43 - INFO - __main__ - Step: 4560 Loss: 10.1658 LR: 0.000030
+12/08/2025 21:02:26 - INFO - __main__ - Step: 4570 Loss: 10.3447 LR: 0.000030
+12/08/2025 21:03:07 - INFO - __main__ - Step: 4580 Loss: 10.1652 LR: 0.000030
+12/08/2025 21:03:48 - INFO - __main__ - Step: 4590 Loss: 10.2222 LR: 0.000030
+12/08/2025 21:04:30 - INFO - __main__ - Step: 4600 Loss: 10.0792 LR: 0.000030
+12/08/2025 21:05:13 - INFO - __main__ - Step: 4610 Loss: 10.2119 LR: 0.000030
+12/08/2025 21:05:54 - INFO - __main__ - Step: 4620 Loss: 10.0849 LR: 0.000030
+12/08/2025 21:06:37 - INFO - __main__ - Step: 4630 Loss: 10.0546 LR: 0.000030
+12/08/2025 21:07:19 - INFO - __main__ - Step: 4640 Loss: 10.2588 LR: 0.000030
+12/08/2025 21:08:01 - INFO - __main__ - Step: 4650 Loss: 10.3376 LR: 0.000030
+12/08/2025 21:08:43 - INFO - __main__ - Step: 4660 Loss: 10.2977 LR: 0.000030
+12/08/2025 21:09:25 - INFO - __main__ - Step: 4670 Loss: 10.3725 LR: 0.000030
+12/08/2025 21:10:07 - INFO - __main__ - Step: 4680 Loss: 10.2226 LR: 0.000030
+12/08/2025 21:10:49 - INFO - __main__ - Step: 4690 Loss: 10.3483 LR: 0.000030
+12/08/2025 21:11:31 - INFO - __main__ - Step: 4700 Loss: 10.3505 LR: 0.000030
+12/08/2025 21:12:12 - INFO - __main__ - Step: 4710 Loss: 10.3326 LR: 0.000030
+12/08/2025 21:12:53 - INFO - __main__ - Step: 4720 Loss: 10.3280 LR: 0.000030
+12/08/2025 21:13:36 - INFO - __main__ - Step: 4730 Loss: 10.1439 LR: 0.000030
+12/08/2025 21:14:17 - INFO - __main__ - Step: 4740 Loss: 10.3052 LR: 0.000030
+12/08/2025 21:14:59 - INFO - __main__ - Step: 4750 Loss: 10.3487 LR: 0.000030
+12/08/2025 21:15:41 - INFO - __main__ - Step: 4760 Loss: 10.2699 LR: 0.000030
+12/08/2025 21:16:23 - INFO - __main__ - Step: 4770 Loss: 10.2466 LR: 0.000030
+12/08/2025 21:17:05 - INFO - __main__ - Step: 4780 Loss: 10.2666 LR: 0.000030
+12/08/2025 21:17:47 - INFO - __main__ - Step: 4790 Loss: 9.8289 LR: 0.000030
+12/08/2025 21:18:28 - INFO - __main__ - Step: 4800 Loss: 10.2504 LR: 0.000030
+12/08/2025 21:19:11 - INFO - __main__ - Step: 4810 Loss: 10.3114 LR: 0.000030
+12/08/2025 21:19:52 - INFO - __main__ - Step: 4820 Loss: 10.2730 LR: 0.000030
+12/08/2025 21:20:34 - INFO - __main__ - Step: 4830 Loss: 10.1306 LR: 0.000030
+12/08/2025 21:21:16 - INFO - __main__ - Step: 4840 Loss: 10.3507 LR: 0.000030
+12/08/2025 21:21:59 - INFO - __main__ - Step: 4850 Loss: 10.1111 LR: 0.000030
+12/08/2025 21:22:41 - INFO - __main__ - Step: 4860 Loss: 10.3416 LR: 0.000030
+12/08/2025 21:23:23 - INFO - __main__ - Step: 4870 Loss: 10.1908 LR: 0.000030
+12/08/2025 21:24:05 - INFO - __main__ - Step: 4880 Loss: 10.3423 LR: 0.000030
+12/08/2025 21:24:47 - INFO - __main__ - Step: 4890 Loss: 10.2651 LR: 0.000030
+12/08/2025 21:25:29 - INFO - __main__ - Step: 4900 Loss: 10.3409 LR: 0.000030
+12/08/2025 21:26:10 - INFO - __main__ - Step: 4910 Loss: 9.9730 LR: 0.000030
+12/08/2025 21:26:52 - INFO - __main__ - Step: 4920 Loss: 10.2333 LR: 0.000030
+12/08/2025 21:27:34 - INFO - __main__ - Step: 4930 Loss: 10.1536 LR: 0.000030
+12/08/2025 21:28:16 - INFO - __main__ - Step: 4940 Loss: 10.2931 LR: 0.000030
+12/08/2025 21:28:57 - INFO - __main__ - Step: 4950 Loss: 10.3159 LR: 0.000030
+12/08/2025 21:29:39 - INFO - __main__ - Step: 4960 Loss: 10.0516 LR: 0.000030
+12/08/2025 21:30:20 - INFO - __main__ - Step: 4970 Loss: 10.1747 LR: 0.000030
+12/08/2025 21:31:01 - INFO - __main__ - Step: 4980 Loss: 10.3765 LR: 0.000030
+12/08/2025 21:31:43 - INFO - __main__ - Step: 4990 Loss: 10.2803 LR: 0.000030
+12/08/2025 21:32:25 - INFO - __main__ - Step: 5000 Loss: 10.3450 LR: 0.000030
+12/08/2025 21:32:25 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000
+12/08/2025 21:32:33 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000/optimizer.bin
+12/08/2025 21:32:33 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000/scheduler.bin
+12/08/2025 21:32:33 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000/sampler.bin
+12/08/2025 21:32:33 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000/random_states_0.pkl
+12/08/2025 21:32:33 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5000
+12/08/2025 21:32:33 - INFO - __main__ - Generating videos for validation...
+12/08/2025 21:32:33 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.71it/s]
+12/08/2025 21:32:39 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 21:33:21 - INFO - __main__ - Step: 5010 Loss: 10.3345 LR: 0.000030
+12/08/2025 21:34:03 - INFO - __main__ - Step: 5020 Loss: 10.3916 LR: 0.000030
+12/08/2025 21:34:45 - INFO - __main__ - Step: 5030 Loss: 10.2735 LR: 0.000030
+12/08/2025 21:35:27 - INFO - __main__ - Step: 5040 Loss: 10.1283 LR: 0.000030
+12/08/2025 21:36:09 - INFO - __main__ - Step: 5050 Loss: 10.2323 LR: 0.000030
+12/08/2025 21:36:50 - INFO - __main__ - Step: 5060 Loss: 10.3613 LR: 0.000030
+12/08/2025 21:37:32 - INFO - __main__ - Step: 5070 Loss: 10.2081 LR: 0.000030
+12/08/2025 21:38:13 - INFO - __main__ - Step: 5080 Loss: 10.2587 LR: 0.000030
+12/08/2025 21:38:55 - INFO - __main__ - Step: 5090 Loss: 10.2870 LR: 0.000030
+12/08/2025 21:39:38 - INFO - __main__ - Step: 5100 Loss: 10.1199 LR: 0.000030
+12/08/2025 21:40:20 - INFO - __main__ - Step: 5110 Loss: 10.2896 LR: 0.000030
+12/08/2025 21:41:04 - INFO - __main__ - Step: 5120 Loss: 10.3799 LR: 0.000030
+12/08/2025 21:41:45 - INFO - __main__ - Step: 5130 Loss: 10.3204 LR: 0.000030
+12/08/2025 21:42:27 - INFO - __main__ - Step: 5140 Loss: 10.2922 LR: 0.000030
+12/08/2025 21:43:09 - INFO - __main__ - Step: 5150 Loss: 9.9432 LR: 0.000030
+12/08/2025 21:43:51 - INFO - __main__ - Step: 5160 Loss: 10.3235 LR: 0.000030
+12/08/2025 21:44:33 - INFO - __main__ - Step: 5170 Loss: 10.3383 LR: 0.000030
+12/08/2025 21:45:14 - INFO - __main__ - Step: 5180 Loss: 10.1858 LR: 0.000030
+12/08/2025 21:45:55 - INFO - __main__ - Step: 5190 Loss: 10.3718 LR: 0.000030
+12/08/2025 21:46:36 - INFO - __main__ - Step: 5200 Loss: 10.1364 LR: 0.000030
+12/08/2025 21:47:18 - INFO - __main__ - Step: 5210 Loss: 10.1594 LR: 0.000030
+12/08/2025 21:47:58 - INFO - __main__ - Step: 5220 Loss: 10.2590 LR: 0.000030
+12/08/2025 21:48:38 - INFO - __main__ - Step: 5230 Loss: 10.1180 LR: 0.000030
+12/08/2025 21:49:19 - INFO - __main__ - Step: 5240 Loss: 10.3417 LR: 0.000030
+12/08/2025 21:50:01 - INFO - __main__ - Step: 5250 Loss: 10.2217 LR: 0.000030
+12/08/2025 21:50:42 - INFO - __main__ - Step: 5260 Loss: 10.1060 LR: 0.000030
+12/08/2025 21:51:24 - INFO - __main__ - Step: 5270 Loss: 10.0291 LR: 0.000030
+12/08/2025 21:52:05 - INFO - __main__ - Step: 5280 Loss: 10.2215 LR: 0.000030
+12/08/2025 21:52:46 - INFO - __main__ - Step: 5290 Loss: 10.3945 LR: 0.000030
+12/08/2025 21:53:28 - INFO - __main__ - Step: 5300 Loss: 10.0282 LR: 0.000030
+12/08/2025 21:54:09 - INFO - __main__ - Step: 5310 Loss: 10.3425 LR: 0.000030
+12/08/2025 21:54:51 - INFO - __main__ - Step: 5320 Loss: 9.9285 LR: 0.000030
+12/08/2025 21:55:32 - INFO - __main__ - Step: 5330 Loss: 10.2162 LR: 0.000030
+12/08/2025 21:56:14 - INFO - __main__ - Step: 5340 Loss: 10.3321 LR: 0.000030
+12/08/2025 21:56:56 - INFO - __main__ - Step: 5350 Loss: 10.0870 LR: 0.000030
+12/08/2025 21:57:38 - INFO - __main__ - Step: 5360 Loss: 10.0999 LR: 0.000030
+12/08/2025 21:58:20 - INFO - __main__ - Step: 5370 Loss: 10.2815 LR: 0.000030
+12/08/2025 21:59:01 - INFO - __main__ - Step: 5380 Loss: 10.3108 LR: 0.000030
+12/08/2025 21:59:43 - INFO - __main__ - Step: 5390 Loss: 10.2583 LR: 0.000030
+12/08/2025 22:00:25 - INFO - __main__ - Step: 5400 Loss: 10.2013 LR: 0.000030
+12/08/2025 22:01:07 - INFO - __main__ - Step: 5410 Loss: 10.2130 LR: 0.000030
+12/08/2025 22:01:49 - INFO - __main__ - Step: 5420 Loss: 10.0639 LR: 0.000030
+12/08/2025 22:02:31 - INFO - __main__ - Step: 5430 Loss: 10.3222 LR: 0.000030
+12/08/2025 22:03:12 - INFO - __main__ - Step: 5440 Loss: 10.4058 LR: 0.000030
+12/08/2025 22:03:55 - INFO - __main__ - Step: 5450 Loss: 10.1913 LR: 0.000030
+12/08/2025 22:04:36 - INFO - __main__ - Step: 5460 Loss: 10.1779 LR: 0.000030
+12/08/2025 22:05:17 - INFO - __main__ - Step: 5470 Loss: 10.3779 LR: 0.000030
+12/08/2025 22:05:59 - INFO - __main__ - Step: 5480 Loss: 10.1930 LR: 0.000030
+12/08/2025 22:06:42 - INFO - __main__ - Step: 5490 Loss: 10.3855 LR: 0.000030
+12/08/2025 22:07:24 - INFO - __main__ - Step: 5500 Loss: 10.2458 LR: 0.000030
+12/08/2025 22:07:24 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500
+12/08/2025 22:07:31 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500/optimizer.bin
+12/08/2025 22:07:31 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500/scheduler.bin
+12/08/2025 22:07:31 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500/sampler.bin
+12/08/2025 22:07:31 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500/random_states_0.pkl
+12/08/2025 22:07:31 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-5500
+12/08/2025 22:07:31 - INFO - __main__ - Generating videos for validation...
+12/08/2025 22:07:31 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.90it/s]
+12/08/2025 22:07:37 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 22:08:19 - INFO - __main__ - Step: 5510 Loss: 10.4149 LR: 0.000030
+12/08/2025 22:09:02 - INFO - __main__ - Step: 5520 Loss: 10.2892 LR: 0.000030
+12/08/2025 22:09:44 - INFO - __main__ - Step: 5530 Loss: 10.3342 LR: 0.000030
+12/08/2025 22:10:27 - INFO - __main__ - Step: 5540 Loss: 10.3278 LR: 0.000030
+12/08/2025 22:11:11 - INFO - __main__ - Step: 5550 Loss: 10.3007 LR: 0.000030
+12/08/2025 22:11:53 - INFO - __main__ - Step: 5560 Loss: 9.9747 LR: 0.000030
+12/08/2025 22:12:34 - INFO - __main__ - Step: 5570 Loss: 9.9621 LR: 0.000030
+12/08/2025 22:13:17 - INFO - __main__ - Step: 5580 Loss: 10.1569 LR: 0.000030
+12/08/2025 22:13:59 - INFO - __main__ - Step: 5590 Loss: 10.2479 LR: 0.000030
+12/08/2025 22:14:41 - INFO - __main__ - Step: 5600 Loss: 10.2491 LR: 0.000030
+12/08/2025 22:15:23 - INFO - __main__ - Step: 5610 Loss: 10.1147 LR: 0.000030
+12/08/2025 22:16:05 - INFO - __main__ - Step: 5620 Loss: 10.3269 LR: 0.000030
+12/08/2025 22:16:48 - INFO - __main__ - Step: 5630 Loss: 10.0841 LR: 0.000030
+12/08/2025 22:17:30 - INFO - __main__ - Step: 5640 Loss: 10.3737 LR: 0.000030
+12/08/2025 22:18:12 - INFO - __main__ - Step: 5650 Loss: 10.3548 LR: 0.000030
+12/08/2025 22:18:54 - INFO - __main__ - Step: 5660 Loss: 10.2859 LR: 0.000030
+12/08/2025 22:19:37 - INFO - __main__ - Step: 5670 Loss: 10.1853 LR: 0.000030
+12/08/2025 22:20:19 - INFO - __main__ - Step: 5680 Loss: 10.3632 LR: 0.000030
+12/08/2025 22:21:03 - INFO - __main__ - Step: 5690 Loss: 10.2320 LR: 0.000030
+12/08/2025 22:21:45 - INFO - __main__ - Step: 5700 Loss: 10.2409 LR: 0.000030
+12/08/2025 22:22:27 - INFO - __main__ - Step: 5710 Loss: 10.3355 LR: 0.000030
+12/08/2025 22:23:10 - INFO - __main__ - Step: 5720 Loss: 10.2148 LR: 0.000030
+12/08/2025 22:23:52 - INFO - __main__ - Step: 5730 Loss: 10.2750 LR: 0.000030
+12/08/2025 22:24:33 - INFO - __main__ - Step: 5740 Loss: 10.2772 LR: 0.000030
+12/08/2025 22:25:15 - INFO - __main__ - Step: 5750 Loss: 10.1679 LR: 0.000030
+12/08/2025 22:25:56 - INFO - __main__ - Step: 5760 Loss: 10.2853 LR: 0.000030
+12/08/2025 22:26:39 - INFO - __main__ - Step: 5770 Loss: 10.3320 LR: 0.000030
+12/08/2025 22:27:22 - INFO - __main__ - Step: 5780 Loss: 10.3134 LR: 0.000030
+12/08/2025 22:28:04 - INFO - __main__ - Step: 5790 Loss: 10.3775 LR: 0.000030
+12/08/2025 22:28:47 - INFO - __main__ - Step: 5800 Loss: 10.1853 LR: 0.000030
+12/08/2025 22:29:27 - INFO - __main__ - Step: 5810 Loss: 10.2957 LR: 0.000030
+12/08/2025 22:30:10 - INFO - __main__ - Step: 5820 Loss: 10.2954 LR: 0.000030
+12/08/2025 22:30:52 - INFO - __main__ - Step: 5830 Loss: 9.9769 LR: 0.000030
+12/08/2025 22:31:34 - INFO - __main__ - Step: 5840 Loss: 10.1726 LR: 0.000030
+12/08/2025 22:32:16 - INFO - __main__ - Step: 5850 Loss: 10.2430 LR: 0.000030
+12/08/2025 22:32:59 - INFO - __main__ - Step: 5860 Loss: 10.3791 LR: 0.000030
+12/08/2025 22:33:41 - INFO - __main__ - Step: 5870 Loss: 10.2243 LR: 0.000030
+12/08/2025 22:34:23 - INFO - __main__ - Step: 5880 Loss: 10.3128 LR: 0.000030
+12/08/2025 22:35:06 - INFO - __main__ - Step: 5890 Loss: 10.3263 LR: 0.000030
+12/08/2025 22:35:47 - INFO - __main__ - Step: 5900 Loss: 10.2545 LR: 0.000030
+12/08/2025 22:36:29 - INFO - __main__ - Step: 5910 Loss: 10.4141 LR: 0.000030
+12/08/2025 22:37:11 - INFO - __main__ - Step: 5920 Loss: 10.2515 LR: 0.000030
+12/08/2025 22:37:53 - INFO - __main__ - Step: 5930 Loss: 10.2135 LR: 0.000030
+12/08/2025 22:38:35 - INFO - __main__ - Step: 5940 Loss: 10.2881 LR: 0.000030
+12/08/2025 22:39:18 - INFO - __main__ - Step: 5950 Loss: 10.1997 LR: 0.000030
+12/08/2025 22:40:01 - INFO - __main__ - Step: 5960 Loss: 10.2877 LR: 0.000030
+12/08/2025 22:40:43 - INFO - __main__ - Step: 5970 Loss: 10.0559 LR: 0.000030
+12/08/2025 22:41:26 - INFO - __main__ - Step: 5980 Loss: 10.1773 LR: 0.000030
+12/08/2025 22:42:07 - INFO - __main__ - Step: 5990 Loss: 10.1486 LR: 0.000030
+12/08/2025 22:42:50 - INFO - __main__ - Step: 6000 Loss: 10.3314 LR: 0.000030
+12/08/2025 22:42:50 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000
+12/08/2025 22:42:57 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000/optimizer.bin
+12/08/2025 22:42:57 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000/scheduler.bin
+12/08/2025 22:42:57 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000/sampler.bin
+12/08/2025 22:42:57 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000/random_states_0.pkl
+12/08/2025 22:42:57 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6000
+12/08/2025 22:42:57 - INFO - __main__ - Generating videos for validation...
+12/08/2025 22:42:57 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.91it/s]
+12/08/2025 22:43:04 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 22:43:46 - INFO - __main__ - Step: 6010 Loss: 10.1129 LR: 0.000030
+12/08/2025 22:44:27 - INFO - __main__ - Step: 6020 Loss: 10.2666 LR: 0.000030
+12/08/2025 22:45:12 - INFO - __main__ - Step: 6030 Loss: 10.3086 LR: 0.000030
+12/08/2025 22:45:54 - INFO - __main__ - Step: 6040 Loss: 10.2977 LR: 0.000030
+12/08/2025 22:46:35 - INFO - __main__ - Step: 6050 Loss: 10.2531 LR: 0.000030
+12/08/2025 22:47:17 - INFO - __main__ - Step: 6060 Loss: 10.1809 LR: 0.000030
+12/08/2025 22:47:59 - INFO - __main__ - Step: 6070 Loss: 10.1784 LR: 0.000030
+12/08/2025 22:48:40 - INFO - __main__ - Step: 6080 Loss: 10.3065 LR: 0.000030
+12/08/2025 22:49:22 - INFO - __main__ - Step: 6090 Loss: 10.3134 LR: 0.000030
+12/08/2025 22:50:04 - INFO - __main__ - Step: 6100 Loss: 10.3101 LR: 0.000030
+12/08/2025 22:50:46 - INFO - __main__ - Step: 6110 Loss: 10.2897 LR: 0.000030
+12/08/2025 22:51:28 - INFO - __main__ - Step: 6120 Loss: 10.1782 LR: 0.000030
+12/08/2025 22:52:11 - INFO - __main__ - Step: 6130 Loss: 10.2543 LR: 0.000030
+12/08/2025 22:52:53 - INFO - __main__ - Step: 6140 Loss: 10.2993 LR: 0.000030
+12/08/2025 22:53:35 - INFO - __main__ - Step: 6150 Loss: 10.2649 LR: 0.000030
+12/08/2025 22:54:16 - INFO - __main__ - Step: 6160 Loss: 10.1428 LR: 0.000030
+12/08/2025 22:54:58 - INFO - __main__ - Step: 6170 Loss: 10.2793 LR: 0.000030
+12/08/2025 22:55:40 - INFO - __main__ - Step: 6180 Loss: 10.2793 LR: 0.000030
+12/08/2025 22:56:21 - INFO - __main__ - Step: 6190 Loss: 10.2194 LR: 0.000030
+12/08/2025 22:57:03 - INFO - __main__ - Step: 6200 Loss: 10.1877 LR: 0.000030
+12/08/2025 22:57:45 - INFO - __main__ - Step: 6210 Loss: 10.0722 LR: 0.000030
+12/08/2025 22:58:27 - INFO - __main__ - Step: 6220 Loss: 9.8783 LR: 0.000030
+12/08/2025 22:59:09 - INFO - __main__ - Step: 6230 Loss: 10.1823 LR: 0.000030
+12/08/2025 22:59:51 - INFO - __main__ - Step: 6240 Loss: 10.1875 LR: 0.000030
+12/08/2025 23:00:32 - INFO - __main__ - Step: 6250 Loss: 10.3461 LR: 0.000030
+12/08/2025 23:01:15 - INFO - __main__ - Step: 6260 Loss: 10.3449 LR: 0.000030
+12/08/2025 23:01:58 - INFO - __main__ - Step: 6270 Loss: 9.9969 LR: 0.000030
+12/08/2025 23:02:39 - INFO - __main__ - Step: 6280 Loss: 10.2543 LR: 0.000030
+12/08/2025 23:03:21 - INFO - __main__ - Step: 6290 Loss: 10.0740 LR: 0.000030
+12/08/2025 23:04:03 - INFO - __main__ - Step: 6300 Loss: 10.1394 LR: 0.000030
+12/08/2025 23:04:44 - INFO - __main__ - Step: 6310 Loss: 10.2553 LR: 0.000030
+12/08/2025 23:05:27 - INFO - __main__ - Step: 6320 Loss: 10.2410 LR: 0.000030
+12/08/2025 23:06:10 - INFO - __main__ - Step: 6330 Loss: 10.3733 LR: 0.000030
+12/08/2025 23:06:51 - INFO - __main__ - Step: 6340 Loss: 10.3512 LR: 0.000030
+12/08/2025 23:07:33 - INFO - __main__ - Step: 6350 Loss: 10.3197 LR: 0.000030
+12/08/2025 23:08:18 - INFO - __main__ - Step: 6360 Loss: 10.2109 LR: 0.000030
+12/08/2025 23:09:00 - INFO - __main__ - Step: 6370 Loss: 10.3247 LR: 0.000030
+12/08/2025 23:09:42 - INFO - __main__ - Step: 6380 Loss: 10.3582 LR: 0.000030
+12/08/2025 23:10:25 - INFO - __main__ - Step: 6390 Loss: 10.3242 LR: 0.000030
+12/08/2025 23:11:07 - INFO - __main__ - Step: 6400 Loss: 10.1779 LR: 0.000030
+12/08/2025 23:11:48 - INFO - __main__ - Step: 6410 Loss: 10.2039 LR: 0.000030
+12/08/2025 23:12:31 - INFO - __main__ - Step: 6420 Loss: 10.2187 LR: 0.000030
+12/08/2025 23:13:12 - INFO - __main__ - Step: 6430 Loss: 10.3584 LR: 0.000030
+12/08/2025 23:13:53 - INFO - __main__ - Step: 6440 Loss: 10.2263 LR: 0.000030
+12/08/2025 23:14:35 - INFO - __main__ - Step: 6450 Loss: 10.2296 LR: 0.000030
+12/08/2025 23:15:16 - INFO - __main__ - Step: 6460 Loss: 10.2907 LR: 0.000030
+12/08/2025 23:15:58 - INFO - __main__ - Step: 6470 Loss: 10.3953 LR: 0.000030
+12/08/2025 23:16:41 - INFO - __main__ - Step: 6480 Loss: 10.4121 LR: 0.000030
+12/08/2025 23:17:23 - INFO - __main__ - Step: 6490 Loss: 10.3521 LR: 0.000030
+12/08/2025 23:18:05 - INFO - __main__ - Step: 6500 Loss: 10.2929 LR: 0.000030
+12/08/2025 23:18:05 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500
+12/08/2025 23:18:15 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500/optimizer.bin
+12/08/2025 23:18:15 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500/scheduler.bin
+12/08/2025 23:18:15 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500/sampler.bin
+12/08/2025 23:18:15 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500/random_states_0.pkl
+12/08/2025 23:18:15 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-6500
+12/08/2025 23:18:15 - INFO - __main__ - Generating videos for validation...
+12/08/2025 23:18:15 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.64it/s]
+12/08/2025 23:18:22 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 23:19:03 - INFO - __main__ - Step: 6510 Loss: 10.1292 LR: 0.000030
+12/08/2025 23:19:44 - INFO - __main__ - Step: 6520 Loss: 10.3520 LR: 0.000030
+12/08/2025 23:20:25 - INFO - __main__ - Step: 6530 Loss: 10.1364 LR: 0.000030
+12/08/2025 23:21:06 - INFO - __main__ - Step: 6540 Loss: 10.2199 LR: 0.000030
+12/08/2025 23:21:46 - INFO - __main__ - Step: 6550 Loss: 10.0978 LR: 0.000030
+12/08/2025 23:22:27 - INFO - __main__ - Step: 6560 Loss: 10.1592 LR: 0.000030
+12/08/2025 23:23:08 - INFO - __main__ - Step: 6570 Loss: 10.1996 LR: 0.000030
+12/08/2025 23:23:52 - INFO - __main__ - Step: 6580 Loss: 10.3180 LR: 0.000030
+12/08/2025 23:24:32 - INFO - __main__ - Step: 6590 Loss: 10.1698 LR: 0.000030
+12/08/2025 23:25:12 - INFO - __main__ - Step: 6600 Loss: 10.3018 LR: 0.000030
+12/08/2025 23:25:53 - INFO - __main__ - Step: 6610 Loss: 10.1288 LR: 0.000030
+12/08/2025 23:26:33 - INFO - __main__ - Step: 6620 Loss: 10.4338 LR: 0.000030
+12/08/2025 23:27:15 - INFO - __main__ - Step: 6630 Loss: 10.2878 LR: 0.000030
+12/08/2025 23:27:56 - INFO - __main__ - Step: 6640 Loss: 10.2953 LR: 0.000030
+12/08/2025 23:28:39 - INFO - __main__ - Step: 6650 Loss: 10.2604 LR: 0.000030
+12/08/2025 23:29:21 - INFO - __main__ - Step: 6660 Loss: 10.1341 LR: 0.000030
+12/08/2025 23:30:03 - INFO - __main__ - Step: 6670 Loss: 10.3370 LR: 0.000030
+12/08/2025 23:30:44 - INFO - __main__ - Step: 6680 Loss: 10.0830 LR: 0.000030
+12/08/2025 23:31:25 - INFO - __main__ - Step: 6690 Loss: 10.2127 LR: 0.000030
+12/08/2025 23:32:06 - INFO - __main__ - Step: 6700 Loss: 9.8881 LR: 0.000030
+12/08/2025 23:32:46 - INFO - __main__ - Step: 6710 Loss: 10.3647 LR: 0.000030
+12/08/2025 23:33:29 - INFO - __main__ - Step: 6720 Loss: 10.2329 LR: 0.000030
+12/08/2025 23:34:10 - INFO - __main__ - Step: 6730 Loss: 10.1342 LR: 0.000030
+12/08/2025 23:34:52 - INFO - __main__ - Step: 6740 Loss: 10.2022 LR: 0.000030
+12/08/2025 23:35:35 - INFO - __main__ - Step: 6750 Loss: 10.3053 LR: 0.000030
+12/08/2025 23:36:17 - INFO - __main__ - Step: 6760 Loss: 10.1120 LR: 0.000030
+12/08/2025 23:36:57 - INFO - __main__ - Step: 6770 Loss: 10.1048 LR: 0.000030
+12/08/2025 23:37:39 - INFO - __main__ - Step: 6780 Loss: 10.3502 LR: 0.000030
+12/08/2025 23:38:21 - INFO - __main__ - Step: 6790 Loss: 10.3046 LR: 0.000030
+12/08/2025 23:39:02 - INFO - __main__ - Step: 6800 Loss: 10.1877 LR: 0.000030
+12/08/2025 23:39:44 - INFO - __main__ - Step: 6810 Loss: 10.0637 LR: 0.000030
+12/08/2025 23:40:26 - INFO - __main__ - Step: 6820 Loss: 10.3750 LR: 0.000030
+12/08/2025 23:41:08 - INFO - __main__ - Step: 6830 Loss: 10.1803 LR: 0.000030
+12/08/2025 23:41:50 - INFO - __main__ - Step: 6840 Loss: 10.3268 LR: 0.000030
+12/08/2025 23:42:33 - INFO - __main__ - Step: 6850 Loss: 10.1543 LR: 0.000030
+12/08/2025 23:43:16 - INFO - __main__ - Step: 6860 Loss: 10.2235 LR: 0.000030
+12/08/2025 23:43:57 - INFO - __main__ - Step: 6870 Loss: 10.1485 LR: 0.000030
+12/08/2025 23:44:38 - INFO - __main__ - Step: 6880 Loss: 10.1428 LR: 0.000030
+12/08/2025 23:45:19 - INFO - __main__ - Step: 6890 Loss: 10.3582 LR: 0.000030
+12/08/2025 23:46:02 - INFO - __main__ - Step: 6900 Loss: 10.2796 LR: 0.000030
+12/08/2025 23:46:43 - INFO - __main__ - Step: 6910 Loss: 10.2365 LR: 0.000030
+12/08/2025 23:47:26 - INFO - __main__ - Step: 6920 Loss: 10.1180 LR: 0.000030
+12/08/2025 23:48:08 - INFO - __main__ - Step: 6930 Loss: 10.2786 LR: 0.000030
+12/08/2025 23:48:50 - INFO - __main__ - Step: 6940 Loss: 10.3998 LR: 0.000030
+12/08/2025 23:49:33 - INFO - __main__ - Step: 6950 Loss: 10.3113 LR: 0.000030
+12/08/2025 23:50:15 - INFO - __main__ - Step: 6960 Loss: 10.3230 LR: 0.000030
+12/08/2025 23:50:57 - INFO - __main__ - Step: 6970 Loss: 10.3072 LR: 0.000030
+12/08/2025 23:51:39 - INFO - __main__ - Step: 6980 Loss: 10.1016 LR: 0.000030
+12/08/2025 23:52:22 - INFO - __main__ - Step: 6990 Loss: 10.2765 LR: 0.000030
+12/08/2025 23:53:04 - INFO - __main__ - Step: 7000 Loss: 10.3480 LR: 0.000030
+12/08/2025 23:53:04 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000
+12/08/2025 23:53:11 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000/optimizer.bin
+12/08/2025 23:53:11 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000/scheduler.bin
+12/08/2025 23:53:11 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000/sampler.bin
+12/08/2025 23:53:11 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000/random_states_0.pkl
+12/08/2025 23:53:11 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7000
+12/08/2025 23:53:11 - INFO - __main__ - Generating videos for validation...
+12/08/2025 23:53:11 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.88it/s]
+12/08/2025 23:53:18 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/08/2025 23:53:59 - INFO - __main__ - Step: 7010 Loss: 10.2368 LR: 0.000030
+12/08/2025 23:54:42 - INFO - __main__ - Step: 7020 Loss: 10.1803 LR: 0.000030
+12/08/2025 23:55:24 - INFO - __main__ - Step: 7030 Loss: 10.2376 LR: 0.000030
+12/08/2025 23:56:05 - INFO - __main__ - Step: 7040 Loss: 10.2110 LR: 0.000030
+12/08/2025 23:56:47 - INFO - __main__ - Step: 7050 Loss: 10.3305 LR: 0.000030
+12/08/2025 23:57:29 - INFO - __main__ - Step: 7060 Loss: 10.2698 LR: 0.000030
+12/08/2025 23:58:10 - INFO - __main__ - Step: 7070 Loss: 10.3615 LR: 0.000030
+12/08/2025 23:58:52 - INFO - __main__ - Step: 7080 Loss: 10.1830 LR: 0.000030
+12/08/2025 23:59:34 - INFO - __main__ - Step: 7090 Loss: 10.1951 LR: 0.000030
+12/09/2025 00:00:17 - INFO - __main__ - Step: 7100 Loss: 10.1074 LR: 0.000030
+12/09/2025 00:00:59 - INFO - __main__ - Step: 7110 Loss: 10.3386 LR: 0.000030
+12/09/2025 00:01:43 - INFO - __main__ - Step: 7120 Loss: 9.8956 LR: 0.000030
+12/09/2025 00:02:25 - INFO - __main__ - Step: 7130 Loss: 10.3152 LR: 0.000030
+12/09/2025 00:03:08 - INFO - __main__ - Step: 7140 Loss: 10.3875 LR: 0.000030
+12/09/2025 00:03:50 - INFO - __main__ - Step: 7150 Loss: 10.3053 LR: 0.000030
+12/09/2025 00:04:32 - INFO - __main__ - Step: 7160 Loss: 10.3367 LR: 0.000030
+12/09/2025 00:05:15 - INFO - __main__ - Step: 7170 Loss: 10.2307 LR: 0.000030
+12/09/2025 00:05:57 - INFO - __main__ - Step: 7180 Loss: 10.2600 LR: 0.000030
+12/09/2025 00:06:40 - INFO - __main__ - Step: 7190 Loss: 10.3188 LR: 0.000030
+12/09/2025 00:07:22 - INFO - __main__ - Step: 7200 Loss: 10.2022 LR: 0.000030
+12/09/2025 00:08:03 - INFO - __main__ - Step: 7210 Loss: 10.2719 LR: 0.000030
+12/09/2025 00:08:43 - INFO - __main__ - Step: 7220 Loss: 10.3750 LR: 0.000030
+12/09/2025 00:09:24 - INFO - __main__ - Step: 7230 Loss: 10.2134 LR: 0.000030
+12/09/2025 00:10:06 - INFO - __main__ - Step: 7240 Loss: 10.3141 LR: 0.000030
+12/09/2025 00:10:48 - INFO - __main__ - Step: 7250 Loss: 10.2763 LR: 0.000030
+12/09/2025 00:11:30 - INFO - __main__ - Step: 7260 Loss: 10.3463 LR: 0.000030
+12/09/2025 00:12:12 - INFO - __main__ - Step: 7270 Loss: 10.0836 LR: 0.000030
+12/09/2025 00:12:53 - INFO - __main__ - Step: 7280 Loss: 10.2993 LR: 0.000030
+12/09/2025 00:13:34 - INFO - __main__ - Step: 7290 Loss: 10.3237 LR: 0.000030
+12/09/2025 00:14:15 - INFO - __main__ - Step: 7300 Loss: 10.2270 LR: 0.000030
+12/09/2025 00:14:56 - INFO - __main__ - Step: 7310 Loss: 10.2369 LR: 0.000030
+12/09/2025 00:15:38 - INFO - __main__ - Step: 7320 Loss: 10.2477 LR: 0.000030
+12/09/2025 00:16:20 - INFO - __main__ - Step: 7330 Loss: 10.3374 LR: 0.000030
+12/09/2025 00:17:02 - INFO - __main__ - Step: 7340 Loss: 9.8238 LR: 0.000030
+12/09/2025 00:17:44 - INFO - __main__ - Step: 7350 Loss: 10.3105 LR: 0.000030
+12/09/2025 00:18:26 - INFO - __main__ - Step: 7360 Loss: 10.1948 LR: 0.000030
+12/09/2025 00:19:06 - INFO - __main__ - Step: 7370 Loss: 10.1476 LR: 0.000030
+12/09/2025 00:19:47 - INFO - __main__ - Step: 7380 Loss: 10.1568 LR: 0.000030
+12/09/2025 00:20:28 - INFO - __main__ - Step: 7390 Loss: 10.3466 LR: 0.000030
+12/09/2025 00:21:09 - INFO - __main__ - Step: 7400 Loss: 10.3168 LR: 0.000030
+12/09/2025 00:21:50 - INFO - __main__ - Step: 7410 Loss: 10.1458 LR: 0.000030
+12/09/2025 00:22:31 - INFO - __main__ - Step: 7420 Loss: 10.2313 LR: 0.000030
+12/09/2025 00:23:10 - INFO - __main__ - Step: 7430 Loss: 9.8185 LR: 0.000030
+12/09/2025 00:23:50 - INFO - __main__ - Step: 7440 Loss: 10.1768 LR: 0.000030
+12/09/2025 00:24:29 - INFO - __main__ - Step: 7450 Loss: 10.3064 LR: 0.000030
+12/09/2025 00:25:10 - INFO - __main__ - Step: 7460 Loss: 10.1902 LR: 0.000030
+12/09/2025 00:25:49 - INFO - __main__ - Step: 7470 Loss: 10.3586 LR: 0.000030
+12/09/2025 00:26:29 - INFO - __main__ - Step: 7480 Loss: 10.2662 LR: 0.000030
+12/09/2025 00:27:09 - INFO - __main__ - Step: 7490 Loss: 10.3064 LR: 0.000030
+12/09/2025 00:27:50 - INFO - __main__ - Step: 7500 Loss: 10.2657 LR: 0.000030
+12/09/2025 00:27:50 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500
+12/09/2025 00:28:00 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500/optimizer.bin
+12/09/2025 00:28:00 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500/scheduler.bin
+12/09/2025 00:28:00 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500/sampler.bin
+12/09/2025 00:28:00 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500/random_states_0.pkl
+12/09/2025 00:28:00 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-7500
+12/09/2025 00:28:00 - INFO - __main__ - Generating videos for validation...
+12/09/2025 00:28:00 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.69it/s]
+12/09/2025 00:28:06 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 00:28:48 - INFO - __main__ - Step: 7510 Loss: 10.1254 LR: 0.000030
+12/09/2025 00:29:29 - INFO - __main__ - Step: 7520 Loss: 10.2718 LR: 0.000030
+12/09/2025 00:30:14 - INFO - __main__ - Step: 7530 Loss: 10.3707 LR: 0.000030
+12/09/2025 00:30:55 - INFO - __main__ - Step: 7540 Loss: 10.2575 LR: 0.000030
+12/09/2025 00:31:37 - INFO - __main__ - Step: 7550 Loss: 10.3026 LR: 0.000030
+12/09/2025 00:32:18 - INFO - __main__ - Step: 7560 Loss: 10.3086 LR: 0.000030
+12/09/2025 00:32:58 - INFO - __main__ - Step: 7570 Loss: 9.9485 LR: 0.000030
+12/09/2025 00:33:38 - INFO - __main__ - Step: 7580 Loss: 10.1990 LR: 0.000030
+12/09/2025 00:34:20 - INFO - __main__ - Step: 7590 Loss: 10.2143 LR: 0.000030
+12/09/2025 00:34:59 - INFO - __main__ - Step: 7600 Loss: 10.3632 LR: 0.000030
+12/09/2025 00:35:40 - INFO - __main__ - Step: 7610 Loss: 10.2531 LR: 0.000030
+12/09/2025 00:36:21 - INFO - __main__ - Step: 7620 Loss: 10.0739 LR: 0.000030
+12/09/2025 00:37:03 - INFO - __main__ - Step: 7630 Loss: 10.2402 LR: 0.000030
+12/09/2025 00:37:45 - INFO - __main__ - Step: 7640 Loss: 10.3283 LR: 0.000030
+12/09/2025 00:38:27 - INFO - __main__ - Step: 7650 Loss: 10.2601 LR: 0.000030
+12/09/2025 00:39:09 - INFO - __main__ - Step: 7660 Loss: 10.2100 LR: 0.000030
+12/09/2025 00:39:50 - INFO - __main__ - Step: 7670 Loss: 10.1779 LR: 0.000030
+12/09/2025 00:40:30 - INFO - __main__ - Step: 7680 Loss: 10.2502 LR: 0.000030
+12/09/2025 00:41:13 - INFO - __main__ - Step: 7690 Loss: 10.2797 LR: 0.000030
+12/09/2025 00:41:54 - INFO - __main__ - Step: 7700 Loss: 10.2384 LR: 0.000030
+12/09/2025 00:42:36 - INFO - __main__ - Step: 7710 Loss: 10.1634 LR: 0.000030
+12/09/2025 00:43:18 - INFO - __main__ - Step: 7720 Loss: 10.1549 LR: 0.000030
+12/09/2025 00:43:59 - INFO - __main__ - Step: 7730 Loss: 10.2102 LR: 0.000030
+12/09/2025 00:44:40 - INFO - __main__ - Step: 7740 Loss: 10.2895 LR: 0.000030
+12/09/2025 00:45:22 - INFO - __main__ - Step: 7750 Loss: 10.2618 LR: 0.000030
+12/09/2025 00:46:02 - INFO - __main__ - Step: 7760 Loss: 10.2558 LR: 0.000030
+12/09/2025 00:46:44 - INFO - __main__ - Step: 7770 Loss: 10.2425 LR: 0.000030
+12/09/2025 00:47:23 - INFO - __main__ - Step: 7780 Loss: 9.7965 LR: 0.000030
+12/09/2025 00:48:04 - INFO - __main__ - Step: 7790 Loss: 10.3123 LR: 0.000030
+12/09/2025 00:48:44 - INFO - __main__ - Step: 7800 Loss: 10.2295 LR: 0.000030
+12/09/2025 00:49:25 - INFO - __main__ - Step: 7810 Loss: 10.3369 LR: 0.000030
+12/09/2025 00:50:07 - INFO - __main__ - Step: 7820 Loss: 10.3905 LR: 0.000030
+12/09/2025 00:50:48 - INFO - __main__ - Step: 7830 Loss: 10.2884 LR: 0.000030
+12/09/2025 00:51:29 - INFO - __main__ - Step: 7840 Loss: 10.2730 LR: 0.000030
+12/09/2025 00:52:12 - INFO - __main__ - Step: 7850 Loss: 10.0763 LR: 0.000030
+12/09/2025 00:52:53 - INFO - __main__ - Step: 7860 Loss: 10.2126 LR: 0.000030
+12/09/2025 00:53:35 - INFO - __main__ - Step: 7870 Loss: 10.0994 LR: 0.000030
+12/09/2025 00:54:15 - INFO - __main__ - Step: 7880 Loss: 10.1571 LR: 0.000030
+12/09/2025 00:54:57 - INFO - __main__ - Step: 7890 Loss: 10.3652 LR: 0.000030
+12/09/2025 00:55:38 - INFO - __main__ - Step: 7900 Loss: 10.4227 LR: 0.000030
+12/09/2025 00:56:18 - INFO - __main__ - Step: 7910 Loss: 10.3050 LR: 0.000030
+12/09/2025 00:57:00 - INFO - __main__ - Step: 7920 Loss: 10.2642 LR: 0.000030
+12/09/2025 00:57:42 - INFO - __main__ - Step: 7930 Loss: 10.2392 LR: 0.000030
+12/09/2025 00:58:23 - INFO - __main__ - Step: 7940 Loss: 10.3895 LR: 0.000030
+12/09/2025 00:59:04 - INFO - __main__ - Step: 7950 Loss: 9.9881 LR: 0.000030
+12/09/2025 00:59:45 - INFO - __main__ - Step: 7960 Loss: 10.2459 LR: 0.000030
+12/09/2025 01:00:27 - INFO - __main__ - Step: 7970 Loss: 10.3424 LR: 0.000030
+12/09/2025 01:01:11 - INFO - __main__ - Step: 7980 Loss: 10.2866 LR: 0.000030
+12/09/2025 01:01:53 - INFO - __main__ - Step: 7990 Loss: 10.2693 LR: 0.000030
+12/09/2025 01:02:36 - INFO - __main__ - Step: 8000 Loss: 10.3038 LR: 0.000030
+12/09/2025 01:02:36 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000
+12/09/2025 01:02:43 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000/optimizer.bin
+12/09/2025 01:02:43 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000/scheduler.bin
+12/09/2025 01:02:43 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000/sampler.bin
+12/09/2025 01:02:43 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000/random_states_0.pkl
+12/09/2025 01:02:43 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8000
+12/09/2025 01:02:43 - INFO - __main__ - Generating videos for validation...
+12/09/2025 01:02:43 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.65it/s]
+12/09/2025 01:02:50 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 01:03:31 - INFO - __main__ - Step: 8010 Loss: 10.1872 LR: 0.000030
+12/09/2025 01:04:14 - INFO - __main__ - Step: 8020 Loss: 10.2005 LR: 0.000030
+12/09/2025 01:04:57 - INFO - __main__ - Step: 8030 Loss: 10.2280 LR: 0.000030
+12/09/2025 01:05:40 - INFO - __main__ - Step: 8040 Loss: 10.2187 LR: 0.000030
+12/09/2025 01:06:22 - INFO - __main__ - Step: 8050 Loss: 10.3216 LR: 0.000030
+12/09/2025 01:07:05 - INFO - __main__ - Step: 8060 Loss: 10.3150 LR: 0.000030
+12/09/2025 01:07:46 - INFO - __main__ - Step: 8070 Loss: 10.2519 LR: 0.000030
+12/09/2025 01:08:27 - INFO - __main__ - Step: 8080 Loss: 10.1123 LR: 0.000030
+12/09/2025 01:09:08 - INFO - __main__ - Step: 8090 Loss: 10.2094 LR: 0.000030
+12/09/2025 01:09:49 - INFO - __main__ - Step: 8100 Loss: 10.2756 LR: 0.000030
+12/09/2025 01:10:33 - INFO - __main__ - Step: 8110 Loss: 10.2123 LR: 0.000030
+12/09/2025 01:11:14 - INFO - __main__ - Step: 8120 Loss: 10.2951 LR: 0.000030
+12/09/2025 01:11:54 - INFO - __main__ - Step: 8130 Loss: 10.2351 LR: 0.000030
+12/09/2025 01:12:34 - INFO - __main__ - Step: 8140 Loss: 10.2263 LR: 0.000030
+12/09/2025 01:13:16 - INFO - __main__ - Step: 8150 Loss: 10.3499 LR: 0.000030
+12/09/2025 01:13:59 - INFO - __main__ - Step: 8160 Loss: 10.1430 LR: 0.000030
+12/09/2025 01:14:41 - INFO - __main__ - Step: 8170 Loss: 10.1110 LR: 0.000030
+12/09/2025 01:15:22 - INFO - __main__ - Step: 8180 Loss: 10.3154 LR: 0.000030
+12/09/2025 01:16:04 - INFO - __main__ - Step: 8190 Loss: 10.2972 LR: 0.000030
+12/09/2025 01:16:44 - INFO - __main__ - Step: 8200 Loss: 10.3163 LR: 0.000030
+12/09/2025 01:17:24 - INFO - __main__ - Step: 8210 Loss: 10.2279 LR: 0.000030
+12/09/2025 01:18:05 - INFO - __main__ - Step: 8220 Loss: 10.1925 LR: 0.000030
+12/09/2025 01:18:45 - INFO - __main__ - Step: 8230 Loss: 10.2277 LR: 0.000030
+12/09/2025 01:19:27 - INFO - __main__ - Step: 8240 Loss: 10.2339 LR: 0.000030
+12/09/2025 01:20:07 - INFO - __main__ - Step: 8250 Loss: 10.2727 LR: 0.000030
+12/09/2025 01:20:46 - INFO - __main__ - Step: 8260 Loss: 10.2094 LR: 0.000030
+12/09/2025 01:21:26 - INFO - __main__ - Step: 8270 Loss: 10.2677 LR: 0.000030
+12/09/2025 01:22:06 - INFO - __main__ - Step: 8280 Loss: 10.2524 LR: 0.000030
+12/09/2025 01:22:47 - INFO - __main__ - Step: 8290 Loss: 10.1834 LR: 0.000030
+12/09/2025 01:23:28 - INFO - __main__ - Step: 8300 Loss: 10.2830 LR: 0.000030
+12/09/2025 01:24:09 - INFO - __main__ - Step: 8310 Loss: 10.2735 LR: 0.000030
+12/09/2025 01:24:49 - INFO - __main__ - Step: 8320 Loss: 10.2476 LR: 0.000030
+12/09/2025 01:25:31 - INFO - __main__ - Step: 8330 Loss: 10.2696 LR: 0.000030
+12/09/2025 01:26:10 - INFO - __main__ - Step: 8340 Loss: 10.2254 LR: 0.000030
+12/09/2025 01:26:51 - INFO - __main__ - Step: 8350 Loss: 9.8749 LR: 0.000030
+12/09/2025 01:27:31 - INFO - __main__ - Step: 8360 Loss: 10.2868 LR: 0.000030
+12/09/2025 01:28:12 - INFO - __main__ - Step: 8370 Loss: 10.3299 LR: 0.000030
+12/09/2025 01:28:53 - INFO - __main__ - Step: 8380 Loss: 10.1383 LR: 0.000030
+12/09/2025 01:29:32 - INFO - __main__ - Step: 8390 Loss: 10.1987 LR: 0.000030
+12/09/2025 01:30:12 - INFO - __main__ - Step: 8400 Loss: 10.3752 LR: 0.000030
+12/09/2025 01:30:52 - INFO - __main__ - Step: 8410 Loss: 10.0485 LR: 0.000030
+12/09/2025 01:31:33 - INFO - __main__ - Step: 8420 Loss: 10.1510 LR: 0.000030
+12/09/2025 01:32:14 - INFO - __main__ - Step: 8430 Loss: 10.0730 LR: 0.000030
+12/09/2025 01:32:55 - INFO - __main__ - Step: 8440 Loss: 10.1951 LR: 0.000030
+12/09/2025 01:33:35 - INFO - __main__ - Step: 8450 Loss: 10.2214 LR: 0.000030
+12/09/2025 01:34:16 - INFO - __main__ - Step: 8460 Loss: 10.2708 LR: 0.000030
+12/09/2025 01:34:57 - INFO - __main__ - Step: 8470 Loss: 9.8752 LR: 0.000030
+12/09/2025 01:35:37 - INFO - __main__ - Step: 8480 Loss: 10.2421 LR: 0.000030
+12/09/2025 01:36:17 - INFO - __main__ - Step: 8490 Loss: 10.2831 LR: 0.000030
+12/09/2025 01:36:57 - INFO - __main__ - Step: 8500 Loss: 10.2521 LR: 0.000030
+12/09/2025 01:36:57 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500
+12/09/2025 01:37:06 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500/optimizer.bin
+12/09/2025 01:37:06 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500/scheduler.bin
+12/09/2025 01:37:06 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500/sampler.bin
+12/09/2025 01:37:06 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500/random_states_0.pkl
+12/09/2025 01:37:06 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-8500
+12/09/2025 01:37:06 - INFO - __main__ - Generating videos for validation...
+12/09/2025 01:37:06 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.80it/s]
+12/09/2025 01:37:12 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 01:37:53 - INFO - __main__ - Step: 8510 Loss: 10.2791 LR: 0.000030
+12/09/2025 01:38:33 - INFO - __main__ - Step: 8520 Loss: 10.2445 LR: 0.000030
+12/09/2025 01:39:14 - INFO - __main__ - Step: 8530 Loss: 10.0240 LR: 0.000030
+12/09/2025 01:39:55 - INFO - __main__ - Step: 8540 Loss: 10.2756 LR: 0.000030
+12/09/2025 01:40:40 - INFO - __main__ - Step: 8550 Loss: 10.1570 LR: 0.000030
+12/09/2025 01:41:21 - INFO - __main__ - Step: 8560 Loss: 10.1915 LR: 0.000030
+12/09/2025 01:42:02 - INFO - __main__ - Step: 8570 Loss: 10.2468 LR: 0.000030
+12/09/2025 01:42:43 - INFO - __main__ - Step: 8580 Loss: 10.2841 LR: 0.000030
+12/09/2025 01:43:26 - INFO - __main__ - Step: 8590 Loss: 10.3508 LR: 0.000030
+12/09/2025 01:44:06 - INFO - __main__ - Step: 8600 Loss: 10.2955 LR: 0.000030
+12/09/2025 01:44:46 - INFO - __main__ - Step: 8610 Loss: 10.2341 LR: 0.000030
+12/09/2025 01:45:26 - INFO - __main__ - Step: 8620 Loss: 10.3108 LR: 0.000030
+12/09/2025 01:46:08 - INFO - __main__ - Step: 8630 Loss: 10.2283 LR: 0.000030
+12/09/2025 01:46:49 - INFO - __main__ - Step: 8640 Loss: 10.1501 LR: 0.000030
+12/09/2025 01:47:30 - INFO - __main__ - Step: 8650 Loss: 10.4081 LR: 0.000030
+12/09/2025 01:48:11 - INFO - __main__ - Step: 8660 Loss: 10.1406 LR: 0.000030
+12/09/2025 01:48:52 - INFO - __main__ - Step: 8670 Loss: 10.1809 LR: 0.000030
+12/09/2025 01:49:33 - INFO - __main__ - Step: 8680 Loss: 10.3772 LR: 0.000030
+12/09/2025 01:50:14 - INFO - __main__ - Step: 8690 Loss: 10.2832 LR: 0.000030
+12/09/2025 01:50:54 - INFO - __main__ - Step: 8700 Loss: 10.3794 LR: 0.000030
+12/09/2025 01:51:35 - INFO - __main__ - Step: 8710 Loss: 9.9623 LR: 0.000030
+12/09/2025 01:52:15 - INFO - __main__ - Step: 8720 Loss: 10.1601 LR: 0.000030
+12/09/2025 01:52:57 - INFO - __main__ - Step: 8730 Loss: 10.2284 LR: 0.000030
+12/09/2025 01:53:39 - INFO - __main__ - Step: 8740 Loss: 10.0540 LR: 0.000030
+12/09/2025 01:54:20 - INFO - __main__ - Step: 8750 Loss: 10.1500 LR: 0.000030
+12/09/2025 01:55:01 - INFO - __main__ - Step: 8760 Loss: 10.0504 LR: 0.000030
+12/09/2025 01:55:43 - INFO - __main__ - Step: 8770 Loss: 10.1850 LR: 0.000030
+12/09/2025 01:56:23 - INFO - __main__ - Step: 8780 Loss: 10.1295 LR: 0.000030
+12/09/2025 01:57:03 - INFO - __main__ - Step: 8790 Loss: 10.3019 LR: 0.000030
+12/09/2025 01:57:44 - INFO - __main__ - Step: 8800 Loss: 10.3225 LR: 0.000030
+12/09/2025 01:58:25 - INFO - __main__ - Step: 8810 Loss: 10.2366 LR: 0.000030
+12/09/2025 01:59:06 - INFO - __main__ - Step: 8820 Loss: 10.2086 LR: 0.000030
+12/09/2025 01:59:47 - INFO - __main__ - Step: 8830 Loss: 10.2784 LR: 0.000030
+12/09/2025 02:00:27 - INFO - __main__ - Step: 8840 Loss: 10.3409 LR: 0.000030
+12/09/2025 02:01:08 - INFO - __main__ - Step: 8850 Loss: 10.2942 LR: 0.000030
+12/09/2025 02:01:49 - INFO - __main__ - Step: 8860 Loss: 10.1809 LR: 0.000030
+12/09/2025 02:02:28 - INFO - __main__ - Step: 8870 Loss: 10.2588 LR: 0.000030
+12/09/2025 02:03:10 - INFO - __main__ - Step: 8880 Loss: 10.1412 LR: 0.000030
+12/09/2025 02:03:52 - INFO - __main__ - Step: 8890 Loss: 10.2051 LR: 0.000030
+12/09/2025 02:04:32 - INFO - __main__ - Step: 8900 Loss: 10.2463 LR: 0.000030
+12/09/2025 02:05:11 - INFO - __main__ - Step: 8910 Loss: 10.3263 LR: 0.000030
+12/09/2025 02:05:52 - INFO - __main__ - Step: 8920 Loss: 10.2405 LR: 0.000030
+12/09/2025 02:06:32 - INFO - __main__ - Step: 8930 Loss: 10.4588 LR: 0.000030
+12/09/2025 02:07:11 - INFO - __main__ - Step: 8940 Loss: 10.0509 LR: 0.000030
+12/09/2025 02:07:51 - INFO - __main__ - Step: 8950 Loss: 10.1175 LR: 0.000030
+12/09/2025 02:08:30 - INFO - __main__ - Step: 8960 Loss: 10.1086 LR: 0.000030
+12/09/2025 02:09:10 - INFO - __main__ - Step: 8970 Loss: 10.2881 LR: 0.000030
+12/09/2025 02:09:49 - INFO - __main__ - Step: 8980 Loss: 9.8539 LR: 0.000030
+12/09/2025 02:10:31 - INFO - __main__ - Step: 8990 Loss: 10.3159 LR: 0.000030
+12/09/2025 02:11:11 - INFO - __main__ - Step: 9000 Loss: 10.3114 LR: 0.000030
+12/09/2025 02:11:11 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000
+12/09/2025 02:11:18 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000/optimizer.bin
+12/09/2025 02:11:18 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000/scheduler.bin
+12/09/2025 02:11:18 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000/sampler.bin
+12/09/2025 02:11:18 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000/random_states_0.pkl
+12/09/2025 02:11:18 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9000
+12/09/2025 02:11:18 - INFO - __main__ - Generating videos for validation...
+12/09/2025 02:11:18 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.79it/s]
+12/09/2025 02:11:25 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 02:12:05 - INFO - __main__ - Step: 9010 Loss: 10.3593 LR: 0.000030
+12/09/2025 02:12:46 - INFO - __main__ - Step: 9020 Loss: 10.3540 LR: 0.000030
+12/09/2025 02:13:30 - INFO - __main__ - Step: 9030 Loss: 10.0327 LR: 0.000030
+12/09/2025 02:14:11 - INFO - __main__ - Step: 9040 Loss: 10.3993 LR: 0.000030
+12/09/2025 02:14:52 - INFO - __main__ - Step: 9050 Loss: 10.1170 LR: 0.000030
+12/09/2025 02:15:33 - INFO - __main__ - Step: 9060 Loss: 10.2321 LR: 0.000030
+12/09/2025 02:16:14 - INFO - __main__ - Step: 9070 Loss: 10.2650 LR: 0.000030
+12/09/2025 02:16:55 - INFO - __main__ - Step: 9080 Loss: 10.3733 LR: 0.000030
+12/09/2025 02:17:36 - INFO - __main__ - Step: 9090 Loss: 10.3145 LR: 0.000030
+12/09/2025 02:18:16 - INFO - __main__ - Step: 9100 Loss: 10.1167 LR: 0.000030
+12/09/2025 02:18:57 - INFO - __main__ - Step: 9110 Loss: 10.1362 LR: 0.000030
+12/09/2025 02:19:38 - INFO - __main__ - Step: 9120 Loss: 10.2166 LR: 0.000030
+12/09/2025 02:20:18 - INFO - __main__ - Step: 9130 Loss: 10.3137 LR: 0.000030
+12/09/2025 02:20:59 - INFO - __main__ - Step: 9140 Loss: 10.3315 LR: 0.000030
+12/09/2025 02:21:40 - INFO - __main__ - Step: 9150 Loss: 10.3195 LR: 0.000030
+12/09/2025 02:22:21 - INFO - __main__ - Step: 9160 Loss: 10.2026 LR: 0.000030
+12/09/2025 02:23:00 - INFO - __main__ - Step: 9170 Loss: 10.3019 LR: 0.000030
+12/09/2025 02:23:42 - INFO - __main__ - Step: 9180 Loss: 10.2923 LR: 0.000030
+12/09/2025 02:24:23 - INFO - __main__ - Step: 9190 Loss: 10.3703 LR: 0.000030
+12/09/2025 02:25:04 - INFO - __main__ - Step: 9200 Loss: 10.0775 LR: 0.000030
+12/09/2025 02:25:44 - INFO - __main__ - Step: 9210 Loss: 10.0284 LR: 0.000030
+12/09/2025 02:26:26 - INFO - __main__ - Step: 9220 Loss: 10.0956 LR: 0.000030
+12/09/2025 02:27:06 - INFO - __main__ - Step: 9230 Loss: 10.3597 LR: 0.000030
+12/09/2025 02:27:47 - INFO - __main__ - Step: 9240 Loss: 10.1692 LR: 0.000030
+12/09/2025 02:28:29 - INFO - __main__ - Step: 9250 Loss: 10.3566 LR: 0.000030
+12/09/2025 02:29:09 - INFO - __main__ - Step: 9260 Loss: 9.9540 LR: 0.000030
+12/09/2025 02:29:50 - INFO - __main__ - Step: 9270 Loss: 10.1716 LR: 0.000030
+12/09/2025 02:30:30 - INFO - __main__ - Step: 9280 Loss: 10.0799 LR: 0.000030
+12/09/2025 02:31:11 - INFO - __main__ - Step: 9290 Loss: 10.2651 LR: 0.000030
+12/09/2025 02:31:52 - INFO - __main__ - Step: 9300 Loss: 10.2888 LR: 0.000030
+12/09/2025 02:32:33 - INFO - __main__ - Step: 9310 Loss: 10.3632 LR: 0.000030
+12/09/2025 02:33:13 - INFO - __main__ - Step: 9320 Loss: 10.1456 LR: 0.000030
+12/09/2025 02:33:54 - INFO - __main__ - Step: 9330 Loss: 10.0606 LR: 0.000030
+12/09/2025 02:34:34 - INFO - __main__ - Step: 9340 Loss: 10.1240 LR: 0.000030
+12/09/2025 02:35:15 - INFO - __main__ - Step: 9350 Loss: 10.3006 LR: 0.000030
+12/09/2025 02:35:57 - INFO - __main__ - Step: 9360 Loss: 10.3873 LR: 0.000030
+12/09/2025 02:36:38 - INFO - __main__ - Step: 9370 Loss: 10.2983 LR: 0.000030
+12/09/2025 02:37:18 - INFO - __main__ - Step: 9380 Loss: 10.3257 LR: 0.000030
+12/09/2025 02:37:58 - INFO - __main__ - Step: 9390 Loss: 10.2567 LR: 0.000030
+12/09/2025 02:38:39 - INFO - __main__ - Step: 9400 Loss: 10.3199 LR: 0.000030
+12/09/2025 02:39:18 - INFO - __main__ - Step: 9410 Loss: 9.8679 LR: 0.000030
+12/09/2025 02:39:59 - INFO - __main__ - Step: 9420 Loss: 10.3053 LR: 0.000030
+12/09/2025 02:40:41 - INFO - __main__ - Step: 9430 Loss: 10.1604 LR: 0.000030
+12/09/2025 02:41:23 - INFO - __main__ - Step: 9440 Loss: 10.2145 LR: 0.000030
+12/09/2025 02:42:03 - INFO - __main__ - Step: 9450 Loss: 10.3038 LR: 0.000030
+12/09/2025 02:42:44 - INFO - __main__ - Step: 9460 Loss: 10.2254 LR: 0.000030
+12/09/2025 02:43:25 - INFO - __main__ - Step: 9470 Loss: 10.4178 LR: 0.000030
+12/09/2025 02:44:06 - INFO - __main__ - Step: 9480 Loss: 10.1861 LR: 0.000030
+12/09/2025 02:44:47 - INFO - __main__ - Step: 9490 Loss: 10.3053 LR: 0.000030
+12/09/2025 02:45:27 - INFO - __main__ - Step: 9500 Loss: 10.0535 LR: 0.000030
+12/09/2025 02:45:27 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500
+12/09/2025 02:45:37 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500/optimizer.bin
+12/09/2025 02:45:37 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500/scheduler.bin
+12/09/2025 02:45:37 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500/sampler.bin
+12/09/2025 02:45:37 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500/random_states_0.pkl
+12/09/2025 02:45:37 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-9500
+12/09/2025 02:45:37 - INFO - __main__ - Generating videos for validation...
+12/09/2025 02:45:37 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.69it/s]
+12/09/2025 02:45:44 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 02:46:25 - INFO - __main__ - Step: 9510 Loss: 10.1563 LR: 0.000030
+12/09/2025 02:47:07 - INFO - __main__ - Step: 9520 Loss: 10.0036 LR: 0.000030
+12/09/2025 02:47:50 - INFO - __main__ - Step: 9530 Loss: 10.0589 LR: 0.000030
+12/09/2025 02:48:32 - INFO - __main__ - Step: 9540 Loss: 10.3363 LR: 0.000030
+12/09/2025 02:49:14 - INFO - __main__ - Step: 9550 Loss: 10.3414 LR: 0.000030
+12/09/2025 02:49:55 - INFO - __main__ - Step: 9560 Loss: 10.2372 LR: 0.000030
+12/09/2025 02:50:37 - INFO - __main__ - Step: 9570 Loss: 10.1867 LR: 0.000030
+12/09/2025 02:51:18 - INFO - __main__ - Step: 9580 Loss: 10.2701 LR: 0.000030
+12/09/2025 02:52:01 - INFO - __main__ - Step: 9590 Loss: 9.9317 LR: 0.000030
+12/09/2025 02:52:43 - INFO - __main__ - Step: 9600 Loss: 10.1395 LR: 0.000030
+12/09/2025 02:53:26 - INFO - __main__ - Step: 9610 Loss: 10.1316 LR: 0.000030
+12/09/2025 02:54:08 - INFO - __main__ - Step: 9620 Loss: 10.2242 LR: 0.000030
+12/09/2025 02:54:51 - INFO - __main__ - Step: 9630 Loss: 10.2056 LR: 0.000030
+12/09/2025 02:55:31 - INFO - __main__ - Step: 9640 Loss: 9.9158 LR: 0.000030
+12/09/2025 02:56:12 - INFO - __main__ - Step: 9650 Loss: 10.2432 LR: 0.000030
+12/09/2025 02:56:53 - INFO - __main__ - Step: 9660 Loss: 10.3903 LR: 0.000030
+12/09/2025 02:57:35 - INFO - __main__ - Step: 9670 Loss: 10.2608 LR: 0.000030
+12/09/2025 02:58:16 - INFO - __main__ - Step: 9680 Loss: 10.2800 LR: 0.000030
+12/09/2025 02:58:57 - INFO - __main__ - Step: 9690 Loss: 10.1646 LR: 0.000030
+12/09/2025 02:59:40 - INFO - __main__ - Step: 9700 Loss: 10.3005 LR: 0.000030
+12/09/2025 03:00:20 - INFO - __main__ - Step: 9710 Loss: 10.3200 LR: 0.000030
+12/09/2025 03:01:03 - INFO - __main__ - Step: 9720 Loss: 10.1391 LR: 0.000030
+12/09/2025 03:01:46 - INFO - __main__ - Step: 9730 Loss: 10.0638 LR: 0.000030
+12/09/2025 03:02:27 - INFO - __main__ - Step: 9740 Loss: 10.3703 LR: 0.000030
+12/09/2025 03:03:08 - INFO - __main__ - Step: 9750 Loss: 10.0388 LR: 0.000030
+12/09/2025 03:03:50 - INFO - __main__ - Step: 9760 Loss: 10.2022 LR: 0.000030
+12/09/2025 03:04:33 - INFO - __main__ - Step: 9770 Loss: 10.2557 LR: 0.000030
+12/09/2025 03:05:15 - INFO - __main__ - Step: 9780 Loss: 10.3068 LR: 0.000030
+12/09/2025 03:05:55 - INFO - __main__ - Step: 9790 Loss: 10.1136 LR: 0.000030
+12/09/2025 03:06:36 - INFO - __main__ - Step: 9800 Loss: 10.0547 LR: 0.000030
+12/09/2025 03:07:18 - INFO - __main__ - Step: 9810 Loss: 10.1827 LR: 0.000030
+12/09/2025 03:08:01 - INFO - __main__ - Step: 9820 Loss: 10.2698 LR: 0.000030
+12/09/2025 03:08:42 - INFO - __main__ - Step: 9830 Loss: 10.3142 LR: 0.000030
+12/09/2025 03:09:25 - INFO - __main__ - Step: 9840 Loss: 10.1912 LR: 0.000030
+12/09/2025 03:10:07 - INFO - __main__ - Step: 9850 Loss: 10.1412 LR: 0.000030
+12/09/2025 03:10:48 - INFO - __main__ - Step: 9860 Loss: 10.3545 LR: 0.000030
+12/09/2025 03:11:30 - INFO - __main__ - Step: 9870 Loss: 10.2420 LR: 0.000030
+12/09/2025 03:12:12 - INFO - __main__ - Step: 9880 Loss: 10.0121 LR: 0.000030
+12/09/2025 03:12:53 - INFO - __main__ - Step: 9890 Loss: 10.1630 LR: 0.000030
+12/09/2025 03:13:35 - INFO - __main__ - Step: 9900 Loss: 10.2106 LR: 0.000030
+12/09/2025 03:14:17 - INFO - __main__ - Step: 9910 Loss: 9.8999 LR: 0.000030
+12/09/2025 03:14:58 - INFO - __main__ - Step: 9920 Loss: 10.2557 LR: 0.000030
+12/09/2025 03:15:42 - INFO - __main__ - Step: 9930 Loss: 9.9058 LR: 0.000030
+12/09/2025 03:16:22 - INFO - __main__ - Step: 9940 Loss: 10.3045 LR: 0.000030
+12/09/2025 03:17:04 - INFO - __main__ - Step: 9950 Loss: 10.1141 LR: 0.000030
+12/09/2025 03:17:46 - INFO - __main__ - Step: 9960 Loss: 9.8588 LR: 0.000030
+12/09/2025 03:18:26 - INFO - __main__ - Step: 9970 Loss: 10.1512 LR: 0.000030
+12/09/2025 03:19:07 - INFO - __main__ - Step: 9980 Loss: 10.2319 LR: 0.000030
+12/09/2025 03:19:48 - INFO - __main__ - Step: 9990 Loss: 10.2942 LR: 0.000030
+12/09/2025 03:20:29 - INFO - __main__ - Step: 10000 Loss: 10.2682 LR: 0.000030
+12/09/2025 03:20:29 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000
+12/09/2025 03:20:36 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/optimizer.bin
+12/09/2025 03:20:36 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/scheduler.bin
+12/09/2025 03:20:36 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/sampler.bin
+12/09/2025 03:20:36 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/random_states_0.pkl
+12/09/2025 03:20:36 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000
+12/09/2025 03:20:36 - INFO - __main__ - Generating videos for validation...
+12/09/2025 03:20:36 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.87it/s]
+12/09/2025 03:20:43 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio
+12/09/2025 03:20:43 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000
+12/09/2025 03:21:33 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/optimizer.bin
+12/09/2025 03:21:33 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/scheduler.bin
+12/09/2025 03:21:33 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/sampler.bin
+12/09/2025 03:21:33 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000/random_states_0.pkl
+12/09/2025 03:21:33 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio/checkpoint-10000
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/requirements.txt b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-metadata.json b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..9f3fc9277d595e50f2605af5f4a0975cbb8a2651
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-08T15:59:43.669474Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "11900305113088"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "f0bt4siuu2gjzmv16hc4nme2mcfrsfsi"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-summary.json b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..3b953ddbecc4ea9ace17ddb013fd672872480265
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_timestamp":1.76525044327647e+09,"generated_videos_first_frame":{"filenames":["media/images/generated_videos_first_frame_10000_8328d2d0556a95ff2759.png","media/images/generated_videos_first_frame_10000_980ee3261a5cf9cce942.png"],"captions":["a cat playing","a girl walking"],"_type":"images/separated","width":448,"height":256,"format":"png","count":2},"_wandb":{"runtime":40912},"_runtime":40912.716925498,"_step":10000,"step_loss":10.268203735351562,"lr":2.9999999999999997e-05,"avg_masking_rate":0.7040722370147705}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-core.log b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..6841dc7c46b112e0e43c985c9bbd45c460faaa0d
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-core.log
@@ -0,0 +1,16 @@
+{"time":"2025-12-08T15:59:43.744475852Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpmaf9gegn/port-3462940.txt","pid":3462940,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-08T15:59:43.744995335Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3462940}
+{"time":"2025-12-08T15:59:43.744970513Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3462940-3463253-4089553982/socket","Net":"unix"}}
+{"time":"2025-12-08T15:59:43.929408895Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-08T15:59:43.935485908Z","level":"INFO","msg":"handleInformInit: received","streamId":"j5rc8ish","id":"1(@)"}
+{"time":"2025-12-08T15:59:44.101834101Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"j5rc8ish","id":"1(@)"}
+{"time":"2025-12-09T03:21:37.737213016Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"j5rc8ish","id":"1(@)"}
+{"time":"2025-12-09T03:21:37.739815436Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"j5rc8ish","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.88051873Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.880564702Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.88057283Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T03:22:18.880597849Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.880709214Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.880741969Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T03:22:18.880796739Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-3462940-3463253-4089553982/socket","Net":"unix"}}
+{"time":"2025-12-09T03:22:18.880837027Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-internal.log b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..95a069e458685459f654cd342fc0f563e3763ff1
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-internal.log
@@ -0,0 +1,12 @@
+{"time":"2025-12-08T15:59:43.935580663Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-08T15:59:44.101633338Z","level":"INFO","msg":"stream: created new stream","id":"j5rc8ish"}
+{"time":"2025-12-08T15:59:44.101738834Z","level":"INFO","msg":"handler: started","stream_id":"j5rc8ish"}
+{"time":"2025-12-08T15:59:44.101827368Z","level":"INFO","msg":"stream: started","id":"j5rc8ish"}
+{"time":"2025-12-08T15:59:44.101850473Z","level":"INFO","msg":"sender: started","stream_id":"j5rc8ish"}
+{"time":"2025-12-08T15:59:44.101851999Z","level":"INFO","msg":"writer: started","stream_id":"j5rc8ish"}
+{"time":"2025-12-09T03:21:37.602552256Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T03:21:37.733763019Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-12-09T03:21:37.737771064Z","level":"INFO","msg":"stream: closing","id":"j5rc8ish"}
+{"time":"2025-12-09T03:21:37.737788193Z","level":"INFO","msg":"handler: closed","stream_id":"j5rc8ish"}
+{"time":"2025-12-09T03:21:37.737842593Z","level":"INFO","msg":"sender: closed","stream_id":"j5rc8ish"}
+{"time":"2025-12-09T03:21:37.73786844Z","level":"INFO","msg":"stream: closed","id":"j5rc8ish"}
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug.log b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..7a4809ac23c1f44e627d3d3eebd7fc37d752fe2b
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug.log
@@ -0,0 +1,27 @@
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_setup.py:_flush():80] Configure stats pid to 3462940
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug.log
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251208_155943-j5rc8ish/logs/debug-internal.log
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_init.py:init():841] calling init triggers
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-08 15:59:43,673 INFO MainThread:3462940 [wandb_init.py:init():889] starting backend
+2025-12-08 15:59:43,929 INFO MainThread:3462940 [wandb_init.py:init():892] sending inform_init request
+2025-12-08 15:59:43,934 INFO MainThread:3462940 [wandb_init.py:init():900] backend started and connected
+2025-12-08 15:59:43,935 INFO MainThread:3462940 [wandb_init.py:init():970] updated telemetry
+2025-12-08 15:59:43,940 INFO MainThread:3462940 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-08 15:59:44,509 INFO MainThread:3462940 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-08 15:59:44,707 INFO MainThread:3462940 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-08 15:59:44,707 INFO MainThread:3462940 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-08 15:59:44,707 INFO MainThread:3462940 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-08 15:59:44,707 INFO MainThread:3462940 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-08 15:59:44,710 INFO MainThread:3462940 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-08 15:59:44,711 INFO MainThread:3462940 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_10_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.1}
+2025-12-09 03:21:37,223 INFO MainThread:3462940 [wandb_run.py:_finish():2287] finishing run jin-bin/meissonic/j5rc8ish
+2025-12-09 03:21:37,225 INFO MainThread:3462940 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0
+2025-12-09 03:21:37,225 INFO MainThread:3462940 [wandb_run.py:_restore():2468] restore
+2025-12-09 03:21:37,225 INFO MainThread:3462940 [wandb_run.py:_restore():2474] restore done
+2025-12-09 03:21:37,736 INFO MainThread:3462940 [wandb_run.py:_footer_sync_info():3862] logging synced files
diff --git a/Meissonic/wandb/run-20251208_155943-j5rc8ish/run-j5rc8ish.wandb b/Meissonic/wandb/run-20251208_155943-j5rc8ish/run-j5rc8ish.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..1cf5c5650e0e08e83074e1c99d544c76de08d3a8
--- /dev/null
+++ b/Meissonic/wandb/run-20251208_155943-j5rc8ish/run-j5rc8ish.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488845c8db48654841acaecdc70247dd695f96008f786b03f3db6824b3222477
+size 10711439
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_4ea9441b252682155006.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_4ea9441b252682155006.png
new file mode 100644
index 0000000000000000000000000000000000000000..e3686752e68ea4903b416f823d7715d05e0722b4
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_4ea9441b252682155006.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ea9441b2526821550065be4282383a91f18e8f289e6f64c5286893c87f4e5e5
+size 235465
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_be5afcc9b61ce7cc9765.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_be5afcc9b61ce7cc9765.png
new file mode 100644
index 0000000000000000000000000000000000000000..5dadeb5a0b3e104d148a2027d8c36f8c835acb99
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1000_be5afcc9b61ce7cc9765.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be5afcc9b61ce7cc976552a10997db35e39d3777b8ba46832f7b730c1fab3104
+size 234944
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_7c59a605f746fefa06f3.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_7c59a605f746fefa06f3.png
new file mode 100644
index 0000000000000000000000000000000000000000..aa69874ba98b31a5706dbdae389a2ff9f07d8bf4
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_7c59a605f746fefa06f3.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c59a605f746fefa06f3c8c0981023f314a8343302ce0498fbb4aebb85b5ea48
+size 234597
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_e846322d8d1fe1da0c06.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_e846322d8d1fe1da0c06.png
new file mode 100644
index 0000000000000000000000000000000000000000..6c57f92a48a45124c8047e429dabfc630e682135
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_1500_e846322d8d1fe1da0c06.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e846322d8d1fe1da0c06af5a8f74c3afcadf53c258aa36a35d4dc90e6722dbc3
+size 229025
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_e7251adb287026b97ff8.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_e7251adb287026b97ff8.png
new file mode 100644
index 0000000000000000000000000000000000000000..288b5357bfd6f44c47c059f1b852223a08d2eea7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_e7251adb287026b97ff8.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7251adb287026b97ff807bb6fa787d23e29837fb1ab75855513e00dc2679e7d
+size 232454
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_fb353e591b1b0dbac386.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_fb353e591b1b0dbac386.png
new file mode 100644
index 0000000000000000000000000000000000000000..eab416e77eed30c536341ec9692595b8f1b922ac
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2000_fb353e591b1b0dbac386.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb353e591b1b0dbac38635e86c24871936ac696f4ff7674771a6ef15a513e45e
+size 229162
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_4254c55c5a44dae8222b.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_4254c55c5a44dae8222b.png
new file mode 100644
index 0000000000000000000000000000000000000000..257d8ee8ee776ff5b4131d710fcfc0c5ca82a250
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_4254c55c5a44dae8222b.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4254c55c5a44dae8222b2f94225010db931e57096a09925bac1e768f58993672
+size 236913
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_880fb5b7bb7d55a41102.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_880fb5b7bb7d55a41102.png
new file mode 100644
index 0000000000000000000000000000000000000000..db3f8e2868d11f353f5051aeb20b5e5c095b4dd8
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_2500_880fb5b7bb7d55a41102.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:880fb5b7bb7d55a411020da08650dc25fe858c588e36c6173cc832251ec76713
+size 234622
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_0af47ac2b0fd0a7b83b9.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_0af47ac2b0fd0a7b83b9.png
new file mode 100644
index 0000000000000000000000000000000000000000..a66371578a8fe0146839e4faac46604f2d924376
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_0af47ac2b0fd0a7b83b9.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0af47ac2b0fd0a7b83b953ffea43750361aa0fa8df8302b1b8e87b311f86db99
+size 235174
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_38859ead3b87553090be.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_38859ead3b87553090be.png
new file mode 100644
index 0000000000000000000000000000000000000000..14629585c7a65a8365edf4a781eb7e55bf217720
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3000_38859ead3b87553090be.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38859ead3b87553090be10142facdc7a4f8780e6d9d0b17eb4abf0bc124b0746
+size 229801
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_1b3f708ccf2664b9bd84.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_1b3f708ccf2664b9bd84.png
new file mode 100644
index 0000000000000000000000000000000000000000..eb64442cbc0de5576a61c8404e2bce41579d5f5e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_1b3f708ccf2664b9bd84.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b3f708ccf2664b9bd84c4142e94f63ed1f1d9fba426b1835eb30e04aad78ab3
+size 240388
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_96fc2c23d9374b5c001f.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_96fc2c23d9374b5c001f.png
new file mode 100644
index 0000000000000000000000000000000000000000..768e69413dff2086b1265e4fd28b894fa044ed6a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_3500_96fc2c23d9374b5c001f.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96fc2c23d9374b5c001fbbcc7e3cf2abc977c4fd89bff8872af08c61ab92cc00
+size 236161
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_7f60fcf85257e0427cb4.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_7f60fcf85257e0427cb4.png
new file mode 100644
index 0000000000000000000000000000000000000000..933734f026856731f07931a02829589ce131bd38
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_7f60fcf85257e0427cb4.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f60fcf85257e0427cb40bd8076e1243f9e42adfe61075674d62b0d5841bdc6a
+size 231968
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_f36bf77eea280b84a34e.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_f36bf77eea280b84a34e.png
new file mode 100644
index 0000000000000000000000000000000000000000..38098b236aed7aa104076c52235d007303a98341
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_4000_f36bf77eea280b84a34e.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f36bf77eea280b84a34ed04791d44d542d3b4425864c9020408af51961bb2ba2
+size 227665
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_92b8064a4e25f8ad3702.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_92b8064a4e25f8ad3702.png
new file mode 100644
index 0000000000000000000000000000000000000000..9783534a5783b293f8e0b68a1fe9172cb02ecc76
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_92b8064a4e25f8ad3702.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92b8064a4e25f8ad370281aa9f5364c7b09de3cd29ae7ba63ca51dad6b402323
+size 227242
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_f6969510d28d905ce414.png b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_f6969510d28d905ce414.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4c5548f611f4b0aee0776407d41af5a021ad14d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/media/images/generated_videos_first_frame_500_f6969510d28d905ce414.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6969510d28d905ce414921aa949d88990ca6b2d81ab742cda228c22accb0028
+size 209920
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/output.log b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..4549e12252fbc0b4acbd0e69ff8f35f4fd7b51eb
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/output.log
@@ -0,0 +1,542 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 67.35it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 06:08:59 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6234.90it/s]
+12/09/2025 06:09:01 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 06:09:01 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 06:09:01 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 06:09:03 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 06:09:19 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 06:09:19 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 06:09:38 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 06:09:41 - INFO - __main__ - Wan backbone parameters are frozen (requires_grad=False)
+12/09/2025 06:09:41 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 06:09:41 - INFO - __main__ - Wan backbone is frozen (lr=0)
+12/09/2025 06:09:41 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 06:09:41 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 06:09:50 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 06:09:50 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 06:09:50 - INFO - __main__ - Dataloader configuration:
+12/09/2025 06:09:50 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 06:09:50 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 06:09:50 - INFO - __main__ - - persistent_workers: True
+12/09/2025 06:09:50 - INFO - __main__ - - pin_memory: True
+12/09/2025 06:09:50 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 06:11:12 - INFO - __main__ - ***** Running training *****
+12/09/2025 06:11:12 - INFO - __main__ - Num training steps = 10000
+12/09/2025 06:11:12 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 06:11:12 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 06:11:12 - INFO - __main__ - Gradient Accumulation steps = 4
+12/09/2025 06:12:04 - INFO - __main__ - Step: 10 Loss: 11.1668 LR: 0.000000
+12/09/2025 06:12:43 - INFO - __main__ - Step: 20 Loss: 11.2193 LR: 0.000000
+12/09/2025 06:13:15 - INFO - __main__ - Step: 30 Loss: 11.2492 LR: 0.000000
+12/09/2025 06:13:53 - INFO - __main__ - Step: 40 Loss: 11.1764 LR: 0.000000
+12/09/2025 06:14:30 - INFO - __main__ - Step: 50 Loss: 11.2046 LR: 0.000000
+12/09/2025 06:15:02 - INFO - __main__ - Step: 60 Loss: 11.1739 LR: 0.000000
+12/09/2025 06:15:38 - INFO - __main__ - Step: 70 Loss: 11.1334 LR: 0.000000
+12/09/2025 06:16:15 - INFO - __main__ - Step: 80 Loss: 11.1807 LR: 0.000000
+12/09/2025 06:16:52 - INFO - __main__ - Step: 90 Loss: 11.1287 LR: 0.000000
+12/09/2025 06:17:27 - INFO - __main__ - Step: 100 Loss: 11.1015 LR: 0.000000
+12/09/2025 06:18:01 - INFO - __main__ - Step: 110 Loss: 11.1268 LR: 0.000000
+12/09/2025 06:18:35 - INFO - __main__ - Step: 120 Loss: 11.0750 LR: 0.000000
+12/09/2025 06:19:10 - INFO - __main__ - Step: 130 Loss: 11.0581 LR: 0.000000
+12/09/2025 06:19:43 - INFO - __main__ - Step: 140 Loss: 11.0381 LR: 0.000000
+12/09/2025 06:20:17 - INFO - __main__ - Step: 150 Loss: 11.0657 LR: 0.000000
+12/09/2025 06:20:51 - INFO - __main__ - Step: 160 Loss: 11.0320 LR: 0.000000
+12/09/2025 06:21:24 - INFO - __main__ - Step: 170 Loss: 11.0644 LR: 0.000000
+12/09/2025 06:21:57 - INFO - __main__ - Step: 180 Loss: 11.0552 LR: 0.000000
+12/09/2025 06:22:32 - INFO - __main__ - Step: 190 Loss: 10.9938 LR: 0.000000
+12/09/2025 06:23:09 - INFO - __main__ - Step: 200 Loss: 10.9979 LR: 0.000000
+12/09/2025 06:23:46 - INFO - __main__ - Step: 210 Loss: 10.9963 LR: 0.000000
+12/09/2025 06:24:21 - INFO - __main__ - Step: 220 Loss: 10.9833 LR: 0.000000
+12/09/2025 06:24:55 - INFO - __main__ - Step: 230 Loss: 10.9983 LR: 0.000000
+12/09/2025 06:25:29 - INFO - __main__ - Step: 240 Loss: 11.0060 LR: 0.000000
+12/09/2025 06:26:08 - INFO - __main__ - Step: 250 Loss: 10.9848 LR: 0.000000
+12/09/2025 06:26:42 - INFO - __main__ - Step: 260 Loss: 10.9357 LR: 0.000000
+12/09/2025 06:27:22 - INFO - __main__ - Step: 270 Loss: 10.9227 LR: 0.000000
+12/09/2025 06:27:58 - INFO - __main__ - Step: 280 Loss: 10.9936 LR: 0.000000
+12/09/2025 06:28:37 - INFO - __main__ - Step: 290 Loss: 10.9555 LR: 0.000000
+12/09/2025 06:29:13 - INFO - __main__ - Step: 300 Loss: 10.9174 LR: 0.000000
+12/09/2025 06:29:49 - INFO - __main__ - Step: 310 Loss: 10.9509 LR: 0.000000
+12/09/2025 06:30:25 - INFO - __main__ - Step: 320 Loss: 10.9304 LR: 0.000000
+12/09/2025 06:31:01 - INFO - __main__ - Step: 330 Loss: 10.9065 LR: 0.000000
+12/09/2025 06:31:37 - INFO - __main__ - Step: 340 Loss: 10.9084 LR: 0.000000
+12/09/2025 06:32:12 - INFO - __main__ - Step: 350 Loss: 11.0131 LR: 0.000000
+12/09/2025 06:32:48 - INFO - __main__ - Step: 360 Loss: 10.8504 LR: 0.000000
+12/09/2025 06:33:25 - INFO - __main__ - Step: 370 Loss: 10.8779 LR: 0.000000
+12/09/2025 06:34:01 - INFO - __main__ - Step: 380 Loss: 10.8450 LR: 0.000000
+12/09/2025 06:34:37 - INFO - __main__ - Step: 390 Loss: 10.8490 LR: 0.000000
+12/09/2025 06:35:14 - INFO - __main__ - Step: 400 Loss: 10.9187 LR: 0.000000
+12/09/2025 06:35:50 - INFO - __main__ - Step: 410 Loss: 10.8653 LR: 0.000000
+12/09/2025 06:36:26 - INFO - __main__ - Step: 420 Loss: 10.8252 LR: 0.000000
+12/09/2025 06:37:01 - INFO - __main__ - Step: 430 Loss: 10.8188 LR: 0.000000
+12/09/2025 06:37:37 - INFO - __main__ - Step: 440 Loss: 10.8241 LR: 0.000000
+12/09/2025 06:38:13 - INFO - __main__ - Step: 450 Loss: 10.8091 LR: 0.000000
+12/09/2025 06:38:48 - INFO - __main__ - Step: 460 Loss: 10.8458 LR: 0.000000
+12/09/2025 06:39:22 - INFO - __main__ - Step: 470 Loss: 10.8439 LR: 0.000000
+12/09/2025 06:39:57 - INFO - __main__ - Step: 480 Loss: 10.8652 LR: 0.000000
+12/09/2025 06:40:32 - INFO - __main__ - Step: 490 Loss: 10.9514 LR: 0.000000
+12/09/2025 06:41:07 - INFO - __main__ - Step: 500 Loss: 10.8374 LR: 0.000000
+12/09/2025 06:41:07 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500
+12/09/2025 06:41:13 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500/optimizer.bin
+12/09/2025 06:41:13 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500/scheduler.bin
+12/09/2025 06:41:13 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500/sampler.bin
+12/09/2025 06:41:13 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500/random_states_0.pkl
+12/09/2025 06:41:13 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-500
+12/09/2025 06:41:13 - INFO - __main__ - Generating videos for validation...
+12/09/2025 06:41:13 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.88it/s]
+12/09/2025 06:41:20 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 06:41:56 - INFO - __main__ - Step: 510 Loss: 10.7682 LR: 0.000000
+12/09/2025 06:42:31 - INFO - __main__ - Step: 520 Loss: 10.8190 LR: 0.000000
+12/09/2025 06:43:07 - INFO - __main__ - Step: 530 Loss: 10.7900 LR: 0.000000
+12/09/2025 06:43:43 - INFO - __main__ - Step: 540 Loss: 10.7587 LR: 0.000000
+12/09/2025 06:44:19 - INFO - __main__ - Step: 550 Loss: 10.8122 LR: 0.000000
+12/09/2025 06:44:54 - INFO - __main__ - Step: 560 Loss: 10.7749 LR: 0.000000
+12/09/2025 06:45:30 - INFO - __main__ - Step: 570 Loss: 10.9821 LR: 0.000000
+12/09/2025 06:46:05 - INFO - __main__ - Step: 580 Loss: 10.7714 LR: 0.000000
+12/09/2025 06:46:42 - INFO - __main__ - Step: 590 Loss: 10.8758 LR: 0.000000
+12/09/2025 06:47:18 - INFO - __main__ - Step: 600 Loss: 10.8953 LR: 0.000000
+12/09/2025 06:47:53 - INFO - __main__ - Step: 610 Loss: 10.7866 LR: 0.000000
+12/09/2025 06:48:30 - INFO - __main__ - Step: 620 Loss: 10.7730 LR: 0.000000
+12/09/2025 06:49:06 - INFO - __main__ - Step: 630 Loss: 10.9168 LR: 0.000000
+12/09/2025 06:49:42 - INFO - __main__ - Step: 640 Loss: 10.7946 LR: 0.000000
+12/09/2025 06:50:18 - INFO - __main__ - Step: 650 Loss: 10.8732 LR: 0.000000
+12/09/2025 06:50:53 - INFO - __main__ - Step: 660 Loss: 10.8493 LR: 0.000000
+12/09/2025 06:51:29 - INFO - __main__ - Step: 670 Loss: 10.7979 LR: 0.000000
+12/09/2025 06:52:05 - INFO - __main__ - Step: 680 Loss: 10.7677 LR: 0.000000
+12/09/2025 06:52:41 - INFO - __main__ - Step: 690 Loss: 10.8272 LR: 0.000000
+12/09/2025 06:53:15 - INFO - __main__ - Step: 700 Loss: 10.7772 LR: 0.000000
+12/09/2025 06:53:50 - INFO - __main__ - Step: 710 Loss: 10.8874 LR: 0.000000
+12/09/2025 06:54:25 - INFO - __main__ - Step: 720 Loss: 10.7958 LR: 0.000000
+12/09/2025 06:54:59 - INFO - __main__ - Step: 730 Loss: 10.7957 LR: 0.000000
+12/09/2025 06:55:34 - INFO - __main__ - Step: 740 Loss: 10.7506 LR: 0.000000
+12/09/2025 06:56:09 - INFO - __main__ - Step: 750 Loss: 10.7045 LR: 0.000000
+12/09/2025 06:56:44 - INFO - __main__ - Step: 760 Loss: 10.7127 LR: 0.000000
+12/09/2025 06:57:19 - INFO - __main__ - Step: 770 Loss: 10.7613 LR: 0.000000
+12/09/2025 06:57:55 - INFO - __main__ - Step: 780 Loss: 10.7651 LR: 0.000000
+12/09/2025 06:58:30 - INFO - __main__ - Step: 790 Loss: 10.7413 LR: 0.000000
+12/09/2025 06:59:05 - INFO - __main__ - Step: 800 Loss: 10.6675 LR: 0.000000
+12/09/2025 06:59:39 - INFO - __main__ - Step: 810 Loss: 10.7217 LR: 0.000000
+12/09/2025 07:00:13 - INFO - __main__ - Step: 820 Loss: 10.9675 LR: 0.000000
+12/09/2025 07:00:48 - INFO - __main__ - Step: 830 Loss: 10.8523 LR: 0.000000
+12/09/2025 07:01:23 - INFO - __main__ - Step: 840 Loss: 10.8386 LR: 0.000000
+12/09/2025 07:01:57 - INFO - __main__ - Step: 850 Loss: 10.9333 LR: 0.000000
+12/09/2025 07:02:32 - INFO - __main__ - Step: 860 Loss: 10.8703 LR: 0.000000
+12/09/2025 07:03:08 - INFO - __main__ - Step: 870 Loss: 10.7036 LR: 0.000000
+12/09/2025 07:03:43 - INFO - __main__ - Step: 880 Loss: 10.7645 LR: 0.000000
+12/09/2025 07:04:17 - INFO - __main__ - Step: 890 Loss: 10.8497 LR: 0.000000
+12/09/2025 07:04:51 - INFO - __main__ - Step: 900 Loss: 10.7641 LR: 0.000000
+12/09/2025 07:05:26 - INFO - __main__ - Step: 910 Loss: 10.7548 LR: 0.000000
+12/09/2025 07:06:01 - INFO - __main__ - Step: 920 Loss: 10.6804 LR: 0.000000
+12/09/2025 07:06:37 - INFO - __main__ - Step: 930 Loss: 10.7162 LR: 0.000000
+12/09/2025 07:07:12 - INFO - __main__ - Step: 940 Loss: 10.8392 LR: 0.000000
+12/09/2025 07:07:48 - INFO - __main__ - Step: 950 Loss: 10.8436 LR: 0.000000
+12/09/2025 07:08:22 - INFO - __main__ - Step: 960 Loss: 10.6719 LR: 0.000000
+12/09/2025 07:08:57 - INFO - __main__ - Step: 970 Loss: 10.7326 LR: 0.000000
+12/09/2025 07:09:32 - INFO - __main__ - Step: 980 Loss: 10.7283 LR: 0.000000
+12/09/2025 07:10:07 - INFO - __main__ - Step: 990 Loss: 10.7694 LR: 0.000000
+12/09/2025 07:10:41 - INFO - __main__ - Step: 1000 Loss: 10.7278 LR: 0.000000
+12/09/2025 07:10:41 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000
+12/09/2025 07:10:46 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000/optimizer.bin
+12/09/2025 07:10:46 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000/scheduler.bin
+12/09/2025 07:10:46 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000/sampler.bin
+12/09/2025 07:10:46 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000/random_states_0.pkl
+12/09/2025 07:10:46 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1000
+12/09/2025 07:10:46 - INFO - __main__ - Generating videos for validation...
+12/09/2025 07:10:46 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:07<00:00, 6.16it/s]
+12/09/2025 07:10:55 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 07:11:30 - INFO - __main__ - Step: 1010 Loss: 10.7099 LR: 0.000000
+12/09/2025 07:12:05 - INFO - __main__ - Step: 1020 Loss: 10.6899 LR: 0.000000
+12/09/2025 07:12:39 - INFO - __main__ - Step: 1030 Loss: 10.7427 LR: 0.000000
+12/09/2025 07:13:14 - INFO - __main__ - Step: 1040 Loss: 10.7364 LR: 0.000000
+12/09/2025 07:13:48 - INFO - __main__ - Step: 1050 Loss: 10.6964 LR: 0.000000
+12/09/2025 07:14:22 - INFO - __main__ - Step: 1060 Loss: 10.7274 LR: 0.000000
+12/09/2025 07:15:02 - INFO - __main__ - Step: 1070 Loss: 10.7467 LR: 0.000000
+12/09/2025 07:15:36 - INFO - __main__ - Step: 1080 Loss: 10.7183 LR: 0.000000
+12/09/2025 07:16:10 - INFO - __main__ - Step: 1090 Loss: 10.7970 LR: 0.000000
+12/09/2025 07:16:45 - INFO - __main__ - Step: 1100 Loss: 10.7245 LR: 0.000000
+12/09/2025 07:17:20 - INFO - __main__ - Step: 1110 Loss: 10.6785 LR: 0.000000
+12/09/2025 07:17:55 - INFO - __main__ - Step: 1120 Loss: 10.8533 LR: 0.000000
+12/09/2025 07:18:29 - INFO - __main__ - Step: 1130 Loss: 10.7524 LR: 0.000000
+12/09/2025 07:19:05 - INFO - __main__ - Step: 1140 Loss: 10.7386 LR: 0.000000
+12/09/2025 07:19:39 - INFO - __main__ - Step: 1150 Loss: 10.7314 LR: 0.000000
+12/09/2025 07:20:14 - INFO - __main__ - Step: 1160 Loss: 10.7112 LR: 0.000000
+12/09/2025 07:20:49 - INFO - __main__ - Step: 1170 Loss: 10.7142 LR: 0.000000
+12/09/2025 07:21:23 - INFO - __main__ - Step: 1180 Loss: 10.7207 LR: 0.000000
+12/09/2025 07:21:57 - INFO - __main__ - Step: 1190 Loss: 10.8444 LR: 0.000000
+12/09/2025 07:22:33 - INFO - __main__ - Step: 1200 Loss: 10.7064 LR: 0.000000
+12/09/2025 07:23:08 - INFO - __main__ - Step: 1210 Loss: 10.7500 LR: 0.000000
+12/09/2025 07:23:43 - INFO - __main__ - Step: 1220 Loss: 10.8374 LR: 0.000000
+12/09/2025 07:24:24 - INFO - __main__ - Step: 1230 Loss: 10.7911 LR: 0.000000
+12/09/2025 07:24:58 - INFO - __main__ - Step: 1240 Loss: 10.7271 LR: 0.000000
+12/09/2025 07:25:33 - INFO - __main__ - Step: 1250 Loss: 10.6836 LR: 0.000000
+12/09/2025 07:26:08 - INFO - __main__ - Step: 1260 Loss: 10.6580 LR: 0.000000
+12/09/2025 07:26:42 - INFO - __main__ - Step: 1270 Loss: 10.7364 LR: 0.000000
+12/09/2025 07:27:17 - INFO - __main__ - Step: 1280 Loss: 10.6940 LR: 0.000000
+12/09/2025 07:27:52 - INFO - __main__ - Step: 1290 Loss: 10.7192 LR: 0.000000
+12/09/2025 07:28:28 - INFO - __main__ - Step: 1300 Loss: 10.7082 LR: 0.000000
+12/09/2025 07:29:03 - INFO - __main__ - Step: 1310 Loss: 10.8125 LR: 0.000000
+12/09/2025 07:29:38 - INFO - __main__ - Step: 1320 Loss: 10.7506 LR: 0.000000
+12/09/2025 07:30:17 - INFO - __main__ - Step: 1330 Loss: 10.6634 LR: 0.000000
+12/09/2025 07:30:51 - INFO - __main__ - Step: 1340 Loss: 10.7885 LR: 0.000000
+12/09/2025 07:31:26 - INFO - __main__ - Step: 1350 Loss: 10.7047 LR: 0.000000
+12/09/2025 07:32:01 - INFO - __main__ - Step: 1360 Loss: 10.6698 LR: 0.000000
+12/09/2025 07:32:35 - INFO - __main__ - Step: 1370 Loss: 10.6433 LR: 0.000000
+12/09/2025 07:33:08 - INFO - __main__ - Step: 1380 Loss: 10.6759 LR: 0.000000
+12/09/2025 07:33:43 - INFO - __main__ - Step: 1390 Loss: 10.7195 LR: 0.000000
+12/09/2025 07:34:19 - INFO - __main__ - Step: 1400 Loss: 10.6907 LR: 0.000000
+12/09/2025 07:34:52 - INFO - __main__ - Step: 1410 Loss: 10.6880 LR: 0.000000
+12/09/2025 07:35:27 - INFO - __main__ - Step: 1420 Loss: 10.8639 LR: 0.000000
+12/09/2025 07:36:02 - INFO - __main__ - Step: 1430 Loss: 10.7094 LR: 0.000000
+12/09/2025 07:36:37 - INFO - __main__ - Step: 1440 Loss: 10.6651 LR: 0.000000
+12/09/2025 07:37:13 - INFO - __main__ - Step: 1450 Loss: 10.7197 LR: 0.000000
+12/09/2025 07:37:49 - INFO - __main__ - Step: 1460 Loss: 10.6842 LR: 0.000000
+12/09/2025 07:38:24 - INFO - __main__ - Step: 1470 Loss: 10.6803 LR: 0.000000
+12/09/2025 07:38:59 - INFO - __main__ - Step: 1480 Loss: 10.7731 LR: 0.000000
+12/09/2025 07:39:33 - INFO - __main__ - Step: 1490 Loss: 10.6577 LR: 0.000000
+12/09/2025 07:40:08 - INFO - __main__ - Step: 1500 Loss: 10.7049 LR: 0.000000
+12/09/2025 07:40:08 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500
+12/09/2025 07:40:14 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500/optimizer.bin
+12/09/2025 07:40:14 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500/scheduler.bin
+12/09/2025 07:40:14 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500/sampler.bin
+12/09/2025 07:40:14 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500/random_states_0.pkl
+12/09/2025 07:40:14 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-1500
+12/09/2025 07:40:14 - INFO - __main__ - Generating videos for validation...
+12/09/2025 07:40:14 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.72it/s]
+12/09/2025 07:40:20 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 07:40:54 - INFO - __main__ - Step: 1510 Loss: 10.6579 LR: 0.000000
+12/09/2025 07:41:31 - INFO - __main__ - Step: 1520 Loss: 10.6986 LR: 0.000000
+12/09/2025 07:42:08 - INFO - __main__ - Step: 1530 Loss: 10.6443 LR: 0.000000
+12/09/2025 07:42:46 - INFO - __main__ - Step: 1540 Loss: 10.8676 LR: 0.000000
+12/09/2025 07:43:21 - INFO - __main__ - Step: 1550 Loss: 10.7138 LR: 0.000000
+12/09/2025 07:43:58 - INFO - __main__ - Step: 1560 Loss: 10.8056 LR: 0.000000
+12/09/2025 07:44:33 - INFO - __main__ - Step: 1570 Loss: 10.8255 LR: 0.000000
+12/09/2025 07:45:08 - INFO - __main__ - Step: 1580 Loss: 10.6627 LR: 0.000000
+12/09/2025 07:45:43 - INFO - __main__ - Step: 1590 Loss: 10.6622 LR: 0.000000
+12/09/2025 07:46:18 - INFO - __main__ - Step: 1600 Loss: 10.7134 LR: 0.000000
+12/09/2025 07:46:52 - INFO - __main__ - Step: 1610 Loss: 10.7862 LR: 0.000000
+12/09/2025 07:47:26 - INFO - __main__ - Step: 1620 Loss: 10.8276 LR: 0.000000
+12/09/2025 07:48:01 - INFO - __main__ - Step: 1630 Loss: 10.7237 LR: 0.000000
+12/09/2025 07:48:37 - INFO - __main__ - Step: 1640 Loss: 10.7666 LR: 0.000000
+12/09/2025 07:49:12 - INFO - __main__ - Step: 1650 Loss: 10.8079 LR: 0.000000
+12/09/2025 07:49:48 - INFO - __main__ - Step: 1660 Loss: 10.7356 LR: 0.000000
+12/09/2025 07:50:23 - INFO - __main__ - Step: 1670 Loss: 10.6761 LR: 0.000000
+12/09/2025 07:50:59 - INFO - __main__ - Step: 1680 Loss: 10.6437 LR: 0.000000
+12/09/2025 07:51:33 - INFO - __main__ - Step: 1690 Loss: 10.6769 LR: 0.000000
+12/09/2025 07:52:09 - INFO - __main__ - Step: 1700 Loss: 10.7399 LR: 0.000000
+12/09/2025 07:52:44 - INFO - __main__ - Step: 1710 Loss: 10.6194 LR: 0.000000
+12/09/2025 07:53:20 - INFO - __main__ - Step: 1720 Loss: 10.6745 LR: 0.000000
+12/09/2025 07:53:55 - INFO - __main__ - Step: 1730 Loss: 10.8154 LR: 0.000000
+12/09/2025 07:54:30 - INFO - __main__ - Step: 1740 Loss: 10.6674 LR: 0.000000
+12/09/2025 07:55:06 - INFO - __main__ - Step: 1750 Loss: 10.6357 LR: 0.000000
+12/09/2025 07:55:43 - INFO - __main__ - Step: 1760 Loss: 10.7442 LR: 0.000000
+12/09/2025 07:56:18 - INFO - __main__ - Step: 1770 Loss: 10.8313 LR: 0.000000
+12/09/2025 07:56:54 - INFO - __main__ - Step: 1780 Loss: 10.8105 LR: 0.000000
+12/09/2025 07:57:29 - INFO - __main__ - Step: 1790 Loss: 10.8072 LR: 0.000000
+12/09/2025 07:58:04 - INFO - __main__ - Step: 1800 Loss: 10.6736 LR: 0.000000
+12/09/2025 07:58:39 - INFO - __main__ - Step: 1810 Loss: 10.6609 LR: 0.000000
+12/09/2025 07:59:16 - INFO - __main__ - Step: 1820 Loss: 10.7257 LR: 0.000000
+12/09/2025 07:59:51 - INFO - __main__ - Step: 1830 Loss: 10.6906 LR: 0.000000
+12/09/2025 08:00:26 - INFO - __main__ - Step: 1840 Loss: 10.6100 LR: 0.000000
+12/09/2025 08:01:02 - INFO - __main__ - Step: 1850 Loss: 10.7107 LR: 0.000000
+12/09/2025 08:01:37 - INFO - __main__ - Step: 1860 Loss: 10.7994 LR: 0.000000
+12/09/2025 08:02:11 - INFO - __main__ - Step: 1870 Loss: 10.6406 LR: 0.000000
+12/09/2025 08:02:46 - INFO - __main__ - Step: 1880 Loss: 10.6008 LR: 0.000000
+12/09/2025 08:03:22 - INFO - __main__ - Step: 1890 Loss: 10.6390 LR: 0.000000
+12/09/2025 08:03:58 - INFO - __main__ - Step: 1900 Loss: 10.8459 LR: 0.000000
+12/09/2025 08:04:32 - INFO - __main__ - Step: 1910 Loss: 10.7128 LR: 0.000000
+12/09/2025 08:05:07 - INFO - __main__ - Step: 1920 Loss: 10.7213 LR: 0.000000
+12/09/2025 08:05:44 - INFO - __main__ - Step: 1930 Loss: 10.6535 LR: 0.000000
+12/09/2025 08:06:18 - INFO - __main__ - Step: 1940 Loss: 10.6877 LR: 0.000000
+12/09/2025 08:06:54 - INFO - __main__ - Step: 1950 Loss: 10.6792 LR: 0.000000
+12/09/2025 08:07:30 - INFO - __main__ - Step: 1960 Loss: 10.7321 LR: 0.000000
+12/09/2025 08:08:06 - INFO - __main__ - Step: 1970 Loss: 10.7119 LR: 0.000000
+12/09/2025 08:08:41 - INFO - __main__ - Step: 1980 Loss: 10.6359 LR: 0.000000
+12/09/2025 08:09:16 - INFO - __main__ - Step: 1990 Loss: 10.7073 LR: 0.000000
+12/09/2025 08:09:51 - INFO - __main__ - Step: 2000 Loss: 10.7035 LR: 0.000000
+12/09/2025 08:09:51 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000
+12/09/2025 08:09:55 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000/optimizer.bin
+12/09/2025 08:09:55 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000/scheduler.bin
+12/09/2025 08:09:55 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000/sampler.bin
+12/09/2025 08:09:55 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000/random_states_0.pkl
+12/09/2025 08:09:55 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2000
+12/09/2025 08:09:55 - INFO - __main__ - Generating videos for validation...
+12/09/2025 08:09:55 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.77it/s]
+12/09/2025 08:10:02 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 08:10:36 - INFO - __main__ - Step: 2010 Loss: 10.8088 LR: 0.000000
+12/09/2025 08:11:11 - INFO - __main__ - Step: 2020 Loss: 10.6305 LR: 0.000000
+12/09/2025 08:11:47 - INFO - __main__ - Step: 2030 Loss: 10.8125 LR: 0.000000
+12/09/2025 08:12:22 - INFO - __main__ - Step: 2040 Loss: 10.8360 LR: 0.000000
+12/09/2025 08:12:58 - INFO - __main__ - Step: 2050 Loss: 10.7444 LR: 0.000000
+12/09/2025 08:13:33 - INFO - __main__ - Step: 2060 Loss: 10.8345 LR: 0.000000
+12/09/2025 08:14:09 - INFO - __main__ - Step: 2070 Loss: 10.7116 LR: 0.000000
+12/09/2025 08:14:44 - INFO - __main__ - Step: 2080 Loss: 10.7805 LR: 0.000000
+12/09/2025 08:15:20 - INFO - __main__ - Step: 2090 Loss: 10.7028 LR: 0.000000
+12/09/2025 08:15:55 - INFO - __main__ - Step: 2100 Loss: 10.6629 LR: 0.000000
+12/09/2025 08:16:29 - INFO - __main__ - Step: 2110 Loss: 10.6475 LR: 0.000000
+12/09/2025 08:17:06 - INFO - __main__ - Step: 2120 Loss: 10.7187 LR: 0.000000
+12/09/2025 08:17:41 - INFO - __main__ - Step: 2130 Loss: 10.6920 LR: 0.000000
+12/09/2025 08:18:15 - INFO - __main__ - Step: 2140 Loss: 10.6203 LR: 0.000000
+12/09/2025 08:18:51 - INFO - __main__ - Step: 2150 Loss: 10.8124 LR: 0.000000
+12/09/2025 08:19:27 - INFO - __main__ - Step: 2160 Loss: 10.6412 LR: 0.000000
+12/09/2025 08:20:02 - INFO - __main__ - Step: 2170 Loss: 10.6837 LR: 0.000000
+12/09/2025 08:20:37 - INFO - __main__ - Step: 2180 Loss: 10.7157 LR: 0.000000
+12/09/2025 08:21:12 - INFO - __main__ - Step: 2190 Loss: 10.6561 LR: 0.000000
+12/09/2025 08:21:48 - INFO - __main__ - Step: 2200 Loss: 10.6555 LR: 0.000000
+12/09/2025 08:22:24 - INFO - __main__ - Step: 2210 Loss: 10.6130 LR: 0.000000
+12/09/2025 08:23:00 - INFO - __main__ - Step: 2220 Loss: 10.6601 LR: 0.000000
+12/09/2025 08:23:37 - INFO - __main__ - Step: 2230 Loss: 10.7609 LR: 0.000000
+12/09/2025 08:24:13 - INFO - __main__ - Step: 2240 Loss: 10.6826 LR: 0.000000
+12/09/2025 08:24:50 - INFO - __main__ - Step: 2250 Loss: 10.6676 LR: 0.000000
+12/09/2025 08:25:26 - INFO - __main__ - Step: 2260 Loss: 10.6843 LR: 0.000000
+12/09/2025 08:26:01 - INFO - __main__ - Step: 2270 Loss: 10.6482 LR: 0.000000
+12/09/2025 08:26:37 - INFO - __main__ - Step: 2280 Loss: 10.7033 LR: 0.000000
+12/09/2025 08:27:13 - INFO - __main__ - Step: 2290 Loss: 10.6908 LR: 0.000000
+12/09/2025 08:27:47 - INFO - __main__ - Step: 2300 Loss: 10.7051 LR: 0.000000
+12/09/2025 08:28:24 - INFO - __main__ - Step: 2310 Loss: 10.6186 LR: 0.000000
+12/09/2025 08:28:59 - INFO - __main__ - Step: 2320 Loss: 10.6609 LR: 0.000000
+12/09/2025 08:29:35 - INFO - __main__ - Step: 2330 Loss: 10.6018 LR: 0.000000
+12/09/2025 08:30:11 - INFO - __main__ - Step: 2340 Loss: 10.6501 LR: 0.000000
+12/09/2025 08:30:50 - INFO - __main__ - Step: 2350 Loss: 10.7127 LR: 0.000000
+12/09/2025 08:31:26 - INFO - __main__ - Step: 2360 Loss: 10.5897 LR: 0.000000
+12/09/2025 08:32:02 - INFO - __main__ - Step: 2370 Loss: 10.6239 LR: 0.000000
+12/09/2025 08:32:39 - INFO - __main__ - Step: 2380 Loss: 10.8009 LR: 0.000000
+12/09/2025 08:33:17 - INFO - __main__ - Step: 2390 Loss: 10.6583 LR: 0.000000
+12/09/2025 08:33:52 - INFO - __main__ - Step: 2400 Loss: 10.7898 LR: 0.000000
+12/09/2025 08:34:29 - INFO - __main__ - Step: 2410 Loss: 10.7053 LR: 0.000000
+12/09/2025 08:35:04 - INFO - __main__ - Step: 2420 Loss: 10.7878 LR: 0.000000
+12/09/2025 08:35:40 - INFO - __main__ - Step: 2430 Loss: 10.7829 LR: 0.000000
+12/09/2025 08:36:16 - INFO - __main__ - Step: 2440 Loss: 10.5827 LR: 0.000000
+12/09/2025 08:36:52 - INFO - __main__ - Step: 2450 Loss: 10.6923 LR: 0.000000
+12/09/2025 08:37:27 - INFO - __main__ - Step: 2460 Loss: 10.7444 LR: 0.000000
+12/09/2025 08:38:03 - INFO - __main__ - Step: 2470 Loss: 10.8070 LR: 0.000000
+12/09/2025 08:38:37 - INFO - __main__ - Step: 2480 Loss: 10.6172 LR: 0.000000
+12/09/2025 08:39:13 - INFO - __main__ - Step: 2490 Loss: 10.7115 LR: 0.000000
+12/09/2025 08:39:49 - INFO - __main__ - Step: 2500 Loss: 10.6463 LR: 0.000000
+12/09/2025 08:39:49 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500
+12/09/2025 08:39:54 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500/optimizer.bin
+12/09/2025 08:39:54 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500/scheduler.bin
+12/09/2025 08:39:54 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500/sampler.bin
+12/09/2025 08:39:54 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500/random_states_0.pkl
+12/09/2025 08:39:54 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-2500
+12/09/2025 08:39:54 - INFO - __main__ - Generating videos for validation...
+12/09/2025 08:39:54 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.79it/s]
+12/09/2025 08:40:00 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 08:40:35 - INFO - __main__ - Step: 2510 Loss: 10.7087 LR: 0.000000
+12/09/2025 08:41:13 - INFO - __main__ - Step: 2520 Loss: 10.6353 LR: 0.000000
+12/09/2025 08:41:50 - INFO - __main__ - Step: 2530 Loss: 10.7710 LR: 0.000000
+12/09/2025 08:42:30 - INFO - __main__ - Step: 2540 Loss: 10.7668 LR: 0.000000
+12/09/2025 08:43:09 - INFO - __main__ - Step: 2550 Loss: 10.8079 LR: 0.000000
+12/09/2025 08:43:46 - INFO - __main__ - Step: 2560 Loss: 10.6643 LR: 0.000000
+12/09/2025 08:44:22 - INFO - __main__ - Step: 2570 Loss: 10.5756 LR: 0.000000
+12/09/2025 08:44:58 - INFO - __main__ - Step: 2580 Loss: 10.6465 LR: 0.000000
+12/09/2025 08:45:35 - INFO - __main__ - Step: 2590 Loss: 10.6285 LR: 0.000000
+12/09/2025 08:46:10 - INFO - __main__ - Step: 2600 Loss: 10.6634 LR: 0.000000
+12/09/2025 08:46:46 - INFO - __main__ - Step: 2610 Loss: 10.7108 LR: 0.000000
+12/09/2025 08:47:22 - INFO - __main__ - Step: 2620 Loss: 10.6576 LR: 0.000000
+12/09/2025 08:47:57 - INFO - __main__ - Step: 2630 Loss: 10.6358 LR: 0.000000
+12/09/2025 08:48:33 - INFO - __main__ - Step: 2640 Loss: 10.7581 LR: 0.000000
+12/09/2025 08:49:09 - INFO - __main__ - Step: 2650 Loss: 10.6845 LR: 0.000000
+12/09/2025 08:49:45 - INFO - __main__ - Step: 2660 Loss: 10.6936 LR: 0.000000
+12/09/2025 08:50:21 - INFO - __main__ - Step: 2670 Loss: 10.6546 LR: 0.000000
+12/09/2025 08:50:58 - INFO - __main__ - Step: 2680 Loss: 10.6183 LR: 0.000000
+12/09/2025 08:51:32 - INFO - __main__ - Step: 2690 Loss: 10.7849 LR: 0.000000
+12/09/2025 08:52:08 - INFO - __main__ - Step: 2700 Loss: 10.7991 LR: 0.000000
+12/09/2025 08:52:44 - INFO - __main__ - Step: 2710 Loss: 10.6829 LR: 0.000000
+12/09/2025 08:53:19 - INFO - __main__ - Step: 2720 Loss: 10.6621 LR: 0.000000
+12/09/2025 08:53:54 - INFO - __main__ - Step: 2730 Loss: 10.7058 LR: 0.000000
+12/09/2025 08:54:30 - INFO - __main__ - Step: 2740 Loss: 10.6618 LR: 0.000000
+12/09/2025 08:55:06 - INFO - __main__ - Step: 2750 Loss: 10.6734 LR: 0.000000
+12/09/2025 08:55:42 - INFO - __main__ - Step: 2760 Loss: 10.6153 LR: 0.000000
+12/09/2025 08:56:18 - INFO - __main__ - Step: 2770 Loss: 10.6321 LR: 0.000000
+12/09/2025 08:56:53 - INFO - __main__ - Step: 2780 Loss: 10.5745 LR: 0.000000
+12/09/2025 08:57:28 - INFO - __main__ - Step: 2790 Loss: 10.6596 LR: 0.000000
+12/09/2025 08:58:04 - INFO - __main__ - Step: 2800 Loss: 10.6588 LR: 0.000000
+12/09/2025 08:58:39 - INFO - __main__ - Step: 2810 Loss: 10.6601 LR: 0.000000
+12/09/2025 08:59:15 - INFO - __main__ - Step: 2820 Loss: 10.7636 LR: 0.000000
+12/09/2025 08:59:50 - INFO - __main__ - Step: 2830 Loss: 10.6506 LR: 0.000000
+12/09/2025 09:00:26 - INFO - __main__ - Step: 2840 Loss: 10.6805 LR: 0.000000
+12/09/2025 09:01:02 - INFO - __main__ - Step: 2850 Loss: 10.6614 LR: 0.000000
+12/09/2025 09:01:38 - INFO - __main__ - Step: 2860 Loss: 10.7792 LR: 0.000000
+12/09/2025 09:02:13 - INFO - __main__ - Step: 2870 Loss: 10.6542 LR: 0.000000
+12/09/2025 09:02:49 - INFO - __main__ - Step: 2880 Loss: 10.6708 LR: 0.000000
+12/09/2025 09:03:25 - INFO - __main__ - Step: 2890 Loss: 10.7061 LR: 0.000000
+12/09/2025 09:04:00 - INFO - __main__ - Step: 2900 Loss: 10.6127 LR: 0.000000
+12/09/2025 09:04:35 - INFO - __main__ - Step: 2910 Loss: 10.6518 LR: 0.000000
+12/09/2025 09:05:12 - INFO - __main__ - Step: 2920 Loss: 10.6581 LR: 0.000000
+12/09/2025 09:05:48 - INFO - __main__ - Step: 2930 Loss: 10.6383 LR: 0.000000
+12/09/2025 09:06:24 - INFO - __main__ - Step: 2940 Loss: 10.8066 LR: 0.000000
+12/09/2025 09:07:01 - INFO - __main__ - Step: 2950 Loss: 10.7880 LR: 0.000000
+12/09/2025 09:07:36 - INFO - __main__ - Step: 2960 Loss: 10.5884 LR: 0.000000
+12/09/2025 09:08:11 - INFO - __main__ - Step: 2970 Loss: 10.7773 LR: 0.000000
+12/09/2025 09:08:48 - INFO - __main__ - Step: 2980 Loss: 10.6863 LR: 0.000000
+12/09/2025 09:09:24 - INFO - __main__ - Step: 2990 Loss: 10.8002 LR: 0.000000
+12/09/2025 09:10:00 - INFO - __main__ - Step: 3000 Loss: 10.7093 LR: 0.000000
+12/09/2025 09:10:00 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000
+12/09/2025 09:10:04 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000/optimizer.bin
+12/09/2025 09:10:04 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000/scheduler.bin
+12/09/2025 09:10:04 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000/sampler.bin
+12/09/2025 09:10:04 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000/random_states_0.pkl
+12/09/2025 09:10:04 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3000
+12/09/2025 09:10:04 - INFO - __main__ - Generating videos for validation...
+12/09/2025 09:10:04 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.02it/s]
+12/09/2025 09:10:10 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 09:10:46 - INFO - __main__ - Step: 3010 Loss: 10.6042 LR: 0.000000
+12/09/2025 09:11:22 - INFO - __main__ - Step: 3020 Loss: 10.7013 LR: 0.000000
+12/09/2025 09:11:58 - INFO - __main__ - Step: 3030 Loss: 10.7090 LR: 0.000000
+12/09/2025 09:12:33 - INFO - __main__ - Step: 3040 Loss: 10.6076 LR: 0.000000
+12/09/2025 09:13:09 - INFO - __main__ - Step: 3050 Loss: 10.6399 LR: 0.000000
+12/09/2025 09:13:45 - INFO - __main__ - Step: 3060 Loss: 10.6489 LR: 0.000000
+12/09/2025 09:14:19 - INFO - __main__ - Step: 3070 Loss: 10.6214 LR: 0.000000
+12/09/2025 09:14:55 - INFO - __main__ - Step: 3080 Loss: 10.7581 LR: 0.000000
+12/09/2025 09:15:31 - INFO - __main__ - Step: 3090 Loss: 10.6609 LR: 0.000000
+12/09/2025 09:16:06 - INFO - __main__ - Step: 3100 Loss: 10.6468 LR: 0.000000
+12/09/2025 09:16:41 - INFO - __main__ - Step: 3110 Loss: 10.6854 LR: 0.000000
+12/09/2025 09:17:26 - INFO - __main__ - Step: 3120 Loss: 10.6509 LR: 0.000000
+12/09/2025 09:18:02 - INFO - __main__ - Step: 3130 Loss: 10.6653 LR: 0.000000
+12/09/2025 09:18:37 - INFO - __main__ - Step: 3140 Loss: 10.6436 LR: 0.000000
+12/09/2025 09:19:14 - INFO - __main__ - Step: 3150 Loss: 10.8747 LR: 0.000000
+12/09/2025 09:19:49 - INFO - __main__ - Step: 3160 Loss: 10.7215 LR: 0.000000
+12/09/2025 09:20:24 - INFO - __main__ - Step: 3170 Loss: 10.6346 LR: 0.000000
+12/09/2025 09:20:59 - INFO - __main__ - Step: 3180 Loss: 10.6490 LR: 0.000000
+12/09/2025 09:21:36 - INFO - __main__ - Step: 3190 Loss: 10.6399 LR: 0.000000
+12/09/2025 09:22:13 - INFO - __main__ - Step: 3200 Loss: 10.6877 LR: 0.000000
+12/09/2025 09:22:49 - INFO - __main__ - Step: 3210 Loss: 10.8075 LR: 0.000000
+12/09/2025 09:23:24 - INFO - __main__ - Step: 3220 Loss: 10.6954 LR: 0.000000
+12/09/2025 09:24:00 - INFO - __main__ - Step: 3230 Loss: 10.6205 LR: 0.000000
+12/09/2025 09:24:36 - INFO - __main__ - Step: 3240 Loss: 10.6941 LR: 0.000000
+12/09/2025 09:25:11 - INFO - __main__ - Step: 3250 Loss: 10.5495 LR: 0.000000
+12/09/2025 09:25:47 - INFO - __main__ - Step: 3260 Loss: 10.7130 LR: 0.000000
+12/09/2025 09:26:22 - INFO - __main__ - Step: 3270 Loss: 10.6809 LR: 0.000000
+12/09/2025 09:26:59 - INFO - __main__ - Step: 3280 Loss: 10.6429 LR: 0.000000
+12/09/2025 09:27:34 - INFO - __main__ - Step: 3290 Loss: 10.7921 LR: 0.000000
+12/09/2025 09:28:10 - INFO - __main__ - Step: 3300 Loss: 10.6165 LR: 0.000000
+12/09/2025 09:28:44 - INFO - __main__ - Step: 3310 Loss: 10.7733 LR: 0.000000
+12/09/2025 09:29:19 - INFO - __main__ - Step: 3320 Loss: 10.7044 LR: 0.000000
+12/09/2025 09:29:54 - INFO - __main__ - Step: 3330 Loss: 10.6457 LR: 0.000000
+12/09/2025 09:30:30 - INFO - __main__ - Step: 3340 Loss: 10.8392 LR: 0.000000
+12/09/2025 09:31:05 - INFO - __main__ - Step: 3350 Loss: 10.7154 LR: 0.000000
+12/09/2025 09:31:40 - INFO - __main__ - Step: 3360 Loss: 10.6910 LR: 0.000000
+12/09/2025 09:32:16 - INFO - __main__ - Step: 3370 Loss: 10.5989 LR: 0.000000
+12/09/2025 09:32:52 - INFO - __main__ - Step: 3380 Loss: 10.6108 LR: 0.000000
+12/09/2025 09:33:28 - INFO - __main__ - Step: 3390 Loss: 10.6229 LR: 0.000000
+12/09/2025 09:34:04 - INFO - __main__ - Step: 3400 Loss: 10.6160 LR: 0.000000
+12/09/2025 09:34:40 - INFO - __main__ - Step: 3410 Loss: 10.6124 LR: 0.000000
+12/09/2025 09:35:14 - INFO - __main__ - Step: 3420 Loss: 10.6181 LR: 0.000000
+12/09/2025 09:35:53 - INFO - __main__ - Step: 3430 Loss: 10.7191 LR: 0.000000
+12/09/2025 09:36:28 - INFO - __main__ - Step: 3440 Loss: 10.7020 LR: 0.000000
+12/09/2025 09:37:04 - INFO - __main__ - Step: 3450 Loss: 10.6007 LR: 0.000000
+12/09/2025 09:37:39 - INFO - __main__ - Step: 3460 Loss: 10.6844 LR: 0.000000
+12/09/2025 09:38:16 - INFO - __main__ - Step: 3470 Loss: 10.6536 LR: 0.000000
+12/09/2025 09:38:51 - INFO - __main__ - Step: 3480 Loss: 10.6183 LR: 0.000000
+12/09/2025 09:39:28 - INFO - __main__ - Step: 3490 Loss: 10.6050 LR: 0.000000
+12/09/2025 09:40:03 - INFO - __main__ - Step: 3500 Loss: 10.6128 LR: 0.000000
+12/09/2025 09:40:03 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500
+12/09/2025 09:40:09 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500/optimizer.bin
+12/09/2025 09:40:09 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500/scheduler.bin
+12/09/2025 09:40:09 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500/sampler.bin
+12/09/2025 09:40:09 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500/random_states_0.pkl
+12/09/2025 09:40:09 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-3500
+12/09/2025 09:40:09 - INFO - __main__ - Generating videos for validation...
+12/09/2025 09:40:09 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.72it/s]
+12/09/2025 09:40:16 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 09:40:51 - INFO - __main__ - Step: 3510 Loss: 10.8156 LR: 0.000000
+12/09/2025 09:41:27 - INFO - __main__ - Step: 3520 Loss: 10.6784 LR: 0.000000
+12/09/2025 09:42:03 - INFO - __main__ - Step: 3530 Loss: 10.7747 LR: 0.000000
+12/09/2025 09:42:40 - INFO - __main__ - Step: 3540 Loss: 10.6523 LR: 0.000000
+12/09/2025 09:43:16 - INFO - __main__ - Step: 3550 Loss: 10.6710 LR: 0.000000
+12/09/2025 09:43:51 - INFO - __main__ - Step: 3560 Loss: 10.6228 LR: 0.000000
+12/09/2025 09:44:25 - INFO - __main__ - Step: 3570 Loss: 10.6325 LR: 0.000000
+12/09/2025 09:45:01 - INFO - __main__ - Step: 3580 Loss: 10.6580 LR: 0.000000
+12/09/2025 09:45:35 - INFO - __main__ - Step: 3590 Loss: 10.6412 LR: 0.000000
+12/09/2025 09:46:10 - INFO - __main__ - Step: 3600 Loss: 10.6338 LR: 0.000000
+12/09/2025 09:46:46 - INFO - __main__ - Step: 3610 Loss: 10.6233 LR: 0.000000
+12/09/2025 09:47:21 - INFO - __main__ - Step: 3620 Loss: 10.7960 LR: 0.000000
+12/09/2025 09:47:57 - INFO - __main__ - Step: 3630 Loss: 10.6824 LR: 0.000000
+12/09/2025 09:48:33 - INFO - __main__ - Step: 3640 Loss: 10.5719 LR: 0.000000
+12/09/2025 09:49:08 - INFO - __main__ - Step: 3650 Loss: 10.7161 LR: 0.000000
+12/09/2025 09:49:43 - INFO - __main__ - Step: 3660 Loss: 10.5788 LR: 0.000000
+12/09/2025 09:50:18 - INFO - __main__ - Step: 3670 Loss: 10.5937 LR: 0.000000
+12/09/2025 09:50:57 - INFO - __main__ - Step: 3680 Loss: 10.6459 LR: 0.000000
+12/09/2025 09:51:36 - INFO - __main__ - Step: 3690 Loss: 10.6017 LR: 0.000000
+12/09/2025 09:52:12 - INFO - __main__ - Step: 3700 Loss: 10.6180 LR: 0.000000
+12/09/2025 09:52:47 - INFO - __main__ - Step: 3710 Loss: 10.7492 LR: 0.000000
+12/09/2025 09:53:25 - INFO - __main__ - Step: 3720 Loss: 10.6616 LR: 0.000000
+12/09/2025 09:54:02 - INFO - __main__ - Step: 3730 Loss: 10.5907 LR: 0.000000
+12/09/2025 09:54:37 - INFO - __main__ - Step: 3740 Loss: 10.6316 LR: 0.000000
+12/09/2025 09:55:15 - INFO - __main__ - Step: 3750 Loss: 10.5829 LR: 0.000000
+12/09/2025 09:55:50 - INFO - __main__ - Step: 3760 Loss: 10.8051 LR: 0.000000
+12/09/2025 09:56:26 - INFO - __main__ - Step: 3770 Loss: 10.6096 LR: 0.000000
+12/09/2025 09:57:02 - INFO - __main__ - Step: 3780 Loss: 10.6737 LR: 0.000000
+12/09/2025 09:57:38 - INFO - __main__ - Step: 3790 Loss: 10.6407 LR: 0.000000
+12/09/2025 09:58:14 - INFO - __main__ - Step: 3800 Loss: 10.6684 LR: 0.000000
+12/09/2025 09:58:51 - INFO - __main__ - Step: 3810 Loss: 10.6076 LR: 0.000000
+12/09/2025 09:59:26 - INFO - __main__ - Step: 3820 Loss: 10.5907 LR: 0.000000
+12/09/2025 10:00:00 - INFO - __main__ - Step: 3830 Loss: 10.7516 LR: 0.000000
+12/09/2025 10:00:38 - INFO - __main__ - Step: 3840 Loss: 10.6090 LR: 0.000000
+12/09/2025 10:01:13 - INFO - __main__ - Step: 3850 Loss: 10.6274 LR: 0.000000
+12/09/2025 10:01:48 - INFO - __main__ - Step: 3860 Loss: 10.7927 LR: 0.000000
+12/09/2025 10:02:23 - INFO - __main__ - Step: 3870 Loss: 10.5754 LR: 0.000000
+12/09/2025 10:02:58 - INFO - __main__ - Step: 3880 Loss: 10.6764 LR: 0.000000
+12/09/2025 10:03:34 - INFO - __main__ - Step: 3890 Loss: 10.5462 LR: 0.000000
+12/09/2025 10:04:10 - INFO - __main__ - Step: 3900 Loss: 10.5982 LR: 0.000000
+12/09/2025 10:04:48 - INFO - __main__ - Step: 3910 Loss: 10.6223 LR: 0.000000
+12/09/2025 10:05:24 - INFO - __main__ - Step: 3920 Loss: 10.7905 LR: 0.000000
+12/09/2025 10:05:59 - INFO - __main__ - Step: 3930 Loss: 10.6241 LR: 0.000000
+12/09/2025 10:06:35 - INFO - __main__ - Step: 3940 Loss: 10.7700 LR: 0.000000
+12/09/2025 10:07:09 - INFO - __main__ - Step: 3950 Loss: 10.6646 LR: 0.000000
+12/09/2025 10:07:44 - INFO - __main__ - Step: 3960 Loss: 10.7494 LR: 0.000000
+12/09/2025 10:08:19 - INFO - __main__ - Step: 3970 Loss: 10.6112 LR: 0.000000
+12/09/2025 10:08:56 - INFO - __main__ - Step: 3980 Loss: 10.7726 LR: 0.000000
+12/09/2025 10:09:31 - INFO - __main__ - Step: 3990 Loss: 10.6219 LR: 0.000000
+12/09/2025 10:10:07 - INFO - __main__ - Step: 4000 Loss: 10.5863 LR: 0.000000
+12/09/2025 10:10:07 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+12/09/2025 10:10:12 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000/optimizer.bin
+12/09/2025 10:10:12 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000/scheduler.bin
+12/09/2025 10:10:12 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000/sampler.bin
+12/09/2025 10:10:12 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000/random_states_0.pkl
+12/09/2025 10:10:12 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+12/09/2025 10:10:12 - INFO - __main__ - Generating videos for validation...
+12/09/2025 10:10:12 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.39it/s]
+12/09/2025 10:10:19 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio
+12/09/2025 10:10:54 - INFO - __main__ - Step: 4010 Loss: 10.6506 LR: 0.000000
+12/09/2025 10:11:31 - INFO - __main__ - Step: 4020 Loss: 10.6645 LR: 0.000000
+12/09/2025 10:12:10 - INFO - __main__ - Step: 4030 Loss: 10.6335 LR: 0.000000
+12/09/2025 10:12:50 - INFO - __main__ - Step: 4040 Loss: 10.6103 LR: 0.000000
+12/09/2025 10:13:29 - INFO - __main__ - Step: 4050 Loss: 10.6712 LR: 0.000000
+12/09/2025 10:14:03 - INFO - __main__ - Step: 4060 Loss: 10.7147 LR: 0.000000
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1435, in
+ frames_tensor = torch.stack(frames, dim=0)
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1283, in main
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1435, in
+[rank0]: frames_tensor = torch.stack(frames, dim=0)
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1283, in main
+[rank0]:
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/requirements.txt b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..882ab777b3727eb58f6fc36980670267858e57b6
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/files/wandb-metadata.json
@@ -0,0 +1,154 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T06:08:56.508700Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--freeze_wan_backbone",
+ "--wan_backbone_lr_ratio",
+ "0.0",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12017912905728"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "bs5hxou77f8nhtr968pz9al3r4zobq2y"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-core.log b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..9d6d9f3bbaebbc8a53a9a1e5442dc4f9afb7e85a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T06:08:56.58235045Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmptuqk2li8/port-461528.txt","pid":461528,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T06:08:56.582820205Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":461528}
+{"time":"2025-12-09T06:08:56.582800746Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-461528-461798-2163184360/socket","Net":"unix"}}
+{"time":"2025-12-09T06:08:56.764351024Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T06:08:56.770261851Z","level":"INFO","msg":"handleInformInit: received","streamId":"ctbp97lz","id":"1(@)"}
+{"time":"2025-12-09T06:08:56.937028325Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ctbp97lz","id":"1(@)"}
+{"time":"2025-12-09T10:14:12.178908066Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-internal.log b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..694f446ebdd8cafb2d0747848cd9c87ab021f02e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T06:08:56.770384765Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T06:08:56.936708556Z","level":"INFO","msg":"stream: created new stream","id":"ctbp97lz"}
+{"time":"2025-12-09T06:08:56.936930986Z","level":"INFO","msg":"handler: started","stream_id":"ctbp97lz"}
+{"time":"2025-12-09T06:08:56.937020175Z","level":"INFO","msg":"stream: started","id":"ctbp97lz"}
+{"time":"2025-12-09T06:08:56.937037269Z","level":"INFO","msg":"writer: started","stream_id":"ctbp97lz"}
+{"time":"2025-12-09T06:08:56.937040368Z","level":"INFO","msg":"sender: started","stream_id":"ctbp97lz"}
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug.log b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..1f240f4310a8b47ce8509c33dcfe86b049f871b4
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_setup.py:_flush():80] Configure stats pid to 461528
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug.log
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_060856-ctbp97lz/logs/debug-internal.log
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_init.py:init():841] calling init triggers
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 06:08:56,511 INFO MainThread:461528 [wandb_init.py:init():889] starting backend
+2025-12-09 06:08:56,764 INFO MainThread:461528 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 06:08:56,768 INFO MainThread:461528 [wandb_init.py:init():900] backend started and connected
+2025-12-09 06:08:56,769 INFO MainThread:461528 [wandb_init.py:init():970] updated telemetry
+2025-12-09 06:08:56,774 INFO MainThread:461528 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 06:08:57,287 INFO MainThread:461528 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 06:08:57,409 INFO MainThread:461528 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 06:08:57,409 INFO MainThread:461528 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 06:08:57,410 INFO MainThread:461528 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 06:08:57,410 INFO MainThread:461528 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 06:08:57,412 INFO MainThread:461528 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 06:08:57,413 INFO MainThread:461528 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': True, 'wan_backbone_lr_ratio': 0.0}
diff --git a/Meissonic/wandb/run-20251209_060856-ctbp97lz/run-ctbp97lz.wandb b/Meissonic/wandb/run-20251209_060856-ctbp97lz/run-ctbp97lz.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..e86288104af59198dac2fa71e0d0a105a8f0cc90
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_060856-ctbp97lz/run-ctbp97lz.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1decb264c3facb98478fd342c9a75ef40f7059408c46241f8c0612470d79f01
+size 3866624
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/files/output.log b/Meissonic/wandb/run-20251209_101545-18h777jw/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..a3a63d61674542da2271205d5d01afe33a22cbfc
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/files/output.log
@@ -0,0 +1,159 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 71.91it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 10:15:49 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6916.40it/s]
+12/09/2025 10:15:50 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 10:15:50 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 10:15:50 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 10:15:51 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 10:16:07 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 10:16:07 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 10:16:25 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 10:16:28 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 10:16:28 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 10:16:28 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 10:16:28 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 10:16:36 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 10:16:36 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 10:16:36 - INFO - __main__ - Dataloader configuration:
+12/09/2025 10:16:36 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 10:16:36 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 10:16:36 - INFO - __main__ - - persistent_workers: True
+12/09/2025 10:16:36 - INFO - __main__ - - pin_memory: True
+12/09/2025 10:16:36 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 10:16:56 - INFO - __main__ - ***** Running training *****
+12/09/2025 10:16:56 - INFO - __main__ - Num training steps = 10000
+12/09/2025 10:16:56 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 10:16:56 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 10:16:56 - INFO - __main__ - Gradient Accumulation steps = 4
+12/09/2025 10:17:46 - INFO - __main__ - Step: 10 Loss: 11.0743 LR: 0.000300
+12/09/2025 10:18:24 - INFO - __main__ - Step: 20 Loss: 11.0744 LR: 0.000300
+12/09/2025 10:19:00 - INFO - __main__ - Step: 30 Loss: 11.0724 LR: 0.000300
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1297, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 358, in forward
+ q=rope_apply(q, grid_sizes, freqs),
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 241, in rope_apply
+ x_i = torch.view_as_complex(x[i, :seq_len].to(torch.float64).reshape(
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1297, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 358, in forward
+[rank0]: q=rope_apply(q, grid_sizes, freqs),
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 241, in rope_apply
+[rank0]: x_i = torch.view_as_complex(x[i, :seq_len].to(torch.float64).reshape(
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/files/requirements.txt b/Meissonic/wandb/run-20251209_101545-18h777jw/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_101545-18h777jw/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..436a09ffa610442cb17cace43f23817fd7c5e738
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/files/wandb-metadata.json
@@ -0,0 +1,151 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T10:15:45.646078Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12040636145664"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "njobkvrswapd2mjdxwfj2z2e0j34qdah"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-core.log b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..aacddadd55054c67fecc60ad574c2bfc5b8d6196
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T10:15:45.717897393Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpq5k3n3kj/port-1710831.txt","pid":1710831,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T10:15:45.718349495Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1710831}
+{"time":"2025-12-09T10:15:45.718359886Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1710831-1711023-3076955892/socket","Net":"unix"}}
+{"time":"2025-12-09T10:15:45.900357571Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T10:15:45.906355067Z","level":"INFO","msg":"handleInformInit: received","streamId":"18h777jw","id":"1(@)"}
+{"time":"2025-12-09T10:15:46.074035273Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"18h777jw","id":"1(@)"}
+{"time":"2025-12-09T10:19:13.502015816Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-internal.log b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..db621149eeaa4432cfffbedb866c25834feec27c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T10:15:45.906609658Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T10:15:46.07379782Z","level":"INFO","msg":"stream: created new stream","id":"18h777jw"}
+{"time":"2025-12-09T10:15:46.073887373Z","level":"INFO","msg":"handler: started","stream_id":"18h777jw"}
+{"time":"2025-12-09T10:15:46.074025424Z","level":"INFO","msg":"stream: started","id":"18h777jw"}
+{"time":"2025-12-09T10:15:46.074035157Z","level":"INFO","msg":"writer: started","stream_id":"18h777jw"}
+{"time":"2025-12-09T10:15:46.074044896Z","level":"INFO","msg":"sender: started","stream_id":"18h777jw"}
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug.log b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..772f0769535dec5a54ec9b63ca1eb6cbe4d96da7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_setup.py:_flush():80] Configure stats pid to 1710831
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug.log
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_101545-18h777jw/logs/debug-internal.log
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_init.py:init():841] calling init triggers
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 10:15:45,648 INFO MainThread:1710831 [wandb_init.py:init():889] starting backend
+2025-12-09 10:15:45,900 INFO MainThread:1710831 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 10:15:45,904 INFO MainThread:1710831 [wandb_init.py:init():900] backend started and connected
+2025-12-09 10:15:45,906 INFO MainThread:1710831 [wandb_init.py:init():970] updated telemetry
+2025-12-09 10:15:45,910 INFO MainThread:1710831 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 10:15:46,338 INFO MainThread:1710831 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 10:15:46,464 INFO MainThread:1710831 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 10:15:46,464 INFO MainThread:1710831 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 10:15:46,464 INFO MainThread:1710831 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 10:15:46,464 INFO MainThread:1710831 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 10:15:46,467 INFO MainThread:1710831 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 10:15:46,468 INFO MainThread:1710831 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_101545-18h777jw/run-18h777jw.wandb b/Meissonic/wandb/run-20251209_101545-18h777jw/run-18h777jw.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..186c17f506ee84f0803218a8aa7165f1cb0073c5
Binary files /dev/null and b/Meissonic/wandb/run-20251209_101545-18h777jw/run-18h777jw.wandb differ
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/config.yaml b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..26aa17fc964b5c177c7bddec762f1468d1f4a278
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/config.yaml
@@ -0,0 +1,294 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 0oe17z8ryn1a3t71a4cnr7g1rc6j1axk:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --pretrained_model_name_or_path
+ - /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12040636317696"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T10:19:40.689127Z"
+ writerId: 0oe17z8ryn1a3t71a4cnr7g1rc6j1axk
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: null
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue
+pretrained_model_name_or_path:
+ value: /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: null
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/output.log b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..31e265b1898d77fd80d12d03cb6197f1c16a43d7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/output.log
@@ -0,0 +1,51 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 69.33it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 10:19:44 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6137.15it/s]
+12/09/2025 10:19:45 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 10:19:45 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 10:20:13 - INFO - __main__ - Parameter counts: backbone=2,191,878,208, other=2,112,033, total=2,193,990,241
+12/09/2025 10:20:13 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 10:20:13 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 10:20:13 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 10:20:20 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 10:20:20 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 10:20:20 - INFO - __main__ - Dataloader configuration:
+12/09/2025 10:20:20 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 10:20:20 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 10:20:20 - INFO - __main__ - - persistent_workers: True
+12/09/2025 10:20:20 - INFO - __main__ - - pin_memory: True
+12/09/2025 10:20:20 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 10:20:41 - INFO - __main__ - ***** Running training *****
+12/09/2025 10:20:41 - INFO - __main__ - Num training steps = 10000
+12/09/2025 10:20:41 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 10:20:41 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 10:20:41 - INFO - __main__ - Gradient Accumulation steps = 4
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1333, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.71 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.52 GiB is free. Process 1765386 has 414.00 MiB memory in use. Process 1765384 has 414.00 MiB memory in use. Process 1765387 has 414.00 MiB memory in use. Process 1765390 has 414.00 MiB memory in use. Process 1765389 has 414.00 MiB memory in use. Process 1765388 has 414.00 MiB memory in use. Process 1765385 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 35.09 GiB memory in use. Of the allocated memory 28.22 GiB is allocated by PyTorch, and 5.77 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1333, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.71 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.52 GiB is free. Process 1765386 has 414.00 MiB memory in use. Process 1765384 has 414.00 MiB memory in use. Process 1765387 has 414.00 MiB memory in use. Process 1765390 has 414.00 MiB memory in use. Process 1765389 has 414.00 MiB memory in use. Process 1765388 has 414.00 MiB memory in use. Process 1765385 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 35.09 GiB memory in use. Of the allocated memory 28.22 GiB is allocated by PyTorch, and 5.77 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/requirements.txt b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..2edfad17a102c08ca6d1383807c4b36e62c7e600
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-metadata.json
@@ -0,0 +1,149 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T10:19:40.689127Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12040636317696"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "0oe17z8ryn1a3t71a4cnr7g1rc6j1axk"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-summary.json b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..43fa43b2931fdff41b1f87a0ba7176f1f1bdf2eb
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":113},"_runtime":113}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-core.log b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..5fdefcaa4d324afc11a0a0b55a85e389252babc8
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T10:19:40.761887998Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpsrhk7a3_/port-1765383.txt","pid":1765383,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T10:19:40.762361725Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1765383}
+{"time":"2025-12-09T10:19:40.762355039Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1765383-1765647-3454298689/socket","Net":"unix"}}
+{"time":"2025-12-09T10:19:40.946446197Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T10:19:40.952049688Z","level":"INFO","msg":"handleInformInit: received","streamId":"6eo06e2t","id":"1(@)"}
+{"time":"2025-12-09T10:19:41.121266057Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"6eo06e2t","id":"1(@)"}
+{"time":"2025-12-09T10:21:34.921702441Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T10:21:34.921773597Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T10:21:34.92176219Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T10:21:34.921874142Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1765383-1765647-3454298689/socket","Net":"unix"}}
+{"time":"2025-12-09T10:21:34.922140875Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T10:21:35.28891892Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T10:21:35.28895153Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T10:21:35.288969226Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-internal.log b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..06bafdde6f66949c297c1927382867d292922437
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T10:19:40.952144615Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T10:19:41.121052434Z","level":"INFO","msg":"stream: created new stream","id":"6eo06e2t"}
+{"time":"2025-12-09T10:19:41.121134879Z","level":"INFO","msg":"handler: started","stream_id":"6eo06e2t"}
+{"time":"2025-12-09T10:19:41.121257683Z","level":"INFO","msg":"stream: started","id":"6eo06e2t"}
+{"time":"2025-12-09T10:19:41.121282803Z","level":"INFO","msg":"writer: started","stream_id":"6eo06e2t"}
+{"time":"2025-12-09T10:19:41.121283706Z","level":"INFO","msg":"sender: started","stream_id":"6eo06e2t"}
+{"time":"2025-12-09T10:21:34.921769871Z","level":"INFO","msg":"stream: closing","id":"6eo06e2t"}
+{"time":"2025-12-09T10:21:35.156164304Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T10:21:35.285709569Z","level":"INFO","msg":"handler: closed","stream_id":"6eo06e2t"}
+{"time":"2025-12-09T10:21:35.28582027Z","level":"INFO","msg":"sender: closed","stream_id":"6eo06e2t"}
+{"time":"2025-12-09T10:21:35.285838151Z","level":"INFO","msg":"stream: closed","id":"6eo06e2t"}
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug.log b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..f6f53434a61a60ca6f77b5d78b834a107a0bbda5
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 10:19:40,691 INFO MainThread:1765383 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 10:19:40,691 INFO MainThread:1765383 [wandb_setup.py:_flush():80] Configure stats pid to 1765383
+2025-12-09 10:19:40,691 INFO MainThread:1765383 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 10:19:40,691 INFO MainThread:1765383 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 10:19:40,691 INFO MainThread:1765383 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 10:19:40,692 INFO MainThread:1765383 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug.log
+2025-12-09 10:19:40,692 INFO MainThread:1765383 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_101940-6eo06e2t/logs/debug-internal.log
+2025-12-09 10:19:40,692 INFO MainThread:1765383 [wandb_init.py:init():841] calling init triggers
+2025-12-09 10:19:40,692 INFO MainThread:1765383 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 10:19:40,692 INFO MainThread:1765383 [wandb_init.py:init():889] starting backend
+2025-12-09 10:19:40,946 INFO MainThread:1765383 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 10:19:40,950 INFO MainThread:1765383 [wandb_init.py:init():900] backend started and connected
+2025-12-09 10:19:40,951 INFO MainThread:1765383 [wandb_init.py:init():970] updated telemetry
+2025-12-09 10:19:40,956 INFO MainThread:1765383 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 10:19:41,378 INFO MainThread:1765383 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 10:19:41,503 INFO MainThread:1765383 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 10:19:41,503 INFO MainThread:1765383 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 10:19:41,503 INFO MainThread:1765383 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 10:19:41,503 INFO MainThread:1765383 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 10:19:41,506 INFO MainThread:1765383 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 10:19:41,507 INFO MainThread:1765383 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': False, 'features_dir': None}
+2025-12-09 10:21:34,921 INFO wandb-AsyncioManager-main:1765383 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 10:21:34,922 INFO wandb-AsyncioManager-main:1765383 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_101940-6eo06e2t/run-6eo06e2t.wandb b/Meissonic/wandb/run-20251209_101940-6eo06e2t/run-6eo06e2t.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..3cfe92bd40c2585a42e441f8c31e8ce2d293c1e2
Binary files /dev/null and b/Meissonic/wandb/run-20251209_101940-6eo06e2t/run-6eo06e2t.wandb differ
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/output.log b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..9da8de2d4a396986ebb07ff4856956f84b4144a8
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/output.log
@@ -0,0 +1,25 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 71.63it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 10:22:38 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 7013.89it/s]
+12/09/2025 10:22:40 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 10:22:40 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 10:23:07 - INFO - __main__ - Parameter counts: backbone=2,191,878,208, other=2,112,033, total=2,193,990,241
+12/09/2025 10:23:07 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 10:23:07 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 10:23:07 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 10:23:14 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 10:23:14 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 10:23:14 - INFO - __main__ - Dataloader configuration:
+12/09/2025 10:23:14 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 10:23:14 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 10:23:14 - INFO - __main__ - - persistent_workers: True
+12/09/2025 10:23:14 - INFO - __main__ - - pin_memory: True
+12/09/2025 10:23:14 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 10:23:35 - INFO - __main__ - ***** Running training *****
+12/09/2025 10:23:35 - INFO - __main__ - Num training steps = 10000
+12/09/2025 10:23:35 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 10:23:35 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 10:23:35 - INFO - __main__ - Gradient Accumulation steps = 8
+12/09/2025 10:24:53 - INFO - __main__ - Step: 10 Loss: 11.0763 LR: 0.000300
+12/09/2025 10:26:06 - INFO - __main__ - Step: 20 Loss: 11.0723 LR: 0.000300
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/requirements.txt b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..f052f5f95fcd556d74968520f21cf5d88fb7b3d0
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/files/wandb-metadata.json
@@ -0,0 +1,149 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T10:22:35.233134Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "8",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12040636452864"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "yzvcufsb52wtryxz5lwk657v5el3944m"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-core.log b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..f95d409c2b51c30023dd2e7355572fbcf6a02336
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T10:22:35.303223801Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpx7g2yu4c/port-1775616.txt","pid":1775616,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T10:22:35.303750614Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1775616}
+{"time":"2025-12-09T10:22:35.303730014Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1775616-1775889-726218309/socket","Net":"unix"}}
+{"time":"2025-12-09T10:22:35.487936562Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T10:22:35.494099777Z","level":"INFO","msg":"handleInformInit: received","streamId":"95y0vlh3","id":"1(@)"}
+{"time":"2025-12-09T10:22:35.66364009Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"95y0vlh3","id":"1(@)"}
+{"time":"2025-12-09T10:26:32.909805593Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-internal.log b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..59ed6746f079f29f00dd0347a6a90d640cf5e059
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T10:22:35.494245676Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T10:22:35.663435247Z","level":"INFO","msg":"stream: created new stream","id":"95y0vlh3"}
+{"time":"2025-12-09T10:22:35.663518064Z","level":"INFO","msg":"handler: started","stream_id":"95y0vlh3"}
+{"time":"2025-12-09T10:22:35.663633531Z","level":"INFO","msg":"stream: started","id":"95y0vlh3"}
+{"time":"2025-12-09T10:22:35.663654716Z","level":"INFO","msg":"writer: started","stream_id":"95y0vlh3"}
+{"time":"2025-12-09T10:22:35.663655985Z","level":"INFO","msg":"sender: started","stream_id":"95y0vlh3"}
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug.log b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..0ff20246283d0b92a338f23a01d862e89369c6fe
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 10:22:35,235 INFO MainThread:1775616 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 10:22:35,235 INFO MainThread:1775616 [wandb_setup.py:_flush():80] Configure stats pid to 1775616
+2025-12-09 10:22:35,235 INFO MainThread:1775616 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 10:22:35,235 INFO MainThread:1775616 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 10:22:35,235 INFO MainThread:1775616 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 10:22:35,236 INFO MainThread:1775616 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug.log
+2025-12-09 10:22:35,236 INFO MainThread:1775616 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_102235-95y0vlh3/logs/debug-internal.log
+2025-12-09 10:22:35,236 INFO MainThread:1775616 [wandb_init.py:init():841] calling init triggers
+2025-12-09 10:22:35,236 INFO MainThread:1775616 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 10:22:35,236 INFO MainThread:1775616 [wandb_init.py:init():889] starting backend
+2025-12-09 10:22:35,488 INFO MainThread:1775616 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 10:22:35,492 INFO MainThread:1775616 [wandb_init.py:init():900] backend started and connected
+2025-12-09 10:22:35,493 INFO MainThread:1775616 [wandb_init.py:init():970] updated telemetry
+2025-12-09 10:22:35,498 INFO MainThread:1775616 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 10:22:35,983 INFO MainThread:1775616 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 10:22:36,107 INFO MainThread:1775616 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 10:22:36,107 INFO MainThread:1775616 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 10:22:36,107 INFO MainThread:1775616 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 10:22:36,107 INFO MainThread:1775616 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 10:22:36,109 INFO MainThread:1775616 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 10:22:36,111 INFO MainThread:1775616 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_102235-95y0vlh3/run-95y0vlh3.wandb b/Meissonic/wandb/run-20251209_102235-95y0vlh3/run-95y0vlh3.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c1f02f0abb2ea15448b6ab628ab3b8d1cf68963b
Binary files /dev/null and b/Meissonic/wandb/run-20251209_102235-95y0vlh3/run-95y0vlh3.wandb differ
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_2a764e89458c3c8d15fb.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_2a764e89458c3c8d15fb.png
new file mode 100644
index 0000000000000000000000000000000000000000..67fd9ce0e135bc0a8d01b659eee676a6b43c8e9e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_2a764e89458c3c8d15fb.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a764e89458c3c8d15fb1b993214628df564089d2d2d0b6e6f304b5042a5b4c2
+size 153669
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_80cf7f467b6a4ea9a5d4.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_80cf7f467b6a4ea9a5d4.png
new file mode 100644
index 0000000000000000000000000000000000000000..c9280a89b8e3b30354f265404ad27523c27e05e9
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1000_80cf7f467b6a4ea9a5d4.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80cf7f467b6a4ea9a5d477c4008984777d1b6b4422f00b945cdb847edeadabaa
+size 143871
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_c5788a4747c03f8f9f49.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_c5788a4747c03f8f9f49.png
new file mode 100644
index 0000000000000000000000000000000000000000..deed70824d03d140b7743b739ff085d9ed8a85c6
Binary files /dev/null and b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_c5788a4747c03f8f9f49.png differ
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_fb32391d5c492e093a1a.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_fb32391d5c492e093a1a.png
new file mode 100644
index 0000000000000000000000000000000000000000..aa9f132b4e3203ba9739eb46384a8de251dd3b29
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_1500_fb32391d5c492e093a1a.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb32391d5c492e093a1a5181dfed7a57173d650f41a8937045043a6654a4584d
+size 144814
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_9a388a1a15b60d9f4438.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_9a388a1a15b60d9f4438.png
new file mode 100644
index 0000000000000000000000000000000000000000..996ffc4309be4eb10301a052e03e5dd8f070cf42
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_9a388a1a15b60d9f4438.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a388a1a15b60d9f44386c6a939bac94001713fd8b9c2583f78b4338bab330ea
+size 120118
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_c2c619bff47ae122a524.png b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_c2c619bff47ae122a524.png
new file mode 100644
index 0000000000000000000000000000000000000000..6491dea77af7d8a9416908347e2761164abe4181
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/media/images/generated_videos_first_frame_500_c2c619bff47ae122a524.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2c619bff47ae122a5247b1528f8a40c85a112141a65dadbc62886b3a9abcb83
+size 127496
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/output.log b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..8b65910600fefd2d8cc88466ac214a4411944776
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/output.log
@@ -0,0 +1,234 @@
+Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 70.39it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 10:26:54 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 12531.00it/s]
+12/09/2025 10:27:00 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 10:27:00 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 10:27:00 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 10:27:00 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 10:27:08 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 10:27:08 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 10:27:08 - INFO - __main__ - Dataloader configuration:
+12/09/2025 10:27:08 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 10:27:08 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 10:27:08 - INFO - __main__ - - persistent_workers: True
+12/09/2025 10:27:08 - INFO - __main__ - - pin_memory: True
+12/09/2025 10:27:08 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 10:27:26 - INFO - __main__ - ***** Running training *****
+12/09/2025 10:27:26 - INFO - __main__ - Num training steps = 10000
+12/09/2025 10:27:26 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 10:27:26 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 10:27:26 - INFO - __main__ - Gradient Accumulation steps = 8
+12/09/2025 10:28:31 - INFO - __main__ - Step: 10 Loss: 10.5756 LR: 0.000300
+12/09/2025 10:29:29 - INFO - __main__ - Step: 20 Loss: 10.5985 LR: 0.000300
+12/09/2025 10:30:30 - INFO - __main__ - Step: 30 Loss: 10.5381 LR: 0.000300
+12/09/2025 10:31:31 - INFO - __main__ - Step: 40 Loss: 10.5820 LR: 0.000300
+12/09/2025 10:32:30 - INFO - __main__ - Step: 50 Loss: 10.5307 LR: 0.000300
+12/09/2025 10:33:29 - INFO - __main__ - Step: 60 Loss: 10.5077 LR: 0.000300
+12/09/2025 10:34:30 - INFO - __main__ - Step: 70 Loss: 10.5897 LR: 0.000300
+12/09/2025 10:35:28 - INFO - __main__ - Step: 80 Loss: 10.5997 LR: 0.000300
+12/09/2025 10:36:25 - INFO - __main__ - Step: 90 Loss: 10.4886 LR: 0.000300
+12/09/2025 10:37:23 - INFO - __main__ - Step: 100 Loss: 10.5784 LR: 0.000300
+12/09/2025 10:38:22 - INFO - __main__ - Step: 110 Loss: 10.4489 LR: 0.000300
+12/09/2025 10:39:19 - INFO - __main__ - Step: 120 Loss: 10.4976 LR: 0.000300
+12/09/2025 10:40:18 - INFO - __main__ - Step: 130 Loss: 10.5232 LR: 0.000300
+12/09/2025 10:41:19 - INFO - __main__ - Step: 140 Loss: 10.5564 LR: 0.000300
+12/09/2025 10:42:22 - INFO - __main__ - Step: 150 Loss: 10.5033 LR: 0.000300
+12/09/2025 10:43:24 - INFO - __main__ - Step: 160 Loss: 10.6194 LR: 0.000300
+12/09/2025 10:44:26 - INFO - __main__ - Step: 170 Loss: 10.5627 LR: 0.000300
+12/09/2025 10:45:28 - INFO - __main__ - Step: 180 Loss: 10.5360 LR: 0.000300
+12/09/2025 10:46:31 - INFO - __main__ - Step: 190 Loss: 10.5164 LR: 0.000300
+12/09/2025 10:47:33 - INFO - __main__ - Step: 200 Loss: 10.4800 LR: 0.000300
+12/09/2025 10:48:35 - INFO - __main__ - Step: 210 Loss: 10.5856 LR: 0.000300
+12/09/2025 10:49:37 - INFO - __main__ - Step: 220 Loss: 10.5771 LR: 0.000300
+12/09/2025 10:50:38 - INFO - __main__ - Step: 230 Loss: 10.5694 LR: 0.000300
+12/09/2025 10:51:39 - INFO - __main__ - Step: 240 Loss: 10.5486 LR: 0.000300
+12/09/2025 10:52:39 - INFO - __main__ - Step: 250 Loss: 10.5517 LR: 0.000300
+12/09/2025 10:53:41 - INFO - __main__ - Step: 260 Loss: 10.4185 LR: 0.000300
+12/09/2025 10:54:45 - INFO - __main__ - Step: 270 Loss: 10.4816 LR: 0.000300
+12/09/2025 10:55:45 - INFO - __main__ - Step: 280 Loss: 10.5435 LR: 0.000300
+12/09/2025 10:56:46 - INFO - __main__ - Step: 290 Loss: 10.5429 LR: 0.000300
+12/09/2025 10:57:49 - INFO - __main__ - Step: 300 Loss: 10.5335 LR: 0.000300
+12/09/2025 10:58:49 - INFO - __main__ - Step: 310 Loss: 10.5825 LR: 0.000300
+12/09/2025 10:59:51 - INFO - __main__ - Step: 320 Loss: 10.5503 LR: 0.000300
+12/09/2025 11:00:52 - INFO - __main__ - Step: 330 Loss: 10.5399 LR: 0.000300
+12/09/2025 11:01:52 - INFO - __main__ - Step: 340 Loss: 10.4736 LR: 0.000300
+12/09/2025 11:02:52 - INFO - __main__ - Step: 350 Loss: 10.5815 LR: 0.000300
+12/09/2025 11:03:57 - INFO - __main__ - Step: 360 Loss: 10.4822 LR: 0.000300
+12/09/2025 11:05:01 - INFO - __main__ - Step: 370 Loss: 10.5418 LR: 0.000300
+12/09/2025 11:06:02 - INFO - __main__ - Step: 380 Loss: 10.4685 LR: 0.000300
+12/09/2025 11:07:04 - INFO - __main__ - Step: 390 Loss: 10.5073 LR: 0.000300
+12/09/2025 11:08:06 - INFO - __main__ - Step: 400 Loss: 10.5048 LR: 0.000300
+12/09/2025 11:09:07 - INFO - __main__ - Step: 410 Loss: 10.4811 LR: 0.000300
+12/09/2025 11:10:09 - INFO - __main__ - Step: 420 Loss: 10.4874 LR: 0.000300
+12/09/2025 11:11:10 - INFO - __main__ - Step: 430 Loss: 10.5045 LR: 0.000300
+12/09/2025 11:12:12 - INFO - __main__ - Step: 440 Loss: 10.4710 LR: 0.000300
+12/09/2025 11:13:13 - INFO - __main__ - Step: 450 Loss: 10.5233 LR: 0.000300
+12/09/2025 11:14:13 - INFO - __main__ - Step: 460 Loss: 10.4441 LR: 0.000300
+12/09/2025 11:15:13 - INFO - __main__ - Step: 470 Loss: 10.4439 LR: 0.000300
+12/09/2025 11:16:13 - INFO - __main__ - Step: 480 Loss: 10.5054 LR: 0.000300
+12/09/2025 11:17:12 - INFO - __main__ - Step: 490 Loss: 10.3729 LR: 0.000300
+12/09/2025 11:18:12 - INFO - __main__ - Step: 500 Loss: 10.5167 LR: 0.000300
+12/09/2025 11:18:12 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500
+12/09/2025 11:18:20 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500/optimizer.bin
+12/09/2025 11:18:20 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500/scheduler.bin
+12/09/2025 11:18:20 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500/sampler.bin
+12/09/2025 11:18:20 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500/random_states_0.pkl
+12/09/2025 11:18:20 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-500
+12/09/2025 11:18:20 - INFO - __main__ - Generating videos for validation...
+12/09/2025 11:18:20 - INFO - __main__ - Generating videos for validation...
+100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.59it/s]
+12/09/2025 11:18:27 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue
+12/09/2025 11:19:28 - INFO - __main__ - Step: 510 Loss: 10.4568 LR: 0.000300
+12/09/2025 11:20:28 - INFO - __main__ - Step: 520 Loss: 10.4816 LR: 0.000300
+12/09/2025 11:21:29 - INFO - __main__ - Step: 530 Loss: 10.3487 LR: 0.000300
+12/09/2025 11:22:29 - INFO - __main__ - Step: 540 Loss: 10.5037 LR: 0.000300
+12/09/2025 11:23:30 - INFO - __main__ - Step: 550 Loss: 10.5109 LR: 0.000300
+12/09/2025 11:24:34 - INFO - __main__ - Step: 560 Loss: 10.4070 LR: 0.000300
+12/09/2025 11:25:36 - INFO - __main__ - Step: 570 Loss: 10.3687 LR: 0.000300
+12/09/2025 11:26:40 - INFO - __main__ - Step: 580 Loss: 10.4315 LR: 0.000300
+12/09/2025 11:27:41 - INFO - __main__ - Step: 590 Loss: 10.5524 LR: 0.000300
+12/09/2025 11:28:43 - INFO - __main__ - Step: 600 Loss: 10.4376 LR: 0.000300
+12/09/2025 11:29:45 - INFO - __main__ - Step: 610 Loss: 10.4982 LR: 0.000300
+12/09/2025 11:30:44 - INFO - __main__ - Step: 620 Loss: 10.5766 LR: 0.000300
+12/09/2025 11:31:46 - INFO - __main__ - Step: 630 Loss: 10.4860 LR: 0.000300
+12/09/2025 11:32:48 - INFO - __main__ - Step: 640 Loss: 10.3650 LR: 0.000300
+12/09/2025 11:33:50 - INFO - __main__ - Step: 650 Loss: 10.3143 LR: 0.000300
+12/09/2025 11:34:51 - INFO - __main__ - Step: 660 Loss: 10.4826 LR: 0.000300
+12/09/2025 11:35:52 - INFO - __main__ - Step: 670 Loss: 10.5079 LR: 0.000300
+12/09/2025 11:36:54 - INFO - __main__ - Step: 680 Loss: 10.4871 LR: 0.000300
+12/09/2025 11:37:55 - INFO - __main__ - Step: 690 Loss: 10.3807 LR: 0.000300
+12/09/2025 11:38:56 - INFO - __main__ - Step: 700 Loss: 10.5205 LR: 0.000300
+12/09/2025 11:39:59 - INFO - __main__ - Step: 710 Loss: 10.5428 LR: 0.000300
+12/09/2025 11:40:59 - INFO - __main__ - Step: 720 Loss: 10.3462 LR: 0.000300
+12/09/2025 11:41:57 - INFO - __main__ - Step: 730 Loss: 10.4895 LR: 0.000300
+12/09/2025 11:42:59 - INFO - __main__ - Step: 740 Loss: 10.4331 LR: 0.000300
+12/09/2025 11:44:02 - INFO - __main__ - Step: 750 Loss: 10.4456 LR: 0.000300
+12/09/2025 11:45:04 - INFO - __main__ - Step: 760 Loss: 10.4432 LR: 0.000300
+12/09/2025 11:46:05 - INFO - __main__ - Step: 770 Loss: 10.4319 LR: 0.000300
+12/09/2025 11:47:05 - INFO - __main__ - Step: 780 Loss: 10.3427 LR: 0.000300
+12/09/2025 11:48:05 - INFO - __main__ - Step: 790 Loss: 10.5008 LR: 0.000300
+12/09/2025 11:49:04 - INFO - __main__ - Step: 800 Loss: 10.4130 LR: 0.000300
+12/09/2025 11:50:04 - INFO - __main__ - Step: 810 Loss: 10.3861 LR: 0.000300
+12/09/2025 11:51:06 - INFO - __main__ - Step: 820 Loss: 10.4996 LR: 0.000300
+12/09/2025 11:52:07 - INFO - __main__ - Step: 830 Loss: 10.4474 LR: 0.000300
+12/09/2025 11:53:09 - INFO - __main__ - Step: 840 Loss: 10.2460 LR: 0.000300
+12/09/2025 11:54:08 - INFO - __main__ - Step: 850 Loss: 10.4203 LR: 0.000300
+12/09/2025 11:55:09 - INFO - __main__ - Step: 860 Loss: 10.3336 LR: 0.000300
+12/09/2025 11:56:11 - INFO - __main__ - Step: 870 Loss: 10.4718 LR: 0.000300
+12/09/2025 11:57:12 - INFO - __main__ - Step: 880 Loss: 10.3754 LR: 0.000300
+12/09/2025 11:58:15 - INFO - __main__ - Step: 890 Loss: 10.4541 LR: 0.000300
+12/09/2025 11:59:16 - INFO - __main__ - Step: 900 Loss: 10.4114 LR: 0.000300
+12/09/2025 12:00:15 - INFO - __main__ - Step: 910 Loss: 10.4875 LR: 0.000300
+12/09/2025 12:01:17 - INFO - __main__ - Step: 920 Loss: 10.4626 LR: 0.000300
+12/09/2025 12:02:19 - INFO - __main__ - Step: 930 Loss: 10.4343 LR: 0.000300
+12/09/2025 12:03:19 - INFO - __main__ - Step: 940 Loss: 10.4718 LR: 0.000300
+12/09/2025 12:04:20 - INFO - __main__ - Step: 950 Loss: 10.4602 LR: 0.000300
+12/09/2025 12:05:20 - INFO - __main__ - Step: 960 Loss: 10.3744 LR: 0.000300
+12/09/2025 12:06:23 - INFO - __main__ - Step: 970 Loss: 10.3601 LR: 0.000300
+12/09/2025 12:07:23 - INFO - __main__ - Step: 980 Loss: 10.4573 LR: 0.000300
+12/09/2025 12:08:24 - INFO - __main__ - Step: 990 Loss: 10.5020 LR: 0.000300
+12/09/2025 12:09:28 - INFO - __main__ - Step: 1000 Loss: 10.3293 LR: 0.000300
+12/09/2025 12:09:28 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000
+12/09/2025 12:09:37 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000/optimizer.bin
+12/09/2025 12:09:37 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000/scheduler.bin
+12/09/2025 12:09:37 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000/sampler.bin
+12/09/2025 12:09:37 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000/random_states_0.pkl
+12/09/2025 12:09:37 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1000
+12/09/2025 12:09:37 - INFO - __main__ - Generating videos for validation...
+12/09/2025 12:09:37 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.80it/s]
+12/09/2025 12:09:45 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue
+12/09/2025 12:10:47 - INFO - __main__ - Step: 1010 Loss: 10.2537 LR: 0.000300
+12/09/2025 12:11:48 - INFO - __main__ - Step: 1020 Loss: 10.3517 LR: 0.000300
+12/09/2025 12:12:46 - INFO - __main__ - Step: 1030 Loss: 10.3686 LR: 0.000300
+12/09/2025 12:13:48 - INFO - __main__ - Step: 1040 Loss: 10.3712 LR: 0.000300
+12/09/2025 12:14:48 - INFO - __main__ - Step: 1050 Loss: 10.4094 LR: 0.000300
+12/09/2025 12:15:48 - INFO - __main__ - Step: 1060 Loss: 10.4581 LR: 0.000300
+12/09/2025 12:16:49 - INFO - __main__ - Step: 1070 Loss: 10.4886 LR: 0.000300
+12/09/2025 12:17:49 - INFO - __main__ - Step: 1080 Loss: 10.3462 LR: 0.000300
+12/09/2025 12:18:50 - INFO - __main__ - Step: 1090 Loss: 10.4209 LR: 0.000300
+12/09/2025 12:19:49 - INFO - __main__ - Step: 1100 Loss: 10.4281 LR: 0.000300
+12/09/2025 12:20:49 - INFO - __main__ - Step: 1110 Loss: 10.4686 LR: 0.000300
+12/09/2025 12:21:51 - INFO - __main__ - Step: 1120 Loss: 10.4089 LR: 0.000300
+12/09/2025 12:22:52 - INFO - __main__ - Step: 1130 Loss: 10.3998 LR: 0.000300
+12/09/2025 12:23:52 - INFO - __main__ - Step: 1140 Loss: 10.3321 LR: 0.000300
+12/09/2025 12:24:54 - INFO - __main__ - Step: 1150 Loss: 10.2142 LR: 0.000300
+12/09/2025 12:25:53 - INFO - __main__ - Step: 1160 Loss: 10.4853 LR: 0.000300
+12/09/2025 12:26:50 - INFO - __main__ - Step: 1170 Loss: 10.5006 LR: 0.000300
+12/09/2025 12:27:49 - INFO - __main__ - Step: 1180 Loss: 10.4385 LR: 0.000300
+12/09/2025 12:28:48 - INFO - __main__ - Step: 1190 Loss: 10.1816 LR: 0.000300
+12/09/2025 12:29:48 - INFO - __main__ - Step: 1200 Loss: 10.3925 LR: 0.000300
+12/09/2025 12:30:48 - INFO - __main__ - Step: 1210 Loss: 10.2949 LR: 0.000300
+12/09/2025 12:31:46 - INFO - __main__ - Step: 1220 Loss: 10.3461 LR: 0.000300
+12/09/2025 12:32:45 - INFO - __main__ - Step: 1230 Loss: 10.4405 LR: 0.000300
+12/09/2025 12:33:44 - INFO - __main__ - Step: 1240 Loss: 10.3677 LR: 0.000300
+12/09/2025 12:34:42 - INFO - __main__ - Step: 1250 Loss: 10.4093 LR: 0.000300
+12/09/2025 12:35:42 - INFO - __main__ - Step: 1260 Loss: 10.4217 LR: 0.000300
+12/09/2025 12:36:42 - INFO - __main__ - Step: 1270 Loss: 10.4606 LR: 0.000300
+12/09/2025 12:37:42 - INFO - __main__ - Step: 1280 Loss: 10.4633 LR: 0.000300
+12/09/2025 12:38:40 - INFO - __main__ - Step: 1290 Loss: 10.3848 LR: 0.000300
+12/09/2025 12:39:38 - INFO - __main__ - Step: 1300 Loss: 10.4616 LR: 0.000300
+12/09/2025 12:40:35 - INFO - __main__ - Step: 1310 Loss: 10.4222 LR: 0.000300
+12/09/2025 12:41:36 - INFO - __main__ - Step: 1320 Loss: 10.4237 LR: 0.000300
+12/09/2025 12:42:36 - INFO - __main__ - Step: 1330 Loss: 10.4675 LR: 0.000300
+12/09/2025 12:43:38 - INFO - __main__ - Step: 1340 Loss: 10.4428 LR: 0.000300
+12/09/2025 12:44:37 - INFO - __main__ - Step: 1350 Loss: 10.3288 LR: 0.000300
+12/09/2025 12:45:37 - INFO - __main__ - Step: 1360 Loss: 10.4652 LR: 0.000300
+12/09/2025 12:46:37 - INFO - __main__ - Step: 1370 Loss: 10.2860 LR: 0.000300
+12/09/2025 12:47:38 - INFO - __main__ - Step: 1380 Loss: 10.4438 LR: 0.000300
+12/09/2025 12:48:38 - INFO - __main__ - Step: 1390 Loss: 10.2577 LR: 0.000300
+12/09/2025 12:49:36 - INFO - __main__ - Step: 1400 Loss: 10.4907 LR: 0.000300
+12/09/2025 12:50:33 - INFO - __main__ - Step: 1410 Loss: 10.4241 LR: 0.000300
+12/09/2025 12:51:32 - INFO - __main__ - Step: 1420 Loss: 10.4317 LR: 0.000300
+12/09/2025 12:52:31 - INFO - __main__ - Step: 1430 Loss: 10.4626 LR: 0.000300
+12/09/2025 12:53:29 - INFO - __main__ - Step: 1440 Loss: 9.9582 LR: 0.000300
+12/09/2025 12:54:25 - INFO - __main__ - Step: 1450 Loss: 10.4416 LR: 0.000300
+12/09/2025 12:55:20 - INFO - __main__ - Step: 1460 Loss: 10.4264 LR: 0.000300
+12/09/2025 12:56:15 - INFO - __main__ - Step: 1470 Loss: 10.4191 LR: 0.000300
+12/09/2025 12:57:12 - INFO - __main__ - Step: 1480 Loss: 10.4106 LR: 0.000300
+12/09/2025 12:58:08 - INFO - __main__ - Step: 1490 Loss: 10.4202 LR: 0.000300
+12/09/2025 12:59:05 - INFO - __main__ - Step: 1500 Loss: 10.3524 LR: 0.000300
+12/09/2025 12:59:05 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500
+12/09/2025 12:59:14 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500/optimizer.bin
+12/09/2025 12:59:14 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500/scheduler.bin
+12/09/2025 12:59:14 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500/sampler.bin
+12/09/2025 12:59:14 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500/random_states_0.pkl
+12/09/2025 12:59:14 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue/checkpoint-1500
+12/09/2025 12:59:14 - INFO - __main__ - Generating videos for validation...
+12/09/2025 12:59:14 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.85it/s]
+12/09/2025 12:59:21 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue
+12/09/2025 13:00:21 - INFO - __main__ - Step: 1510 Loss: 10.4142 LR: 0.000300
+12/09/2025 13:01:20 - INFO - __main__ - Step: 1520 Loss: 10.3434 LR: 0.000300
+12/09/2025 13:02:21 - INFO - __main__ - Step: 1530 Loss: 10.3446 LR: 0.000300
+12/09/2025 13:03:20 - INFO - __main__ - Step: 1540 Loss: 10.2433 LR: 0.000300
+12/09/2025 13:04:20 - INFO - __main__ - Step: 1550 Loss: 10.4155 LR: 0.000300
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1333, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1333, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/requirements.txt b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..34608b872588695b6b3f38c7194f68f73d071956
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/files/wandb-metadata.json
@@ -0,0 +1,149 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T10:26:51.247187Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "8",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12040636624896"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "hsmu5rjpjbgh0nwq7xjadpomdbmu3xn2"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-core.log b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..89b2bd21bb34cfc83b9539dd7386fb9855fe0e6e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T10:26:51.315145192Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpmv5pwgec/port-1814887.txt","pid":1814887,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T10:26:51.315676461Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1814887}
+{"time":"2025-12-09T10:26:51.315686707Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1814887-1815159-3368749728/socket","Net":"unix"}}
+{"time":"2025-12-09T10:26:51.501584188Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T10:26:51.507573641Z","level":"INFO","msg":"handleInformInit: received","streamId":"55o5soqg","id":"1(@)"}
+{"time":"2025-12-09T10:26:51.697675637Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"55o5soqg","id":"1(@)"}
+{"time":"2025-12-09T13:05:17.894692268Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-internal.log b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..ae66f536156494f6b2ddf0b1cdd0561c04fb4ccc
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T10:26:51.507674532Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T10:26:51.697475367Z","level":"INFO","msg":"stream: created new stream","id":"55o5soqg"}
+{"time":"2025-12-09T10:26:51.697546226Z","level":"INFO","msg":"handler: started","stream_id":"55o5soqg"}
+{"time":"2025-12-09T10:26:51.697669497Z","level":"INFO","msg":"stream: started","id":"55o5soqg"}
+{"time":"2025-12-09T10:26:51.697687428Z","level":"INFO","msg":"sender: started","stream_id":"55o5soqg"}
+{"time":"2025-12-09T10:26:51.697686518Z","level":"INFO","msg":"writer: started","stream_id":"55o5soqg"}
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug.log b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..c528c5b2fa4eadfcdcf0a783845ba08938d1bf8f
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_setup.py:_flush():80] Configure stats pid to 1814887
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug.log
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_102651-55o5soqg/logs/debug-internal.log
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_init.py:init():841] calling init triggers
+2025-12-09 10:26:51,249 INFO MainThread:1814887 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 10:26:51,250 INFO MainThread:1814887 [wandb_init.py:init():889] starting backend
+2025-12-09 10:26:51,501 INFO MainThread:1814887 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 10:26:51,506 INFO MainThread:1814887 [wandb_init.py:init():900] backend started and connected
+2025-12-09 10:26:51,507 INFO MainThread:1814887 [wandb_init.py:init():970] updated telemetry
+2025-12-09 10:26:51,511 INFO MainThread:1814887 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 10:26:52,078 INFO MainThread:1814887 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 10:26:52,203 INFO MainThread:1814887 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 10:26:52,203 INFO MainThread:1814887 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 10:26:52,203 INFO MainThread:1814887 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 10:26:52,203 INFO MainThread:1814887 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 10:26:52,206 INFO MainThread:1814887 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 10:26:52,207 INFO MainThread:1814887 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': False, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_102651-55o5soqg/run-55o5soqg.wandb b/Meissonic/wandb/run-20251209_102651-55o5soqg/run-55o5soqg.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a5f09ea7316d40c41ea7b4ab9e7a6454e9cb2fde
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_102651-55o5soqg/run-55o5soqg.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2425b0a0a1225df970143645fbb05621ecc43f561be094223de12c44897e1caf
+size 2392064
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/files/config.yaml b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3691389a26af5dbc4ce4c901951467b8924280fe
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/config.yaml
@@ -0,0 +1,297 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ bldt1h2u4zo6vc79xvm2e3ynt6e3tebc:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --pretrained_model_name_or_path
+ - /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "8"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093761687552"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T13:50:22.825258Z"
+ writerId: bldt1h2u4zo6vc79xvm2e3ynt6e3tebc
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 8
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: false
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: null
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/files/output.log b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..a85066a6de9c97e299c0b71d51818b320dc24c12
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/output.log
@@ -0,0 +1,54 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 68.30it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 13:50:26 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+12/09/2025 13:50:26 - INFO - __main__ - Using precomputed features - video tokenizer will only be used for mask_token_id
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5728.81it/s]
+12/09/2025 13:50:32 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 13:50:32 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 13:50:32 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 13:50:32 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 13:50:32 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 13:50:32 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 13:50:32 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 13:50:32 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 13:50:32 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 13:50:32 - INFO - __main__ - Dataloader configuration:
+12/09/2025 13:50:32 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 13:50:32 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 13:50:32 - INFO - __main__ - - persistent_workers: True
+12/09/2025 13:50:32 - INFO - __main__ - - pin_memory: True
+12/09/2025 13:50:32 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 13:50:48 - INFO - __main__ - ***** Running training *****
+12/09/2025 13:50:48 - INFO - __main__ - Num training steps = 10000
+12/09/2025 13:50:48 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 13:50:48 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 13:50:48 - INFO - __main__ - Gradient Accumulation steps = 8
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1322, in main
+ loss = F.cross_entropy(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+ return torch._C._nn.cross_entropy_loss(
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 4.18 GiB is free. Process 42760 has 414.00 MiB memory in use. Process 42757 has 414.00 MiB memory in use. Process 42758 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 32.43 GiB memory in use. Process 42759 has 414.00 MiB memory in use. Process 42754 has 414.00 MiB memory in use. Process 42755 has 414.00 MiB memory in use. Process 42756 has 414.00 MiB memory in use. Of the allocated memory 31.07 GiB is allocated by PyTorch, and 274.38 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1485, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1322, in main
+[rank0]: loss = F.cross_entropy(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+[rank0]: return torch._C._nn.cross_entropy_loss(
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 4.18 GiB is free. Process 42760 has 414.00 MiB memory in use. Process 42757 has 414.00 MiB memory in use. Process 42758 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 32.43 GiB memory in use. Process 42759 has 414.00 MiB memory in use. Process 42754 has 414.00 MiB memory in use. Process 42755 has 414.00 MiB memory in use. Process 42756 has 414.00 MiB memory in use. Of the allocated memory 31.07 GiB is allocated by PyTorch, and 274.38 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/files/requirements.txt b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..77df7e2b6713a804b3c8382827a181a22bab0525
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-metadata.json
@@ -0,0 +1,152 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T13:50:22.825258Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "8",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093761687552"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "bldt1h2u4zo6vc79xvm2e3ynt6e3tebc"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-summary.json b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..978ee889fedb890efdfac2b1684d5a836afcc2b0
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":54},"_runtime":54}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-core.log b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..56473134125ed9b6925e646ca947f55302e249ad
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-core.log
@@ -0,0 +1,12 @@
+{"time":"2025-12-09T13:50:22.900042475Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpao4xz0yf/port-42753.txt","pid":42753,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T13:50:22.900572652Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":42753}
+{"time":"2025-12-09T13:50:22.90058245Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-42753-43005-1356035954/socket","Net":"unix"}}
+{"time":"2025-12-09T13:50:23.081102199Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T13:50:23.08910113Z","level":"INFO","msg":"handleInformInit: received","streamId":"dl453cfo","id":"1(@)"}
+{"time":"2025-12-09T13:50:23.255636669Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"dl453cfo","id":"1(@)"}
+{"time":"2025-12-09T13:51:18.281641871Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T13:51:18.281712181Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T13:51:18.281774707Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T13:51:18.281728224Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T13:51:18.28204532Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-42753-43005-1356035954/socket","Net":"unix"}}
+{"time":"2025-12-09T13:51:18.516370005Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-internal.log b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..5694067477fda531a59d251bb2202cd7520df961
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-internal.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T13:50:23.089330458Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T13:50:23.255438447Z","level":"INFO","msg":"stream: created new stream","id":"dl453cfo"}
+{"time":"2025-12-09T13:50:23.255518205Z","level":"INFO","msg":"handler: started","stream_id":"dl453cfo"}
+{"time":"2025-12-09T13:50:23.2556299Z","level":"INFO","msg":"stream: started","id":"dl453cfo"}
+{"time":"2025-12-09T13:50:23.255647387Z","level":"INFO","msg":"writer: started","stream_id":"dl453cfo"}
+{"time":"2025-12-09T13:50:23.255650645Z","level":"INFO","msg":"sender: started","stream_id":"dl453cfo"}
+{"time":"2025-12-09T13:51:18.281710591Z","level":"INFO","msg":"stream: closing","id":"dl453cfo"}
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug.log b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..5838a332244f3e2820ca78a07ca1c64f48c0c009
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 13:50:22,828 INFO MainThread:42753 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 13:50:22,828 INFO MainThread:42753 [wandb_setup.py:_flush():80] Configure stats pid to 42753
+2025-12-09 13:50:22,828 INFO MainThread:42753 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 13:50:22,828 INFO MainThread:42753 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug.log
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_135022-dl453cfo/logs/debug-internal.log
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_init.py:init():841] calling init triggers
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 13:50:22,829 INFO MainThread:42753 [wandb_init.py:init():889] starting backend
+2025-12-09 13:50:23,081 INFO MainThread:42753 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 13:50:23,087 INFO MainThread:42753 [wandb_init.py:init():900] backend started and connected
+2025-12-09 13:50:23,088 INFO MainThread:42753 [wandb_init.py:init():970] updated telemetry
+2025-12-09 13:50:23,093 INFO MainThread:42753 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 13:50:23,595 INFO MainThread:42753 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 13:50:23,719 INFO MainThread:42753 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 13:50:23,719 INFO MainThread:42753 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 13:50:23,720 INFO MainThread:42753 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 13:50:23,720 INFO MainThread:42753 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 13:50:23,722 INFO MainThread:42753 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 13:50:23,723 INFO MainThread:42753 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': False, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 13:51:18,281 INFO wandb-AsyncioManager-main:42753 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 13:51:18,281 INFO wandb-AsyncioManager-main:42753 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_135022-dl453cfo/run-dl453cfo.wandb b/Meissonic/wandb/run-20251209_135022-dl453cfo/run-dl453cfo.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..30977402b08eb894463b83d7de7a007255879a32
Binary files /dev/null and b/Meissonic/wandb/run-20251209_135022-dl453cfo/run-dl453cfo.wandb differ
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/config.yaml b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..202132cf41af2c7ba127f7a9237a8f5d729945e3
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/config.yaml
@@ -0,0 +1,297 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ me1mlu82n7mm3lyja37emytxtqqfliqv:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --pretrained_model_name_or_path
+ - /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "8"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093761908736"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T13:56:38.527057Z"
+ writerId: me1mlu82n7mm3lyja37emytxtqqfliqv
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 8
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: null
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/output.log b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..e50157ddbcaeb13d14e24dc750d677404541fd9c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/output.log
@@ -0,0 +1,18 @@
+12/09/2025 13:56:39 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 13:56:39 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 13:56:39 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 13:56:39 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4950.28it/s]
+12/09/2025 13:56:40 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1587, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 609, in main
+ dummy_tokens = video_tokenizer.encode(dummy_video) # [1, F', H', W']
+AttributeError: 'MinimalTokenizer' object has no attribute 'encode'
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1587, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 609, in main
+[rank0]: dummy_tokens = video_tokenizer.encode(dummy_video) # [1, F', H', W']
+[rank0]: AttributeError: 'MinimalTokenizer' object has no attribute 'encode'
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/requirements.txt b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..6452039bc89e441abff9075489ceb1b93dec85c5
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-metadata.json
@@ -0,0 +1,152 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T13:56:38.527057Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "8",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093761908736"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "me1mlu82n7mm3lyja37emytxtqqfliqv"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-summary.json b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0a620d0c1047a4dd8a400939b6da246ed8063a7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-core.log b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..7debe050b38fd74f80ee40630043fc418f23b372
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T13:56:38.595965875Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp6xfqqjho/port-48464.txt","pid":48464,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T13:56:38.596570264Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":48464}
+{"time":"2025-12-09T13:56:38.596573389Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-48464-48765-3576475713/socket","Net":"unix"}}
+{"time":"2025-12-09T13:56:38.782286222Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T13:56:38.788282902Z","level":"INFO","msg":"handleInformInit: received","streamId":"lmzlczvn","id":"1(@)"}
+{"time":"2025-12-09T13:56:39.557589177Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"lmzlczvn","id":"1(@)"}
+{"time":"2025-12-09T13:56:40.672527157Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T13:56:40.67257452Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T13:56:40.672567954Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T13:56:40.672615099Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T13:56:40.672642951Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-48464-48765-3576475713/socket","Net":"unix"}}
+{"time":"2025-12-09T13:56:41.19927464Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T13:56:41.199306855Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T13:56:41.199316754Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-internal.log b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..808ee64021b9ea38bfb842421bcb8f5ddd1fb637
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T13:56:38.788417234Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T13:56:39.557024041Z","level":"INFO","msg":"stream: created new stream","id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:39.557152083Z","level":"INFO","msg":"handler: started","stream_id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:39.557575502Z","level":"INFO","msg":"stream: started","id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:39.557588441Z","level":"INFO","msg":"writer: started","stream_id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:39.557588566Z","level":"INFO","msg":"sender: started","stream_id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:40.672577239Z","level":"INFO","msg":"stream: closing","id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:41.080716886Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T13:56:41.195394384Z","level":"INFO","msg":"handler: closed","stream_id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:41.196096167Z","level":"INFO","msg":"sender: closed","stream_id":"lmzlczvn"}
+{"time":"2025-12-09T13:56:41.196108619Z","level":"INFO","msg":"stream: closed","id":"lmzlczvn"}
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug.log b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..5cbd0df5230c04811c709dbf6950288b833d8201
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 13:56:38,529 INFO MainThread:48464 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 13:56:38,529 INFO MainThread:48464 [wandb_setup.py:_flush():80] Configure stats pid to 48464
+2025-12-09 13:56:38,529 INFO MainThread:48464 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 13:56:38,529 INFO MainThread:48464 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug.log
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_135638-lmzlczvn/logs/debug-internal.log
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_init.py:init():841] calling init triggers
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 13:56:38,530 INFO MainThread:48464 [wandb_init.py:init():889] starting backend
+2025-12-09 13:56:38,782 INFO MainThread:48464 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 13:56:38,786 INFO MainThread:48464 [wandb_init.py:init():900] backend started and connected
+2025-12-09 13:56:38,787 INFO MainThread:48464 [wandb_init.py:init():970] updated telemetry
+2025-12-09 13:56:38,792 INFO MainThread:48464 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 13:56:39,777 INFO MainThread:48464 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 13:56:39,900 INFO MainThread:48464 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 13:56:39,900 INFO MainThread:48464 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 13:56:39,900 INFO MainThread:48464 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 13:56:39,900 INFO MainThread:48464 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 13:56:39,903 INFO MainThread:48464 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 13:56:39,904 INFO MainThread:48464 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 13:56:40,672 INFO wandb-AsyncioManager-main:48464 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 13:56:40,672 INFO wandb-AsyncioManager-main:48464 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_135638-lmzlczvn/run-lmzlczvn.wandb b/Meissonic/wandb/run-20251209_135638-lmzlczvn/run-lmzlczvn.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..9e62e7d6e3f702d179c67d6a7e9c08a48fb037d2
Binary files /dev/null and b/Meissonic/wandb/run-20251209_135638-lmzlczvn/run-lmzlczvn.wandb differ
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/files/config.yaml b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..418f911e212549fad9b75d6be2c1cec17ef8250e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/config.yaml
@@ -0,0 +1,297 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 4c3apb56ykx61de6cbks2kgguigh72w8:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --pretrained_model_name_or_path
+ - /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "8"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093762088960"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T13:59:02.075119Z"
+ writerId: 4c3apb56ykx61de6cbks2kgguigh72w8
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 8
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: /mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: null
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/files/output.log b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..85d46cba282205fb8c9c2eb1822df345b1c534f5
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/output.log
@@ -0,0 +1,60 @@
+12/09/2025 13:59:03 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 13:59:03 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 13:59:03 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 13:59:03 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5598.80it/s]
+12/09/2025 13:59:04 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 13:59:04 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 13:59:04 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 13:59:04 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 13:59:34 - INFO - __main__ - Parameter counts: backbone=2,191,878,208, other=2,112,033, total=2,193,990,241
+12/09/2025 13:59:34 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 13:59:34 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 13:59:34 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 13:59:34 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 13:59:34 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 13:59:34 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 13:59:34 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 13:59:34 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 13:59:34 - INFO - __main__ - Dataloader configuration:
+12/09/2025 13:59:34 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 13:59:34 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 13:59:34 - INFO - __main__ - - persistent_workers: True
+12/09/2025 13:59:34 - INFO - __main__ - - pin_memory: True
+12/09/2025 13:59:34 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 13:59:36 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 13:59:36 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 13:59:36 - INFO - __main__ - ***** Running training *****
+12/09/2025 13:59:36 - INFO - __main__ - Num training steps = 10000
+12/09/2025 13:59:36 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 13:59:36 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 13:59:36 - INFO - __main__ - Gradient Accumulation steps = 8
+12/09/2025 13:59:37 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1675, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1492, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 3.84 GiB is free. Process 50348 has 414.00 MiB memory in use. Process 50344 has 414.00 MiB memory in use. Process 50345 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 32.77 GiB memory in use. Process 50349 has 414.00 MiB memory in use. Process 50347 has 414.00 MiB memory in use. Process 50350 has 414.00 MiB memory in use. Process 50346 has 414.00 MiB memory in use. Of the allocated memory 30.94 GiB is allocated by PyTorch, and 753.42 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1675, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1492, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 3.84 GiB is free. Process 50348 has 414.00 MiB memory in use. Process 50344 has 414.00 MiB memory in use. Process 50345 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 32.77 GiB memory in use. Process 50349 has 414.00 MiB memory in use. Process 50347 has 414.00 MiB memory in use. Process 50350 has 414.00 MiB memory in use. Process 50346 has 414.00 MiB memory in use. Of the allocated memory 30.94 GiB is allocated by PyTorch, and 753.42 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/files/requirements.txt b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..38f593a42902dd4775e0362819f414cf7ca54e5f
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-metadata.json
@@ -0,0 +1,152 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T13:59:02.075119Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--pretrained_model_name_or_path",
+ "/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "8",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093762088960"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "4c3apb56ykx61de6cbks2kgguigh72w8"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-summary.json b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..48f7d9a062ccb1f6a06780718d54c131168a079a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":77},"_runtime":77}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-core.log b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..bc12488022a3371df43b6280e3d8cd0bf84f6715
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T13:59:02.143839344Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpzs4b0rhb/port-50343.txt","pid":50343,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T13:59:02.144318261Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":50343}
+{"time":"2025-12-09T13:59:02.144330176Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-50343-50653-3284053146/socket","Net":"unix"}}
+{"time":"2025-12-09T13:59:02.330489605Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T13:59:02.335981868Z","level":"INFO","msg":"handleInformInit: received","streamId":"iktd65kq","id":"1(@)"}
+{"time":"2025-12-09T13:59:03.106386993Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"iktd65kq","id":"1(@)"}
+{"time":"2025-12-09T14:00:20.892712387Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T14:00:20.892825862Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T14:00:20.892814995Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T14:00:20.892953913Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T14:00:20.892964195Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-50343-50653-3284053146/socket","Net":"unix"}}
+{"time":"2025-12-09T14:00:21.260765955Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T14:00:21.26078412Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T14:00:21.260792765Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-internal.log b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..2cc97844edd328fc54fa01bda864f764035ad9ac
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T13:59:02.336109648Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T13:59:03.106167184Z","level":"INFO","msg":"stream: created new stream","id":"iktd65kq"}
+{"time":"2025-12-09T13:59:03.106257767Z","level":"INFO","msg":"handler: started","stream_id":"iktd65kq"}
+{"time":"2025-12-09T13:59:03.106377335Z","level":"INFO","msg":"stream: started","id":"iktd65kq"}
+{"time":"2025-12-09T13:59:03.106393438Z","level":"INFO","msg":"writer: started","stream_id":"iktd65kq"}
+{"time":"2025-12-09T13:59:03.106402792Z","level":"INFO","msg":"sender: started","stream_id":"iktd65kq"}
+{"time":"2025-12-09T14:00:20.892812789Z","level":"INFO","msg":"stream: closing","id":"iktd65kq"}
+{"time":"2025-12-09T14:00:21.165060913Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T14:00:21.257647014Z","level":"INFO","msg":"handler: closed","stream_id":"iktd65kq"}
+{"time":"2025-12-09T14:00:21.257734093Z","level":"INFO","msg":"sender: closed","stream_id":"iktd65kq"}
+{"time":"2025-12-09T14:00:21.257740775Z","level":"INFO","msg":"stream: closed","id":"iktd65kq"}
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug.log b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..96a4d9ff8dd6c01db6eae94c966a874c69ddd64a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_setup.py:_flush():80] Configure stats pid to 50343
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug.log
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_135902-iktd65kq/logs/debug-internal.log
+2025-12-09 13:59:02,077 INFO MainThread:50343 [wandb_init.py:init():841] calling init triggers
+2025-12-09 13:59:02,078 INFO MainThread:50343 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 13:59:02,078 INFO MainThread:50343 [wandb_init.py:init():889] starting backend
+2025-12-09 13:59:02,330 INFO MainThread:50343 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 13:59:02,334 INFO MainThread:50343 [wandb_init.py:init():900] backend started and connected
+2025-12-09 13:59:02,335 INFO MainThread:50343 [wandb_init.py:init():970] updated telemetry
+2025-12-09 13:59:02,339 INFO MainThread:50343 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 13:59:03,425 INFO MainThread:50343 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 13:59:03,550 INFO MainThread:50343 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 13:59:03,551 INFO MainThread:50343 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 13:59:03,551 INFO MainThread:50343 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 13:59:03,551 INFO MainThread:50343 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 13:59:03,554 INFO MainThread:50343 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 13:59:03,555 INFO MainThread:50343 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': '/mnt/Meissonic/output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio/checkpoint-4000', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': None, 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 14:00:20,892 INFO wandb-AsyncioManager-main:50343 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 14:00:20,893 INFO wandb-AsyncioManager-main:50343 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_135902-iktd65kq/run-iktd65kq.wandb b/Meissonic/wandb/run-20251209_135902-iktd65kq/run-iktd65kq.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..d639d1bdce2204e33d6b4debf3b098896874335a
Binary files /dev/null and b/Meissonic/wandb/run-20251209_135902-iktd65kq/run-iktd65kq.wandb differ
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/files/config.yaml b/Meissonic/wandb/run-20251209_140322-otx47txf/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c0af9e5372997e0b0c7e6b4de22e023385b476f0
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/files/config.yaml
@@ -0,0 +1,245 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ p6u11ug9znfnhceiv1d6rhmfrzlq1mla:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093762334720"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T14:03:22.803470Z"
+ writerId: p6u11ug9znfnhceiv1d6rhmfrzlq1mla
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: false
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0
+dataloader_num_workers:
+ value: 4
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: false
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: null
+instance_data_image:
+ value: null
+instance_dataset:
+ value: null
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 50
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 500
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: null
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: null
+num_frames:
+ value: 16
+output_dir:
+ value: muse_training
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: null
+split_vae_encode:
+ value: null
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: false
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value: null
+validation_steps:
+ value: 100
+variant:
+ value: null
+video_height:
+ value: 480
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 848
+wan_backbone_lr_ratio:
+ value: 0.1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/files/output.log b/Meissonic/wandb/run-20251209_140322-otx47txf/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..8b33dfa6a4eaca808318f8960994a28e957e3458
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/files/output.log
@@ -0,0 +1,40 @@
+12/09/2025 14:03:24 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 14:03:24 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 14:03:24 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 14:03:24 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9731.56it/s]
+12/09/2025 14:03:25 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 14:03:25 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 14:03:25 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 14:03:25 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 14:03:25 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:03:25 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 14:03:40 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:03:40 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:03:43 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 14:03:43 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 14:03:43 - INFO - __main__ - Wan backbone lr = 0.000030 (base_lr * 0.1)
+12/09/2025 14:03:43 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 14:03:43 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 14:03:43 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 14:03:43 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 14:03:43 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 14:03:43 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 14:03:43 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 14:03:43 - INFO - __main__ - Dataloader configuration:
+12/09/2025 14:03:43 - INFO - __main__ - - num_workers: 4 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 14:03:43 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 14:03:43 - INFO - __main__ - - persistent_workers: True
+12/09/2025 14:03:43 - INFO - __main__ - - pin_memory: True
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1675, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1255, in main
+ num_training_steps=args.max_train_steps * accelerator.num_processes,
+TypeError: unsupported operand type(s) for *: 'NoneType' and 'int'
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1675, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1255, in main
+[rank0]: num_training_steps=args.max_train_steps * accelerator.num_processes,
+[rank0]: TypeError: unsupported operand type(s) for *: 'NoneType' and 'int'
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/files/requirements.txt b/Meissonic/wandb/run-20251209_140322-otx47txf/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b79eb8ecbf1f91854938d44f842652ced559903
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-metadata.json
@@ -0,0 +1,102 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T14:03:22.803470Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093762334720"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "p6u11ug9znfnhceiv1d6rhmfrzlq1mla"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-summary.json b/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba532d630a2f6e9086432ef98a4e1304e8ad3f55
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":19,"_wandb":{"runtime":19}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-core.log b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..526c4ea90ec4e80077185b540c003af4257001db
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T14:03:22.872698559Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpylheuc_2/port-55260.txt","pid":55260,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T14:03:22.873176264Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":55260}
+{"time":"2025-12-09T14:03:22.873179108Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-55260-55565-1676773212/socket","Net":"unix"}}
+{"time":"2025-12-09T14:03:23.059133915Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T14:03:23.065070162Z","level":"INFO","msg":"handleInformInit: received","streamId":"otx47txf","id":"1(@)"}
+{"time":"2025-12-09T14:03:23.735180283Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"otx47txf","id":"1(@)"}
+{"time":"2025-12-09T14:03:43.812586417Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T14:03:43.812651667Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T14:03:43.81269649Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T14:03:43.812663556Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T14:03:43.812794162Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-55260-55565-1676773212/socket","Net":"unix"}}
+{"time":"2025-12-09T14:03:44.315715108Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T14:03:44.315733252Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T14:03:44.315742409Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-internal.log b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..3addcc7bfa6cdfe8b1e0e416ef71641a7de1a787
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T14:03:23.065199508Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T14:03:23.734948808Z","level":"INFO","msg":"stream: created new stream","id":"otx47txf"}
+{"time":"2025-12-09T14:03:23.735040885Z","level":"INFO","msg":"handler: started","stream_id":"otx47txf"}
+{"time":"2025-12-09T14:03:23.735172493Z","level":"INFO","msg":"stream: started","id":"otx47txf"}
+{"time":"2025-12-09T14:03:23.735189016Z","level":"INFO","msg":"writer: started","stream_id":"otx47txf"}
+{"time":"2025-12-09T14:03:23.735215661Z","level":"INFO","msg":"sender: started","stream_id":"otx47txf"}
+{"time":"2025-12-09T14:03:43.812647879Z","level":"INFO","msg":"stream: closing","id":"otx47txf"}
+{"time":"2025-12-09T14:03:44.162140778Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T14:03:44.312677483Z","level":"INFO","msg":"handler: closed","stream_id":"otx47txf"}
+{"time":"2025-12-09T14:03:44.312772008Z","level":"INFO","msg":"sender: closed","stream_id":"otx47txf"}
+{"time":"2025-12-09T14:03:44.312779439Z","level":"INFO","msg":"stream: closed","id":"otx47txf"}
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug.log b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8832f2b526253fccac140c2bf52a432b3fb5845b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_setup.py:_flush():80] Configure stats pid to 55260
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug.log
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_140322-otx47txf/logs/debug-internal.log
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_init.py:init():841] calling init triggers
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 14:03:22,806 INFO MainThread:55260 [wandb_init.py:init():889] starting backend
+2025-12-09 14:03:23,059 INFO MainThread:55260 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 14:03:23,063 INFO MainThread:55260 [wandb_init.py:init():900] backend started and connected
+2025-12-09 14:03:23,064 INFO MainThread:55260 [wandb_init.py:init():970] updated telemetry
+2025-12-09 14:03:23,069 INFO MainThread:55260 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 14:03:24,156 INFO MainThread:55260 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 14:03:24,279 INFO MainThread:55260 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 14:03:24,280 INFO MainThread:55260 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 14:03:24,280 INFO MainThread:55260 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 14:03:24,280 INFO MainThread:55260 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 14:03:24,282 INFO MainThread:55260 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 14:03:24,283 INFO MainThread:55260 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': None, 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': None, 'instance_data_image': None, 'use_8bit_adam': False, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': 2, 'allow_tf32': False, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': 'muse_training', 'seed': None, 'logging_dir': 'logs', 'max_train_steps': None, 'checkpointing_steps': 500, 'logging_steps': 50, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 500, 'validation_steps': 100, 'mixed_precision': None, 'report_to': 'wandb', 'validation_prompts': None, 'resolution': 512, 'split_vae_encode': None, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.0, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': False, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 480, 'video_width': 848, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.1, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 14:03:43,812 INFO wandb-AsyncioManager-main:55260 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 14:03:43,812 INFO wandb-AsyncioManager-main:55260 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_140322-otx47txf/run-otx47txf.wandb b/Meissonic/wandb/run-20251209_140322-otx47txf/run-otx47txf.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..612540832748e52d1bd1fe3b6b0c9fd7c5410813
Binary files /dev/null and b/Meissonic/wandb/run-20251209_140322-otx47txf/run-otx47txf.wandb differ
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/config.yaml b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0576510c9410a5424ae3c5268c540d2d0943ace3
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/config.yaml
@@ -0,0 +1,245 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ cau9z7jh4phx1m0wq4r3ub8xd0sumfyz:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093762543616"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T14:08:07.626312Z"
+ writerId: cau9z7jh4phx1m0wq4r3ub8xd0sumfyz
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: false
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0
+dataloader_num_workers:
+ value: 4
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: false
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: null
+instance_data_image:
+ value: null
+instance_dataset:
+ value: null
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 50
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 500
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: null
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: null
+num_frames:
+ value: 16
+output_dir:
+ value: muse_training
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: null
+split_vae_encode:
+ value: null
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: false
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value: null
+validation_steps:
+ value: 100
+variant:
+ value: null
+video_height:
+ value: 480
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 848
+wan_backbone_lr_ratio:
+ value: 0.1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/output.log b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..185a699b5650a82a33df26db911f0279ca01eaae
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/output.log
@@ -0,0 +1,278 @@
+12/09/2025 14:08:08 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 14:08:08 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 14:08:08 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 14:08:08 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9721.90it/s]
+12/09/2025 14:08:09 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 14:08:09 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 14:08:09 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 14:08:09 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 14:08:09 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:08:09 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 14:08:25 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:08:25 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:08:27 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 14:08:27 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 14:08:27 - INFO - __main__ - Wan backbone lr = 0.000030 (base_lr * 0.1)
+12/09/2025 14:08:27 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 14:08:27 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 14:08:27 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 14:08:27 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 14:08:27 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 14:08:27 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 14:08:27 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 14:08:27 - INFO - __main__ - Dataloader configuration:
+12/09/2025 14:08:27 - INFO - __main__ - - num_workers: 4 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 14:08:27 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 14:08:27 - INFO - __main__ - - persistent_workers: True
+12/09/2025 14:08:27 - INFO - __main__ - - pin_memory: True
+12/09/2025 14:08:27 - WARNING - __main__ - max_train_steps not specified, defaulting to 1 epoch (8 steps)
+12/09/2025 14:08:27 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 14:08:30 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 14:08:30 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 14:08:30 - INFO - __main__ - ***** Running training *****
+12/09/2025 14:08:30 - INFO - __main__ - Num training steps = 8
+12/09/2025 14:08:30 - INFO - __main__ - Instantaneous batch size per device = 16
+12/09/2025 14:08:30 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 128
+12/09/2025 14:08:30 - INFO - __main__ - Gradient Accumulation steps = 1
+/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/compile_fx.py:312: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+ warnings.warn(
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1685, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1466, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1026, in forward
+ out_list = self.backbone(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in torch_dynamo_resume_in_forward_at_471
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 837, in call_wrapped
+ return self._wrapped_call(self, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 413, in __call__
+ raise e
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 400, in __call__
+ return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File ".140", line 33, in forward
+ submod_1 = self.compiled_submod_1(l_context_, l_self_modules_cross_attn_modules_k_parameters_weight_, l_self_modules_cross_attn_modules_k_parameters_bias_, l_self_modules_cross_attn_modules_norm_k_parameters_weight_, l_self_modules_cross_attn_modules_v_parameters_weight_, l_self_modules_cross_attn_modules_v_parameters_bias_, getitem, l_self_modules_cross_attn_modules_o_parameters_weight_, l_self_modules_cross_attn_modules_o_parameters_bias_, getitem_1, l_e_3_, l_e_4_, l_e_5_); l_context_ = l_self_modules_cross_attn_modules_k_parameters_weight_ = l_self_modules_cross_attn_modules_k_parameters_bias_ = l_self_modules_cross_attn_modules_norm_k_parameters_weight_ = l_self_modules_cross_attn_modules_v_parameters_weight_ = l_self_modules_cross_attn_modules_v_parameters_bias_ = getitem = l_self_modules_cross_attn_modules_o_parameters_weight_ = l_self_modules_cross_attn_modules_o_parameters_bias_ = getitem_1 = l_e_3_ = l_e_4_ = l_e_5_ = None
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 213, in forward
+ x = self.submod(*args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1130, in forward
+ return compiled_fn(full_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 339, in runtime_wrapper
+ all_outs = call_func_at_runtime_with_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+ out = normalize_as_list(f(args))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 103, in g
+ return f(*args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply
+ return super().apply(*args, **kwargs) # type: ignore[misc]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2118, in forward
+ fw_outs = call_func_at_runtime_with_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+ out = normalize_as_list(f(args))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 629, in wrapper
+ return compiled_fn(runtime_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 526, in wrapper
+ return compiled_fn(runtime_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 724, in inner_fn
+ outs = compiled_fn(args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 613, in __call__
+ return self.current_callable(inputs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3017, in run
+ out = model(new_inputs)
+ File "/opt/dlami/nvme/tmp_user/torchinductor_ubuntu/i4/ci4fv7peoif3y74vjxhcaw2jns2xsolgu3lq4o3qolhl2zs34f5j.py", line 543, in call
+ triton_red_fused_add_addmm_mul_native_layer_norm_squeeze_view_5.run(buf16, buf20, primals_10, primals_9, primals_12, primals_11, buf17, buf21, 127200, 1536, stream=stream0)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1272, in run
+ self.autotune_to_one_config(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1048, in autotune_to_one_config
+ timings = self.benchmark_all_configs(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1022, in benchmark_all_configs
+ timings = {
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1023, in
+ launcher: self.bench(launcher, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 891, in bench
+ return benchmarker.benchmark_gpu(kernel_call, rep=40)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 39, in wrapper
+ return fn(self, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 250, in benchmark_gpu
+ _callable()
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 856, in kernel_call
+ cloned_args, cloned_kwargs = self.maybe_clone_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 994, in maybe_clone_args
+ cloned_args = [
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 995, in
+ prepare_arg(name, arg)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 990, in prepare_arg
+ return clone_preserve_strides(arg)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3037, in clone_preserve_strides
+ buffer = torch.as_strided(x, (needed_size,), (1,)).clone()
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 746.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 136.62 MiB is free. Process 58677 has 414.00 MiB memory in use. Process 58676 has 414.00 MiB memory in use. Process 58678 has 414.00 MiB memory in use. Process 58682 has 414.00 MiB memory in use. Process 58679 has 414.00 MiB memory in use. Process 58681 has 414.00 MiB memory in use. Process 58680 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 36.48 GiB memory in use. Of the allocated memory 34.81 GiB is allocated by PyTorch, and 592.31 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1685, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1466, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1026, in forward
+[rank0]: out_list = self.backbone(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in torch_dynamo_resume_in_forward_at_471
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 837, in call_wrapped
+[rank0]: return self._wrapped_call(self, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 413, in __call__
+[rank0]: raise e
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 400, in __call__
+[rank0]: return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File ".140", line 33, in forward
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 213, in forward
+[rank0]: x = self.submod(*args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1130, in forward
+[rank0]: return compiled_fn(full_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 339, in runtime_wrapper
+[rank0]: all_outs = call_func_at_runtime_with_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+[rank0]: out = normalize_as_list(f(args))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 103, in g
+[rank0]: return f(*args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply
+[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2118, in forward
+[rank0]: fw_outs = call_func_at_runtime_with_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+[rank0]: out = normalize_as_list(f(args))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 629, in wrapper
+[rank0]: return compiled_fn(runtime_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 526, in wrapper
+[rank0]: return compiled_fn(runtime_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 724, in inner_fn
+[rank0]: outs = compiled_fn(args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 613, in __call__
+[rank0]: return self.current_callable(inputs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3017, in run
+[rank0]: out = model(new_inputs)
+[rank0]: File "/opt/dlami/nvme/tmp_user/torchinductor_ubuntu/i4/ci4fv7peoif3y74vjxhcaw2jns2xsolgu3lq4o3qolhl2zs34f5j.py", line 543, in call
+[rank0]: triton_red_fused_add_addmm_mul_native_layer_norm_squeeze_view_5.run(buf16, buf20, primals_10, primals_9, primals_12, primals_11, buf17, buf21, 127200, 1536, stream=stream0)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1272, in run
+[rank0]: self.autotune_to_one_config(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1048, in autotune_to_one_config
+[rank0]: timings = self.benchmark_all_configs(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1022, in benchmark_all_configs
+[rank0]: timings = {
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1023, in
+[rank0]: launcher: self.bench(launcher, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 891, in bench
+[rank0]: return benchmarker.benchmark_gpu(kernel_call, rep=40)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 39, in wrapper
+[rank0]: return fn(self, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 250, in benchmark_gpu
+[rank0]: _callable()
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 856, in kernel_call
+[rank0]: cloned_args, cloned_kwargs = self.maybe_clone_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 994, in maybe_clone_args
+[rank0]: cloned_args = [
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 995, in
+[rank0]: prepare_arg(name, arg)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 990, in prepare_arg
+[rank0]: return clone_preserve_strides(arg)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3037, in clone_preserve_strides
+[rank0]: buffer = torch.as_strided(x, (needed_size,), (1,)).clone()
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 746.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 136.62 MiB is free. Process 58677 has 414.00 MiB memory in use. Process 58676 has 414.00 MiB memory in use. Process 58678 has 414.00 MiB memory in use. Process 58682 has 414.00 MiB memory in use. Process 58679 has 414.00 MiB memory in use. Process 58681 has 414.00 MiB memory in use. Process 58680 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 36.48 GiB memory in use. Of the allocated memory 34.81 GiB is allocated by PyTorch, and 592.31 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/requirements.txt b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..f2c3ea882d8ee13561ef4b36a94f07cdb4ca3e8e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-metadata.json
@@ -0,0 +1,102 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T14:08:07.626312Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093762543616"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "cau9z7jh4phx1m0wq4r3ub8xd0sumfyz"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-summary.json b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..65efeb016520ebca12e5019029f5a6f6602702bd
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":73},"_runtime":73}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-core.log b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..ca37eb728d9fd18b80ee4aa5b2c62bfc0424a665
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T14:08:07.696352758Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpjw7ydas_/port-58675.txt","pid":58675,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T14:08:07.696873887Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":58675}
+{"time":"2025-12-09T14:08:07.696843305Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-58675-58965-572711673/socket","Net":"unix"}}
+{"time":"2025-12-09T14:08:07.882459638Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T14:08:07.889241598Z","level":"INFO","msg":"handleInformInit: received","streamId":"2xc2ptdw","id":"1(@)"}
+{"time":"2025-12-09T14:08:08.583749275Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2xc2ptdw","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.287327877Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.287407746Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T14:09:22.287397988Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.287501059Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.287501873Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-58675-58965-572711673/socket","Net":"unix"}}
+{"time":"2025-12-09T14:09:22.651880493Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.651899801Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T14:09:22.651909159Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-internal.log b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..5e3e99b5e69e38a503db48f6ff147cd303e2c190
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T14:08:07.889337777Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T14:08:08.583510509Z","level":"INFO","msg":"stream: created new stream","id":"2xc2ptdw"}
+{"time":"2025-12-09T14:08:08.583604343Z","level":"INFO","msg":"handler: started","stream_id":"2xc2ptdw"}
+{"time":"2025-12-09T14:08:08.583742797Z","level":"INFO","msg":"stream: started","id":"2xc2ptdw"}
+{"time":"2025-12-09T14:08:08.583756217Z","level":"INFO","msg":"writer: started","stream_id":"2xc2ptdw"}
+{"time":"2025-12-09T14:08:08.583763508Z","level":"INFO","msg":"sender: started","stream_id":"2xc2ptdw"}
+{"time":"2025-12-09T14:09:22.287406211Z","level":"INFO","msg":"stream: closing","id":"2xc2ptdw"}
+{"time":"2025-12-09T14:09:22.52809835Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T14:09:22.648874259Z","level":"INFO","msg":"handler: closed","stream_id":"2xc2ptdw"}
+{"time":"2025-12-09T14:09:22.648958362Z","level":"INFO","msg":"sender: closed","stream_id":"2xc2ptdw"}
+{"time":"2025-12-09T14:09:22.648969064Z","level":"INFO","msg":"stream: closed","id":"2xc2ptdw"}
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug.log b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..e64e60f8226a04d8ea3bc535cbe8eb19a1ec509b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug.log
@@ -0,0 +1,37 @@
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_setup.py:_flush():80] Configure stats pid to 58675
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug.log
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_140807-2xc2ptdw/logs/debug-internal.log
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_init.py:init():841] calling init triggers
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 14:08:07,629 INFO MainThread:58675 [wandb_init.py:init():889] starting backend
+2025-12-09 14:08:07,882 INFO MainThread:58675 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 14:08:07,887 INFO MainThread:58675 [wandb_init.py:init():900] backend started and connected
+2025-12-09 14:08:07,890 INFO MainThread:58675 [wandb_init.py:init():970] updated telemetry
+2025-12-09 14:08:07,895 INFO MainThread:58675 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 14:08:08,800 INFO MainThread:58675 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 14:08:08,923 INFO MainThread:58675 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 14:08:08,923 INFO MainThread:58675 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 14:08:08,923 INFO MainThread:58675 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 14:08:08,923 INFO MainThread:58675 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 14:08:08,926 INFO MainThread:58675 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 14:08:08,927 INFO MainThread:58675 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': None, 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': None, 'instance_data_image': None, 'use_8bit_adam': False, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': 2, 'allow_tf32': False, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': 'muse_training', 'seed': None, 'logging_dir': 'logs', 'max_train_steps': None, 'checkpointing_steps': 500, 'logging_steps': 50, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 500, 'validation_steps': 100, 'mixed_precision': None, 'report_to': 'wandb', 'validation_prompts': None, 'resolution': 512, 'split_vae_encode': None, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.0, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': False, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 480, 'video_width': 848, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.1, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 14:09:22,287 INFO wandb-AsyncioManager-main:58675 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 14:09:22,287 INFO wandb-AsyncioManager-main:58675 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
+2025-12-09 14:09:23,401 ERROR wandb-AsyncioManager-main:58675 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback.
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions
+ await fn()
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 38, in publish
+ await self._send_server_request(request)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/wandb/sdk/lib/service/service_client.py", line 64, in _send_server_request
+ await self._writer.drain()
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/asyncio/streams.py", line 371, in drain
+ await self._protocol._drain_helper()
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/asyncio/streams.py", line 167, in _drain_helper
+ raise ConnectionResetError('Connection lost')
+ConnectionResetError: Connection lost
diff --git a/Meissonic/wandb/run-20251209_140807-2xc2ptdw/run-2xc2ptdw.wandb b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/run-2xc2ptdw.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a7dbcc1e4fbd4089384dce1cc03b9ec8ecc399a2
Binary files /dev/null and b/Meissonic/wandb/run-20251209_140807-2xc2ptdw/run-2xc2ptdw.wandb differ
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/config.yaml b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..cb42651aa401f0850266b3e4c0e47227d88c3701
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/config.yaml
@@ -0,0 +1,245 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 6yeky5r0m22mgji4j68ozyjfe5hn9j2a:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093762883584"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T14:14:05.817929Z"
+ writerId: 6yeky5r0m22mgji4j68ozyjfe5hn9j2a
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: false
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0
+dataloader_num_workers:
+ value: 4
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: false
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: null
+instance_data_image:
+ value: null
+instance_dataset:
+ value: null
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 50
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 500
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: null
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: null
+num_frames:
+ value: 16
+output_dir:
+ value: muse_training
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: null
+split_vae_encode:
+ value: null
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 16
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: false
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value: null
+validation_steps:
+ value: 100
+variant:
+ value: null
+video_height:
+ value: 480
+video_tokenizer_model_id:
+ value: Cosmos-1.0-Tokenizer-DV8x16x16
+video_width:
+ value: 848
+wan_backbone_lr_ratio:
+ value: 0.1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/output.log b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..f4fc892ca73a17eb7c5ca6e2663edb3a0da306ea
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/output.log
@@ -0,0 +1,277 @@
+12/09/2025 14:14:07 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 14:14:07 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 14:14:07 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 14:14:07 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9855.70it/s]
+12/09/2025 14:14:07 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 14:14:07 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 14:14:07 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 14:14:07 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 14:14:07 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:14:08 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 14:14:24 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:14:24 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:14:26 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 14:14:26 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 14:14:26 - INFO - __main__ - Wan backbone lr = 0.000030 (base_lr * 0.1)
+12/09/2025 14:14:26 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 14:14:26 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 14:14:26 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 14:14:26 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 14:14:26 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 14:14:26 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 14:14:26 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 14:14:26 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/09/2025 14:14:26 - INFO - __main__ - Dataloader configuration:
+12/09/2025 14:14:26 - INFO - __main__ - - num_workers: 4 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 14:14:26 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 14:14:26 - INFO - __main__ - - persistent_workers: True
+12/09/2025 14:14:26 - INFO - __main__ - - pin_memory: True
+12/09/2025 14:14:26 - WARNING - __main__ - max_train_steps not specified, defaulting to 1 epoch (8 steps)
+12/09/2025 14:14:26 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 14:14:28 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 14:14:28 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 14:14:28 - INFO - __main__ - ***** Running training *****
+12/09/2025 14:14:28 - INFO - __main__ - Num training steps = 8
+12/09/2025 14:14:28 - INFO - __main__ - Instantaneous batch size per device = 16
+12/09/2025 14:14:28 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 128
+12/09/2025 14:14:28 - INFO - __main__ - Gradient Accumulation steps = 1
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1498, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1026, in forward
+ out_list = self.backbone(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in torch_dynamo_resume_in_forward_at_471
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 837, in call_wrapped
+ return self._wrapped_call(self, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 413, in __call__
+ raise e
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 400, in __call__
+ return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File ".138", line 33, in forward
+ submod_1 = self.compiled_submod_1(l_context_, l_self_modules_cross_attn_modules_k_parameters_weight_, l_self_modules_cross_attn_modules_k_parameters_bias_, l_self_modules_cross_attn_modules_norm_k_parameters_weight_, l_self_modules_cross_attn_modules_v_parameters_weight_, l_self_modules_cross_attn_modules_v_parameters_bias_, getitem, l_self_modules_cross_attn_modules_o_parameters_weight_, l_self_modules_cross_attn_modules_o_parameters_bias_, getitem_1, l_e_3_, l_e_4_, l_e_5_); l_context_ = l_self_modules_cross_attn_modules_k_parameters_weight_ = l_self_modules_cross_attn_modules_k_parameters_bias_ = l_self_modules_cross_attn_modules_norm_k_parameters_weight_ = l_self_modules_cross_attn_modules_v_parameters_weight_ = l_self_modules_cross_attn_modules_v_parameters_bias_ = getitem = l_self_modules_cross_attn_modules_o_parameters_weight_ = l_self_modules_cross_attn_modules_o_parameters_bias_ = getitem_1 = l_e_3_ = l_e_4_ = l_e_5_ = None
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 213, in forward
+ x = self.submod(*args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1130, in forward
+ return compiled_fn(full_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 339, in runtime_wrapper
+ all_outs = call_func_at_runtime_with_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+ out = normalize_as_list(f(args))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 103, in g
+ return f(*args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply
+ return super().apply(*args, **kwargs) # type: ignore[misc]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2118, in forward
+ fw_outs = call_func_at_runtime_with_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+ out = normalize_as_list(f(args))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 629, in wrapper
+ return compiled_fn(runtime_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 526, in wrapper
+ return compiled_fn(runtime_args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 724, in inner_fn
+ outs = compiled_fn(args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 613, in __call__
+ return self.current_callable(inputs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3017, in run
+ out = model(new_inputs)
+ File "/opt/dlami/nvme/tmp_user/torchinductor_ubuntu/i4/ci4fv7peoif3y74vjxhcaw2jns2xsolgu3lq4o3qolhl2zs34f5j.py", line 543, in call
+ triton_red_fused_add_addmm_mul_native_layer_norm_squeeze_view_5.run(buf16, buf20, primals_10, primals_9, primals_12, primals_11, buf17, buf21, 127200, 1536, stream=stream0)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1272, in run
+ self.autotune_to_one_config(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1048, in autotune_to_one_config
+ timings = self.benchmark_all_configs(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1022, in benchmark_all_configs
+ timings = {
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1023, in
+ launcher: self.bench(launcher, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 891, in bench
+ return benchmarker.benchmark_gpu(kernel_call, rep=40)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 39, in wrapper
+ return fn(self, *args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 250, in benchmark_gpu
+ _callable()
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 856, in kernel_call
+ cloned_args, cloned_kwargs = self.maybe_clone_args(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 994, in maybe_clone_args
+ cloned_args = [
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 995, in
+ prepare_arg(name, arg)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 990, in prepare_arg
+ return clone_preserve_strides(arg)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3037, in clone_preserve_strides
+ buffer = torch.as_strided(x, (needed_size,), (1,)).clone()
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 746.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 136.62 MiB is free. Process 68014 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 36.48 GiB memory in use. Process 68012 has 414.00 MiB memory in use. Process 68011 has 414.00 MiB memory in use. Process 68013 has 414.00 MiB memory in use. Process 68016 has 414.00 MiB memory in use. Process 68010 has 414.00 MiB memory in use. Process 68015 has 414.00 MiB memory in use. Of the allocated memory 34.81 GiB is allocated by PyTorch, and 592.31 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1498, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1026, in forward
+[rank0]: out_list = self.backbone(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in torch_dynamo_resume_in_forward_at_471
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 837, in call_wrapped
+[rank0]: return self._wrapped_call(self, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 413, in __call__
+[rank0]: raise e
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/fx/graph_module.py", line 400, in __call__
+[rank0]: return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File ".138", line 33, in forward
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/backends/distributed.py", line 213, in forward
+[rank0]: x = self.submod(*args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/aot_autograd.py", line 1130, in forward
+[rank0]: return compiled_fn(full_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 339, in runtime_wrapper
+[rank0]: all_outs = call_func_at_runtime_with_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+[rank0]: out = normalize_as_list(f(args))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 103, in g
+[rank0]: return f(*args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply
+[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2118, in forward
+[rank0]: fw_outs = call_func_at_runtime_with_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
+[rank0]: out = normalize_as_list(f(args))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 629, in wrapper
+[rank0]: return compiled_fn(runtime_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 526, in wrapper
+[rank0]: return compiled_fn(runtime_args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 724, in inner_fn
+[rank0]: outs = compiled_fn(args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 613, in __call__
+[rank0]: return self.current_callable(inputs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3017, in run
+[rank0]: out = model(new_inputs)
+[rank0]: File "/opt/dlami/nvme/tmp_user/torchinductor_ubuntu/i4/ci4fv7peoif3y74vjxhcaw2jns2xsolgu3lq4o3qolhl2zs34f5j.py", line 543, in call
+[rank0]: triton_red_fused_add_addmm_mul_native_layer_norm_squeeze_view_5.run(buf16, buf20, primals_10, primals_9, primals_12, primals_11, buf17, buf21, 127200, 1536, stream=stream0)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1272, in run
+[rank0]: self.autotune_to_one_config(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1048, in autotune_to_one_config
+[rank0]: timings = self.benchmark_all_configs(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1022, in benchmark_all_configs
+[rank0]: timings = {
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1023, in
+[rank0]: launcher: self.bench(launcher, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 891, in bench
+[rank0]: return benchmarker.benchmark_gpu(kernel_call, rep=40)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 39, in wrapper
+[rank0]: return fn(self, *args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/benchmarking.py", line 250, in benchmark_gpu
+[rank0]: _callable()
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 856, in kernel_call
+[rank0]: cloned_args, cloned_kwargs = self.maybe_clone_args(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 994, in maybe_clone_args
+[rank0]: cloned_args = [
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 995, in
+[rank0]: prepare_arg(name, arg)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 990, in prepare_arg
+[rank0]: return clone_preserve_strides(arg)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3037, in clone_preserve_strides
+[rank0]: buffer = torch.as_strided(x, (needed_size,), (1,)).clone()
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 746.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 136.62 MiB is free. Process 68014 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 36.48 GiB memory in use. Process 68012 has 414.00 MiB memory in use. Process 68011 has 414.00 MiB memory in use. Process 68013 has 414.00 MiB memory in use. Process 68016 has 414.00 MiB memory in use. Process 68010 has 414.00 MiB memory in use. Process 68015 has 414.00 MiB memory in use. Of the allocated memory 34.81 GiB is allocated by PyTorch, and 592.31 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/requirements.txt b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..b744d390864719718398117fdd944e687d270574
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-metadata.json
@@ -0,0 +1,102 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T14:14:05.817929Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093762883584"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "6yeky5r0m22mgji4j68ozyjfe5hn9j2a"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-summary.json b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3e80f52165bcc0e5fdf4a7be18b9b5db42e53a8
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":57,"_wandb":{"runtime":57}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-core.log b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..e56c66ac1e53d8372b8b4b42c4d1577dada154b6
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T14:14:05.889317786Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpm9xxyxn_/port-68009.txt","pid":68009,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T14:14:05.88993588Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":68009}
+{"time":"2025-12-09T14:14:05.889907813Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-68009-68307-4265498081/socket","Net":"unix"}}
+{"time":"2025-12-09T14:14:06.073781124Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T14:14:06.079762596Z","level":"INFO","msg":"handleInformInit: received","streamId":"x4hogfbn","id":"1(@)"}
+{"time":"2025-12-09T14:14:06.847672609Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"x4hogfbn","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.374591532Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.374771022Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.374793745Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T14:15:04.374873463Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.374929441Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-68009-68307-4265498081/socket","Net":"unix"}}
+{"time":"2025-12-09T14:15:04.807072953Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.80709144Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T14:15:04.807100361Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-internal.log b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..58ec434b3bcd95996b8615c1f4ec0e30b9048346
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T14:14:06.079873033Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T14:14:06.847376958Z","level":"INFO","msg":"stream: created new stream","id":"x4hogfbn"}
+{"time":"2025-12-09T14:14:06.847456859Z","level":"INFO","msg":"handler: started","stream_id":"x4hogfbn"}
+{"time":"2025-12-09T14:14:06.847572206Z","level":"INFO","msg":"sender: started","stream_id":"x4hogfbn"}
+{"time":"2025-12-09T14:14:06.847560437Z","level":"INFO","msg":"writer: started","stream_id":"x4hogfbn"}
+{"time":"2025-12-09T14:14:06.847536307Z","level":"INFO","msg":"stream: started","id":"x4hogfbn"}
+{"time":"2025-12-09T14:15:04.374776579Z","level":"INFO","msg":"stream: closing","id":"x4hogfbn"}
+{"time":"2025-12-09T14:15:04.650892563Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T14:15:04.803844803Z","level":"INFO","msg":"handler: closed","stream_id":"x4hogfbn"}
+{"time":"2025-12-09T14:15:04.803956881Z","level":"INFO","msg":"sender: closed","stream_id":"x4hogfbn"}
+{"time":"2025-12-09T14:15:04.803963926Z","level":"INFO","msg":"stream: closed","id":"x4hogfbn"}
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug.log b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..f0bb32dc1987fbb207fd3380466826c47de1cb0c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_setup.py:_flush():80] Configure stats pid to 68009
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug.log
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_141405-x4hogfbn/logs/debug-internal.log
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_init.py:init():841] calling init triggers
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 14:14:05,820 INFO MainThread:68009 [wandb_init.py:init():889] starting backend
+2025-12-09 14:14:06,074 INFO MainThread:68009 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 14:14:06,078 INFO MainThread:68009 [wandb_init.py:init():900] backend started and connected
+2025-12-09 14:14:06,079 INFO MainThread:68009 [wandb_init.py:init():970] updated telemetry
+2025-12-09 14:14:06,083 INFO MainThread:68009 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 14:14:07,064 INFO MainThread:68009 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 14:14:07,191 INFO MainThread:68009 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 14:14:07,191 INFO MainThread:68009 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 14:14:07,191 INFO MainThread:68009 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 14:14:07,191 INFO MainThread:68009 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 14:14:07,194 INFO MainThread:68009 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 14:14:07,195 INFO MainThread:68009 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': None, 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': None, 'instance_data_image': None, 'use_8bit_adam': False, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': 2, 'allow_tf32': False, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': 'muse_training', 'seed': None, 'logging_dir': 'logs', 'max_train_steps': None, 'checkpointing_steps': 500, 'logging_steps': 50, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 16, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 500, 'validation_steps': 100, 'mixed_precision': None, 'report_to': 'wandb', 'validation_prompts': None, 'resolution': 512, 'split_vae_encode': None, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.0, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': False, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 480, 'video_width': 848, 'video_tokenizer_model_id': 'Cosmos-1.0-Tokenizer-DV8x16x16', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.1, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 14:15:04,374 INFO wandb-AsyncioManager-main:68009 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 14:15:04,375 INFO wandb-AsyncioManager-main:68009 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_141405-x4hogfbn/run-x4hogfbn.wandb b/Meissonic/wandb/run-20251209_141405-x4hogfbn/run-x4hogfbn.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..1cc51377dcf97bf04b6719c301b1468d44156774
Binary files /dev/null and b/Meissonic/wandb/run-20251209_141405-x4hogfbn/run-x4hogfbn.wandb differ
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/config.yaml b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..895cfa4ed779ada88f03bac12ccc4bdbc12dd427
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/config.yaml
@@ -0,0 +1,301 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ kb0bzb40td6ovt07v98pjfl246wmk2wf:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12093763047424"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T14:16:18.415636Z"
+ writerId: kb0bzb40td6ovt07v98pjfl246wmk2wf
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/output.log b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..5447bc5f8b12927544d46288ec6fc82a8bec1b70
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/output.log
@@ -0,0 +1,79 @@
+12/09/2025 14:16:19 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 14:16:19 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 14:16:19 - INFO - __main__ - Loading minimal video tokenizer config to get mask_token_id and codebook_size...
+12/09/2025 14:16:19 - INFO - __main__ - Loading video tokenizer temporarily to get mask_token_id and codebook_size...
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9362.29it/s]
+12/09/2025 14:16:20 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 14:16:20 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 14:16:20 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 14:16:20 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 14:16:20 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:16:20 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 14:16:37 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:16:37 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:16:39 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 14:16:41 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 14:16:41 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 14:16:41 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 14:16:41 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 14:16:41 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 14:16:41 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 14:16:41 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 14:16:41 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 14:16:41 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 14:16:41 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/09/2025 14:16:41 - INFO - __main__ - Dataloader configuration:
+12/09/2025 14:16:41 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 14:16:41 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 14:16:41 - INFO - __main__ - - persistent_workers: True
+12/09/2025 14:16:41 - INFO - __main__ - - pin_memory: True
+12/09/2025 14:16:41 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 14:16:43 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 14:16:43 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 14:16:43 - INFO - __main__ - ***** Running training *****
+12/09/2025 14:16:43 - INFO - __main__ - Num training steps = 10000
+12/09/2025 14:16:43 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 14:16:43 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 8
+12/09/2025 14:16:43 - INFO - __main__ - Gradient Accumulation steps = 1
+12/09/2025 14:16:44 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1534, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 7.23 GiB is free. Process 70750 has 414.00 MiB memory in use. Process 70753 has 414.00 MiB memory in use. Process 70749 has 414.00 MiB memory in use. Process 70754 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 29.39 GiB memory in use. Process 70752 has 414.00 MiB memory in use. Process 70748 has 414.00 MiB memory in use. Process 70751 has 414.00 MiB memory in use. Of the allocated memory 28.07 GiB is allocated by PyTorch, and 234.73 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1534, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.58 GiB. GPU 0 has a total capacity of 39.49 GiB of which 7.23 GiB is free. Process 70750 has 414.00 MiB memory in use. Process 70753 has 414.00 MiB memory in use. Process 70749 has 414.00 MiB memory in use. Process 70754 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 29.39 GiB memory in use. Process 70752 has 414.00 MiB memory in use. Process 70748 has 414.00 MiB memory in use. Process 70751 has 414.00 MiB memory in use. Of the allocated memory 28.07 GiB is allocated by PyTorch, and 234.73 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/requirements.txt b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..00478088d19afd60c3b885136e8859d8ff427a49
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T14:16:18.415636Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093763047424"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "kb0bzb40td6ovt07v98pjfl246wmk2wf"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-summary.json b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..8e8b56dc1e46b00cf7f7377c59e01f228d41fe3f
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":62,"_wandb":{"runtime":62}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-core.log b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..74662a552ddef4b9b28334a85abbc20fc8b10d58
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T14:16:18.48416748Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp1m1_z9x0/port-70747.txt","pid":70747,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T14:16:18.48459797Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":70747}
+{"time":"2025-12-09T14:16:18.484606503Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-70747-71068-298534146/socket","Net":"unix"}}
+{"time":"2025-12-09T14:16:18.671246579Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T14:16:18.677309787Z","level":"INFO","msg":"handleInformInit: received","streamId":"v5rp4n5k","id":"1(@)"}
+{"time":"2025-12-09T14:16:19.246997822Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"v5rp4n5k","id":"1(@)"}
+{"time":"2025-12-09T14:17:21.807072348Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T14:17:21.807124574Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T14:17:21.807121112Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T14:17:21.80732553Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-70747-71068-298534146/socket","Net":"unix"}}
+{"time":"2025-12-09T14:17:21.807357877Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T14:17:22.230073379Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T14:17:22.230098436Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T14:17:22.23010832Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-internal.log b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..3f9845251800ac24a906597a7a46535a1ad1624e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T14:16:18.677456814Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T14:16:19.246695362Z","level":"INFO","msg":"stream: created new stream","id":"v5rp4n5k"}
+{"time":"2025-12-09T14:16:19.246790913Z","level":"INFO","msg":"handler: started","stream_id":"v5rp4n5k"}
+{"time":"2025-12-09T14:16:19.246950858Z","level":"INFO","msg":"writer: started","stream_id":"v5rp4n5k"}
+{"time":"2025-12-09T14:16:19.246919928Z","level":"INFO","msg":"stream: started","id":"v5rp4n5k"}
+{"time":"2025-12-09T14:16:19.246954337Z","level":"INFO","msg":"sender: started","stream_id":"v5rp4n5k"}
+{"time":"2025-12-09T14:17:21.807132231Z","level":"INFO","msg":"stream: closing","id":"v5rp4n5k"}
+{"time":"2025-12-09T14:17:22.084826724Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T14:17:22.226568945Z","level":"INFO","msg":"handler: closed","stream_id":"v5rp4n5k"}
+{"time":"2025-12-09T14:17:22.226683646Z","level":"INFO","msg":"sender: closed","stream_id":"v5rp4n5k"}
+{"time":"2025-12-09T14:17:22.226691201Z","level":"INFO","msg":"stream: closed","id":"v5rp4n5k"}
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug.log b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ec28e257839b1b934107d99c46c88719436fffd6
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_setup.py:_flush():80] Configure stats pid to 70747
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug.log
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_141618-v5rp4n5k/logs/debug-internal.log
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_init.py:init():841] calling init triggers
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 14:16:18,418 INFO MainThread:70747 [wandb_init.py:init():889] starting backend
+2025-12-09 14:16:18,671 INFO MainThread:70747 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 14:16:18,675 INFO MainThread:70747 [wandb_init.py:init():900] backend started and connected
+2025-12-09 14:16:18,677 INFO MainThread:70747 [wandb_init.py:init():970] updated telemetry
+2025-12-09 14:16:18,681 INFO MainThread:70747 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 14:16:19,460 INFO MainThread:70747 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 14:16:19,585 INFO MainThread:70747 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 14:16:19,585 INFO MainThread:70747 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 14:16:19,585 INFO MainThread:70747 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 14:16:19,585 INFO MainThread:70747 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 14:16:19,589 INFO MainThread:70747 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 14:16:19,589 INFO MainThread:70747 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 14:17:21,807 INFO wandb-AsyncioManager-main:70747 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 14:17:21,807 INFO wandb-AsyncioManager-main:70747 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_141618-v5rp4n5k/run-v5rp4n5k.wandb b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/run-v5rp4n5k.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..4cb0daf65fbfbfe5223fb1f4a2a95d8c12f2825a
Binary files /dev/null and b/Meissonic/wandb/run-20251209_141618-v5rp4n5k/run-v5rp4n5k.wandb differ
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_7b2c7dbea7c77c3a3523.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_7b2c7dbea7c77c3a3523.png
new file mode 100644
index 0000000000000000000000000000000000000000..fe9d0c71e1bb9e1427dd51b01336e7555f94cfd1
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_7b2c7dbea7c77c3a3523.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b2c7dbea7c77c3a35238f0fb5e799c84a960eb7fb86424199551945ee4bd12f
+size 142225
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_d3b01b8e129b539a85ed.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_d3b01b8e129b539a85ed.png
new file mode 100644
index 0000000000000000000000000000000000000000..4a80cb85e634b71c019cb30a98026cef72b6de7a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1000_d3b01b8e129b539a85ed.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b01b8e129b539a85ed3a366f5dda9d8a2ddeeaf4c86fdf85666b1fed702412
+size 151513
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_287117d5d7643ba31ec4.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_287117d5d7643ba31ec4.png
new file mode 100644
index 0000000000000000000000000000000000000000..446ff5a0efc460653a3ba2816a1aba0f75f004a8
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_287117d5d7643ba31ec4.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287117d5d7643ba31ec4edb4220888e3e3039632e354ab390fc18b48722eeabe
+size 144805
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_f6b18ba278e34d44baab.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_f6b18ba278e34d44baab.png
new file mode 100644
index 0000000000000000000000000000000000000000..d7280ec4c28cbe24e7c9af972e31ea0de2755f94
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_1500_f6b18ba278e34d44baab.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6b18ba278e34d44baabc384a855c5179e8a1bada0a2a0278bdeb1ca432f1ac5
+size 153161
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_321720abba124381620b.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_321720abba124381620b.png
new file mode 100644
index 0000000000000000000000000000000000000000..062e91e05b7b5d2f115315e2a3cc6fd2273eebcb
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_321720abba124381620b.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:321720abba124381620b47e9787cba8eb803ad1c4072172fd3800750374d7536
+size 142016
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_fa7af054654656754134.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_fa7af054654656754134.png
new file mode 100644
index 0000000000000000000000000000000000000000..9588194ccc88b3fb9f50423b2b9102d25120cf06
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2000_fa7af054654656754134.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa7af05465465675413476e517f43ff87b883cbf3a4bf2608c4081c4d58d8929
+size 153715
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_e6c1efef5a74bd11c582.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_e6c1efef5a74bd11c582.png
new file mode 100644
index 0000000000000000000000000000000000000000..3460543dcd7c86f9810c764bee6527374572eaa6
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_e6c1efef5a74bd11c582.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c1efef5a74bd11c5828c14cea7fa6bf3a30be1e7802808372a2a9a6dc70570
+size 146709
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_f00b3e2c752ac3cf926a.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_f00b3e2c752ac3cf926a.png
new file mode 100644
index 0000000000000000000000000000000000000000..9dea257094b6aa0266a742ee5054d82139931f79
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_2500_f00b3e2c752ac3cf926a.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f00b3e2c752ac3cf926abd1ea3f05edd8971039f15c2942bf0f1b0f461474d20
+size 139409
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_67d5ba7897e123897b95.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_67d5ba7897e123897b95.png
new file mode 100644
index 0000000000000000000000000000000000000000..b056ae0ee00d37c8788b9b287aeb3c3e4750a57c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_67d5ba7897e123897b95.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67d5ba7897e123897b95025d14df4dec47f70dce18f06b438e40a2d23849dba2
+size 149238
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_9c128d777c7dab549107.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_9c128d777c7dab549107.png
new file mode 100644
index 0000000000000000000000000000000000000000..cb792e41b902b2ff19190ae6ee9ec5107b3b2152
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3000_9c128d777c7dab549107.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c128d777c7dab5491071e2e6a43807bdb349d7bab1d8f8426085cc6619d8446
+size 141809
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_4274b237825ef8cf5d05.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_4274b237825ef8cf5d05.png
new file mode 100644
index 0000000000000000000000000000000000000000..43c10cec82a705101e7be098b137740f7bb33d3d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_4274b237825ef8cf5d05.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4274b237825ef8cf5d05dfb41e4ed36fd5fe68ed60a35d9c9a8f93e75a3e671a
+size 140719
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_de7aecbbb4729ab5af9d.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_de7aecbbb4729ab5af9d.png
new file mode 100644
index 0000000000000000000000000000000000000000..f6cdc3dcb18bc50f8fedb6536d2dcabcb5c7c203
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_3500_de7aecbbb4729ab5af9d.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de7aecbbb4729ab5af9d55510c945bfc145abcc757a87d55a726687e2aca4841
+size 145540
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_09fa45bbfff36049e141.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_09fa45bbfff36049e141.png
new file mode 100644
index 0000000000000000000000000000000000000000..39ad914a2ae6a66a4c753749ab5b532aa800f81c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_09fa45bbfff36049e141.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09fa45bbfff36049e1418e1c56d43cf7cde9f6e74b57269422bd9132aee4b135
+size 162875
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_d8fc778d368d5c2cb79c.png b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_d8fc778d368d5c2cb79c.png
new file mode 100644
index 0000000000000000000000000000000000000000..4279bc6a87471945e644d0aaa512d4e46b42424b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/media/images/generated_videos_first_frame_500_d8fc778d368d5c2cb79c.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8fc778d368d5c2cb79c7b208b465f91f0df0b3653679648d8f4ba8d3d0ebeaf
+size 172462
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/output.log b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..eb25380dc7f24da8c6f954dcd4815ef0c539d644
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/output.log
@@ -0,0 +1,569 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 47.74it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 14:17:43 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8237.97it/s]
+12/09/2025 14:17:44 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 14:17:44 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 14:17:44 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:17:45 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 14:18:01 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:18:01 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 14:18:03 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 14:18:04 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 14:18:04 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 14:18:04 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 14:18:04 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 14:18:11 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 14:18:11 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 14:18:11 - INFO - __main__ - Dataloader configuration:
+12/09/2025 14:18:11 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 14:18:11 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 14:18:11 - INFO - __main__ - - persistent_workers: True
+12/09/2025 14:18:11 - INFO - __main__ - - pin_memory: True
+12/09/2025 14:18:11 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 14:18:28 - INFO - __main__ - ***** Running training *****
+12/09/2025 14:18:28 - INFO - __main__ - Num training steps = 10000
+12/09/2025 14:18:28 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 14:18:28 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 8
+12/09/2025 14:18:28 - INFO - __main__ - Gradient Accumulation steps = 1
+12/09/2025 14:18:44 - INFO - __main__ - Step: 10 Loss: 11.0786 LR: 0.000300
+12/09/2025 14:18:52 - INFO - __main__ - Step: 20 Loss: 11.0749 LR: 0.000300
+12/09/2025 14:19:00 - INFO - __main__ - Step: 30 Loss: 11.0746 LR: 0.000300
+12/09/2025 14:19:07 - INFO - __main__ - Step: 40 Loss: 11.0719 LR: 0.000300
+12/09/2025 14:19:14 - INFO - __main__ - Step: 50 Loss: 11.0722 LR: 0.000300
+12/09/2025 14:19:22 - INFO - __main__ - Step: 60 Loss: 11.0700 LR: 0.000300
+12/09/2025 14:19:29 - INFO - __main__ - Step: 70 Loss: 11.0685 LR: 0.000300
+12/09/2025 14:19:37 - INFO - __main__ - Step: 80 Loss: 11.0656 LR: 0.000300
+12/09/2025 14:19:44 - INFO - __main__ - Step: 90 Loss: 11.0620 LR: 0.000300
+12/09/2025 14:19:52 - INFO - __main__ - Step: 100 Loss: 11.0578 LR: 0.000300
+12/09/2025 14:19:59 - INFO - __main__ - Step: 110 Loss: 11.0488 LR: 0.000300
+12/09/2025 14:20:07 - INFO - __main__ - Step: 120 Loss: 11.0397 LR: 0.000300
+12/09/2025 14:20:14 - INFO - __main__ - Step: 130 Loss: 11.0290 LR: 0.000300
+12/09/2025 14:20:22 - INFO - __main__ - Step: 140 Loss: 11.0018 LR: 0.000300
+12/09/2025 14:20:29 - INFO - __main__ - Step: 150 Loss: 10.9748 LR: 0.000300
+12/09/2025 14:20:36 - INFO - __main__ - Step: 160 Loss: 10.9484 LR: 0.000300
+12/09/2025 14:20:44 - INFO - __main__ - Step: 170 Loss: 10.9394 LR: 0.000300
+12/09/2025 14:20:51 - INFO - __main__ - Step: 180 Loss: 10.8992 LR: 0.000300
+12/09/2025 14:20:58 - INFO - __main__ - Step: 190 Loss: 10.8861 LR: 0.000300
+12/09/2025 14:21:06 - INFO - __main__ - Step: 200 Loss: 10.8595 LR: 0.000300
+12/09/2025 14:21:13 - INFO - __main__ - Step: 210 Loss: 10.8192 LR: 0.000300
+12/09/2025 14:21:20 - INFO - __main__ - Step: 220 Loss: 10.8125 LR: 0.000300
+12/09/2025 14:21:28 - INFO - __main__ - Step: 230 Loss: 10.8000 LR: 0.000300
+12/09/2025 14:21:36 - INFO - __main__ - Step: 240 Loss: 10.7867 LR: 0.000300
+12/09/2025 14:21:43 - INFO - __main__ - Step: 250 Loss: 10.7790 LR: 0.000300
+12/09/2025 14:21:51 - INFO - __main__ - Step: 260 Loss: 10.7543 LR: 0.000300
+12/09/2025 14:21:58 - INFO - __main__ - Step: 270 Loss: 10.7506 LR: 0.000300
+12/09/2025 14:22:06 - INFO - __main__ - Step: 280 Loss: 10.7433 LR: 0.000300
+12/09/2025 14:22:14 - INFO - __main__ - Step: 290 Loss: 10.7032 LR: 0.000300
+12/09/2025 14:22:21 - INFO - __main__ - Step: 300 Loss: 10.6494 LR: 0.000300
+12/09/2025 14:22:28 - INFO - __main__ - Step: 310 Loss: 10.6978 LR: 0.000300
+12/09/2025 14:22:36 - INFO - __main__ - Step: 320 Loss: 10.7025 LR: 0.000300
+12/09/2025 14:22:44 - INFO - __main__ - Step: 330 Loss: 10.6732 LR: 0.000300
+12/09/2025 14:22:51 - INFO - __main__ - Step: 340 Loss: 10.6721 LR: 0.000300
+12/09/2025 14:22:59 - INFO - __main__ - Step: 350 Loss: 10.6566 LR: 0.000300
+12/09/2025 14:23:07 - INFO - __main__ - Step: 360 Loss: 10.6685 LR: 0.000300
+12/09/2025 14:23:16 - INFO - __main__ - Step: 370 Loss: 10.5958 LR: 0.000300
+12/09/2025 14:23:23 - INFO - __main__ - Step: 380 Loss: 10.6540 LR: 0.000300
+12/09/2025 14:23:32 - INFO - __main__ - Step: 390 Loss: 10.6569 LR: 0.000300
+12/09/2025 14:23:40 - INFO - __main__ - Step: 400 Loss: 10.5789 LR: 0.000300
+12/09/2025 14:23:48 - INFO - __main__ - Step: 410 Loss: 10.5968 LR: 0.000300
+12/09/2025 14:23:57 - INFO - __main__ - Step: 420 Loss: 10.5736 LR: 0.000300
+12/09/2025 14:24:05 - INFO - __main__ - Step: 430 Loss: 10.6496 LR: 0.000300
+12/09/2025 14:24:13 - INFO - __main__ - Step: 440 Loss: 10.5696 LR: 0.000300
+12/09/2025 14:24:20 - INFO - __main__ - Step: 450 Loss: 10.5662 LR: 0.000300
+12/09/2025 14:24:28 - INFO - __main__ - Step: 460 Loss: 10.5439 LR: 0.000300
+12/09/2025 14:24:36 - INFO - __main__ - Step: 470 Loss: 10.5309 LR: 0.000300
+12/09/2025 14:24:45 - INFO - __main__ - Step: 480 Loss: 10.5726 LR: 0.000300
+12/09/2025 14:24:53 - INFO - __main__ - Step: 490 Loss: 10.5672 LR: 0.000300
+12/09/2025 14:25:02 - INFO - __main__ - Step: 500 Loss: 10.5413 LR: 0.000300
+12/09/2025 14:25:02 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500
+12/09/2025 14:25:11 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500/optimizer.bin
+12/09/2025 14:25:11 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500/scheduler.bin
+12/09/2025 14:25:11 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500/sampler.bin
+12/09/2025 14:25:11 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500/random_states_0.pkl
+12/09/2025 14:25:11 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-500
+12/09/2025 14:25:11 - INFO - __main__ - Generating videos for validation...
+12/09/2025 14:25:11 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.60it/s]
+12/09/2025 14:25:18 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 14:25:28 - INFO - __main__ - Step: 510 Loss: 10.5050 LR: 0.000300
+12/09/2025 14:25:36 - INFO - __main__ - Step: 520 Loss: 10.5404 LR: 0.000300
+12/09/2025 14:25:44 - INFO - __main__ - Step: 530 Loss: 10.5635 LR: 0.000300
+12/09/2025 14:25:52 - INFO - __main__ - Step: 540 Loss: 10.4973 LR: 0.000300
+12/09/2025 14:26:00 - INFO - __main__ - Step: 550 Loss: 10.4899 LR: 0.000300
+12/09/2025 14:26:08 - INFO - __main__ - Step: 560 Loss: 10.5921 LR: 0.000300
+12/09/2025 14:26:16 - INFO - __main__ - Step: 570 Loss: 10.5204 LR: 0.000300
+12/09/2025 14:26:24 - INFO - __main__ - Step: 580 Loss: 10.5032 LR: 0.000300
+12/09/2025 14:26:32 - INFO - __main__ - Step: 590 Loss: 10.5714 LR: 0.000300
+12/09/2025 14:26:41 - INFO - __main__ - Step: 600 Loss: 10.5378 LR: 0.000300
+12/09/2025 14:26:49 - INFO - __main__ - Step: 610 Loss: 10.4954 LR: 0.000300
+12/09/2025 14:26:57 - INFO - __main__ - Step: 620 Loss: 10.4548 LR: 0.000300
+12/09/2025 14:27:05 - INFO - __main__ - Step: 630 Loss: 10.4652 LR: 0.000300
+12/09/2025 14:27:14 - INFO - __main__ - Step: 640 Loss: 10.5761 LR: 0.000300
+12/09/2025 14:27:22 - INFO - __main__ - Step: 650 Loss: 10.4299 LR: 0.000300
+12/09/2025 14:27:30 - INFO - __main__ - Step: 660 Loss: 10.6078 LR: 0.000300
+12/09/2025 14:27:39 - INFO - __main__ - Step: 670 Loss: 10.3975 LR: 0.000300
+12/09/2025 14:27:48 - INFO - __main__ - Step: 680 Loss: 10.5504 LR: 0.000300
+12/09/2025 14:27:56 - INFO - __main__ - Step: 690 Loss: 10.5034 LR: 0.000300
+12/09/2025 14:28:04 - INFO - __main__ - Step: 700 Loss: 10.5199 LR: 0.000300
+12/09/2025 14:28:12 - INFO - __main__ - Step: 710 Loss: 10.4411 LR: 0.000300
+12/09/2025 14:28:21 - INFO - __main__ - Step: 720 Loss: 10.5867 LR: 0.000300
+12/09/2025 14:28:30 - INFO - __main__ - Step: 730 Loss: 10.5285 LR: 0.000300
+12/09/2025 14:28:38 - INFO - __main__ - Step: 740 Loss: 10.5524 LR: 0.000300
+12/09/2025 14:28:47 - INFO - __main__ - Step: 750 Loss: 10.4680 LR: 0.000300
+12/09/2025 14:28:55 - INFO - __main__ - Step: 760 Loss: 10.5404 LR: 0.000300
+12/09/2025 14:29:04 - INFO - __main__ - Step: 770 Loss: 10.4509 LR: 0.000300
+12/09/2025 14:29:12 - INFO - __main__ - Step: 780 Loss: 10.5221 LR: 0.000300
+12/09/2025 14:29:21 - INFO - __main__ - Step: 790 Loss: 10.4907 LR: 0.000300
+12/09/2025 14:29:29 - INFO - __main__ - Step: 800 Loss: 10.3947 LR: 0.000300
+12/09/2025 14:29:38 - INFO - __main__ - Step: 810 Loss: 10.5406 LR: 0.000300
+12/09/2025 14:29:46 - INFO - __main__ - Step: 820 Loss: 10.4586 LR: 0.000300
+12/09/2025 14:29:55 - INFO - __main__ - Step: 830 Loss: 10.5223 LR: 0.000300
+12/09/2025 14:30:04 - INFO - __main__ - Step: 840 Loss: 10.4387 LR: 0.000300
+12/09/2025 14:30:12 - INFO - __main__ - Step: 850 Loss: 10.5751 LR: 0.000300
+12/09/2025 14:30:20 - INFO - __main__ - Step: 860 Loss: 10.4888 LR: 0.000300
+12/09/2025 14:30:28 - INFO - __main__ - Step: 870 Loss: 10.5258 LR: 0.000300
+12/09/2025 14:30:37 - INFO - __main__ - Step: 880 Loss: 10.3787 LR: 0.000300
+12/09/2025 14:30:45 - INFO - __main__ - Step: 890 Loss: 10.3855 LR: 0.000300
+12/09/2025 14:30:53 - INFO - __main__ - Step: 900 Loss: 10.3843 LR: 0.000300
+12/09/2025 14:31:02 - INFO - __main__ - Step: 910 Loss: 10.4360 LR: 0.000300
+12/09/2025 14:31:10 - INFO - __main__ - Step: 920 Loss: 10.5285 LR: 0.000300
+12/09/2025 14:31:18 - INFO - __main__ - Step: 930 Loss: 10.4288 LR: 0.000300
+12/09/2025 14:31:27 - INFO - __main__ - Step: 940 Loss: 10.5211 LR: 0.000300
+12/09/2025 14:31:36 - INFO - __main__ - Step: 950 Loss: 10.4077 LR: 0.000300
+12/09/2025 14:31:45 - INFO - __main__ - Step: 960 Loss: 10.3606 LR: 0.000300
+12/09/2025 14:31:53 - INFO - __main__ - Step: 970 Loss: 10.3525 LR: 0.000300
+12/09/2025 14:32:02 - INFO - __main__ - Step: 980 Loss: 10.4681 LR: 0.000300
+12/09/2025 14:32:10 - INFO - __main__ - Step: 990 Loss: 10.4996 LR: 0.000300
+12/09/2025 14:32:19 - INFO - __main__ - Step: 1000 Loss: 10.4602 LR: 0.000300
+12/09/2025 14:32:19 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000
+12/09/2025 14:32:26 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000/optimizer.bin
+12/09/2025 14:32:26 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000/scheduler.bin
+12/09/2025 14:32:26 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000/sampler.bin
+12/09/2025 14:32:26 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000/random_states_0.pkl
+12/09/2025 14:32:26 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1000
+12/09/2025 14:32:26 - INFO - __main__ - Generating videos for validation...
+12/09/2025 14:32:26 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.77it/s]
+12/09/2025 14:32:34 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 14:32:42 - INFO - __main__ - Step: 1010 Loss: 10.5081 LR: 0.000300
+12/09/2025 14:32:50 - INFO - __main__ - Step: 1020 Loss: 10.4993 LR: 0.000300
+12/09/2025 14:32:58 - INFO - __main__ - Step: 1030 Loss: 10.3718 LR: 0.000300
+12/09/2025 14:33:07 - INFO - __main__ - Step: 1040 Loss: 10.5117 LR: 0.000300
+12/09/2025 14:33:15 - INFO - __main__ - Step: 1050 Loss: 10.3794 LR: 0.000300
+12/09/2025 14:33:23 - INFO - __main__ - Step: 1060 Loss: 10.5579 LR: 0.000300
+12/09/2025 14:33:32 - INFO - __main__ - Step: 1070 Loss: 10.4656 LR: 0.000300
+12/09/2025 14:33:40 - INFO - __main__ - Step: 1080 Loss: 10.5221 LR: 0.000300
+12/09/2025 14:33:48 - INFO - __main__ - Step: 1090 Loss: 10.4562 LR: 0.000300
+12/09/2025 14:33:56 - INFO - __main__ - Step: 1100 Loss: 10.4584 LR: 0.000300
+12/09/2025 14:34:03 - INFO - __main__ - Step: 1110 Loss: 10.4657 LR: 0.000300
+12/09/2025 14:34:11 - INFO - __main__ - Step: 1120 Loss: 10.3170 LR: 0.000300
+12/09/2025 14:34:20 - INFO - __main__ - Step: 1130 Loss: 10.4157 LR: 0.000300
+12/09/2025 14:34:28 - INFO - __main__ - Step: 1140 Loss: 10.5589 LR: 0.000300
+12/09/2025 14:34:36 - INFO - __main__ - Step: 1150 Loss: 10.4533 LR: 0.000300
+12/09/2025 14:34:45 - INFO - __main__ - Step: 1160 Loss: 10.4369 LR: 0.000300
+12/09/2025 14:34:53 - INFO - __main__ - Step: 1170 Loss: 10.3897 LR: 0.000300
+12/09/2025 14:35:01 - INFO - __main__ - Step: 1180 Loss: 10.4845 LR: 0.000300
+12/09/2025 14:35:09 - INFO - __main__ - Step: 1190 Loss: 10.3569 LR: 0.000300
+12/09/2025 14:35:17 - INFO - __main__ - Step: 1200 Loss: 10.4201 LR: 0.000300
+12/09/2025 14:35:25 - INFO - __main__ - Step: 1210 Loss: 10.3938 LR: 0.000300
+12/09/2025 14:35:34 - INFO - __main__ - Step: 1220 Loss: 10.3758 LR: 0.000300
+12/09/2025 14:35:43 - INFO - __main__ - Step: 1230 Loss: 10.5326 LR: 0.000300
+12/09/2025 14:35:51 - INFO - __main__ - Step: 1240 Loss: 10.3612 LR: 0.000300
+12/09/2025 14:36:00 - INFO - __main__ - Step: 1250 Loss: 10.4457 LR: 0.000300
+12/09/2025 14:36:08 - INFO - __main__ - Step: 1260 Loss: 10.4828 LR: 0.000300
+12/09/2025 14:36:16 - INFO - __main__ - Step: 1270 Loss: 10.4687 LR: 0.000300
+12/09/2025 14:36:25 - INFO - __main__ - Step: 1280 Loss: 10.3611 LR: 0.000300
+12/09/2025 14:36:33 - INFO - __main__ - Step: 1290 Loss: 10.4610 LR: 0.000300
+12/09/2025 14:36:42 - INFO - __main__ - Step: 1300 Loss: 10.4953 LR: 0.000300
+12/09/2025 14:36:50 - INFO - __main__ - Step: 1310 Loss: 10.4658 LR: 0.000300
+12/09/2025 14:36:59 - INFO - __main__ - Step: 1320 Loss: 10.4092 LR: 0.000300
+12/09/2025 14:37:07 - INFO - __main__ - Step: 1330 Loss: 10.5397 LR: 0.000300
+12/09/2025 14:37:15 - INFO - __main__ - Step: 1340 Loss: 10.4840 LR: 0.000300
+12/09/2025 14:37:23 - INFO - __main__ - Step: 1350 Loss: 10.4958 LR: 0.000300
+12/09/2025 14:37:32 - INFO - __main__ - Step: 1360 Loss: 10.5926 LR: 0.000300
+12/09/2025 14:37:40 - INFO - __main__ - Step: 1370 Loss: 10.4392 LR: 0.000300
+12/09/2025 14:37:49 - INFO - __main__ - Step: 1380 Loss: 10.5155 LR: 0.000300
+12/09/2025 14:37:57 - INFO - __main__ - Step: 1390 Loss: 10.4504 LR: 0.000300
+12/09/2025 14:38:06 - INFO - __main__ - Step: 1400 Loss: 10.5094 LR: 0.000300
+12/09/2025 14:38:14 - INFO - __main__ - Step: 1410 Loss: 10.4910 LR: 0.000300
+12/09/2025 14:38:22 - INFO - __main__ - Step: 1420 Loss: 10.4078 LR: 0.000300
+12/09/2025 14:38:30 - INFO - __main__ - Step: 1430 Loss: 10.4891 LR: 0.000300
+12/09/2025 14:38:39 - INFO - __main__ - Step: 1440 Loss: 10.4848 LR: 0.000300
+12/09/2025 14:38:47 - INFO - __main__ - Step: 1450 Loss: 10.3720 LR: 0.000300
+12/09/2025 14:38:55 - INFO - __main__ - Step: 1460 Loss: 10.5407 LR: 0.000300
+12/09/2025 14:39:04 - INFO - __main__ - Step: 1470 Loss: 10.4086 LR: 0.000300
+12/09/2025 14:39:12 - INFO - __main__ - Step: 1480 Loss: 10.4516 LR: 0.000300
+12/09/2025 14:39:20 - INFO - __main__ - Step: 1490 Loss: 10.4011 LR: 0.000300
+12/09/2025 14:39:28 - INFO - __main__ - Step: 1500 Loss: 10.4982 LR: 0.000300
+12/09/2025 14:39:28 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500
+12/09/2025 14:39:35 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500/optimizer.bin
+12/09/2025 14:39:35 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500/scheduler.bin
+12/09/2025 14:39:35 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500/sampler.bin
+12/09/2025 14:39:35 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500/random_states_0.pkl
+12/09/2025 14:39:35 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-1500
+12/09/2025 14:39:35 - INFO - __main__ - Generating videos for validation...
+12/09/2025 14:39:35 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.62it/s]
+12/09/2025 14:39:42 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 14:39:51 - INFO - __main__ - Step: 1510 Loss: 10.3688 LR: 0.000300
+12/09/2025 14:39:59 - INFO - __main__ - Step: 1520 Loss: 10.4142 LR: 0.000300
+12/09/2025 14:40:08 - INFO - __main__ - Step: 1530 Loss: 10.4045 LR: 0.000300
+12/09/2025 14:40:16 - INFO - __main__ - Step: 1540 Loss: 10.4536 LR: 0.000300
+12/09/2025 14:40:24 - INFO - __main__ - Step: 1550 Loss: 10.4827 LR: 0.000300
+12/09/2025 14:40:33 - INFO - __main__ - Step: 1560 Loss: 10.4726 LR: 0.000300
+12/09/2025 14:40:41 - INFO - __main__ - Step: 1570 Loss: 10.4621 LR: 0.000300
+12/09/2025 14:40:50 - INFO - __main__ - Step: 1580 Loss: 10.4076 LR: 0.000300
+12/09/2025 14:40:57 - INFO - __main__ - Step: 1590 Loss: 10.3771 LR: 0.000300
+12/09/2025 14:41:05 - INFO - __main__ - Step: 1600 Loss: 10.4990 LR: 0.000300
+12/09/2025 14:41:13 - INFO - __main__ - Step: 1610 Loss: 10.4034 LR: 0.000300
+12/09/2025 14:41:21 - INFO - __main__ - Step: 1620 Loss: 10.4673 LR: 0.000300
+12/09/2025 14:41:30 - INFO - __main__ - Step: 1630 Loss: 10.5685 LR: 0.000300
+12/09/2025 14:41:38 - INFO - __main__ - Step: 1640 Loss: 10.2373 LR: 0.000300
+12/09/2025 14:41:47 - INFO - __main__ - Step: 1650 Loss: 10.3953 LR: 0.000300
+12/09/2025 14:41:54 - INFO - __main__ - Step: 1660 Loss: 10.4320 LR: 0.000300
+12/09/2025 14:42:02 - INFO - __main__ - Step: 1670 Loss: 10.3989 LR: 0.000300
+12/09/2025 14:42:10 - INFO - __main__ - Step: 1680 Loss: 10.3984 LR: 0.000300
+12/09/2025 14:42:18 - INFO - __main__ - Step: 1690 Loss: 10.3753 LR: 0.000300
+12/09/2025 14:42:26 - INFO - __main__ - Step: 1700 Loss: 10.3394 LR: 0.000300
+12/09/2025 14:42:34 - INFO - __main__ - Step: 1710 Loss: 10.3646 LR: 0.000300
+12/09/2025 14:42:43 - INFO - __main__ - Step: 1720 Loss: 10.4932 LR: 0.000300
+12/09/2025 14:42:50 - INFO - __main__ - Step: 1730 Loss: 10.2850 LR: 0.000300
+12/09/2025 14:42:58 - INFO - __main__ - Step: 1740 Loss: 10.4153 LR: 0.000300
+12/09/2025 14:43:05 - INFO - __main__ - Step: 1750 Loss: 10.4864 LR: 0.000300
+12/09/2025 14:43:13 - INFO - __main__ - Step: 1760 Loss: 10.3920 LR: 0.000300
+12/09/2025 14:43:21 - INFO - __main__ - Step: 1770 Loss: 10.4139 LR: 0.000300
+12/09/2025 14:43:29 - INFO - __main__ - Step: 1780 Loss: 10.4104 LR: 0.000300
+12/09/2025 14:43:37 - INFO - __main__ - Step: 1790 Loss: 10.3351 LR: 0.000300
+12/09/2025 14:43:45 - INFO - __main__ - Step: 1800 Loss: 10.4453 LR: 0.000300
+12/09/2025 14:43:53 - INFO - __main__ - Step: 1810 Loss: 10.4127 LR: 0.000300
+12/09/2025 14:44:01 - INFO - __main__ - Step: 1820 Loss: 10.4920 LR: 0.000300
+12/09/2025 14:44:09 - INFO - __main__ - Step: 1830 Loss: 10.4515 LR: 0.000300
+12/09/2025 14:44:16 - INFO - __main__ - Step: 1840 Loss: 10.4176 LR: 0.000300
+12/09/2025 14:44:24 - INFO - __main__ - Step: 1850 Loss: 10.4255 LR: 0.000300
+12/09/2025 14:44:32 - INFO - __main__ - Step: 1860 Loss: 10.3986 LR: 0.000300
+12/09/2025 14:44:40 - INFO - __main__ - Step: 1870 Loss: 10.3737 LR: 0.000300
+12/09/2025 14:44:47 - INFO - __main__ - Step: 1880 Loss: 10.4050 LR: 0.000300
+12/09/2025 14:44:55 - INFO - __main__ - Step: 1890 Loss: 10.3659 LR: 0.000300
+12/09/2025 14:45:02 - INFO - __main__ - Step: 1900 Loss: 10.3975 LR: 0.000300
+12/09/2025 14:45:10 - INFO - __main__ - Step: 1910 Loss: 10.4211 LR: 0.000300
+12/09/2025 14:45:19 - INFO - __main__ - Step: 1920 Loss: 10.3395 LR: 0.000300
+12/09/2025 14:45:27 - INFO - __main__ - Step: 1930 Loss: 10.4324 LR: 0.000300
+12/09/2025 14:45:35 - INFO - __main__ - Step: 1940 Loss: 10.5370 LR: 0.000300
+12/09/2025 14:45:43 - INFO - __main__ - Step: 1950 Loss: 10.4142 LR: 0.000300
+12/09/2025 14:45:51 - INFO - __main__ - Step: 1960 Loss: 10.3945 LR: 0.000300
+12/09/2025 14:45:58 - INFO - __main__ - Step: 1970 Loss: 10.4885 LR: 0.000300
+12/09/2025 14:46:07 - INFO - __main__ - Step: 1980 Loss: 10.4398 LR: 0.000300
+12/09/2025 14:46:15 - INFO - __main__ - Step: 1990 Loss: 10.3821 LR: 0.000300
+12/09/2025 14:46:23 - INFO - __main__ - Step: 2000 Loss: 10.4714 LR: 0.000300
+12/09/2025 14:46:23 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000
+12/09/2025 14:46:31 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000/optimizer.bin
+12/09/2025 14:46:31 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000/scheduler.bin
+12/09/2025 14:46:31 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000/sampler.bin
+12/09/2025 14:46:31 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000/random_states_0.pkl
+12/09/2025 14:46:31 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2000
+12/09/2025 14:46:31 - INFO - __main__ - Generating videos for validation...
+12/09/2025 14:46:31 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.74it/s]
+12/09/2025 14:46:37 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 14:46:46 - INFO - __main__ - Step: 2010 Loss: 10.4913 LR: 0.000300
+12/09/2025 14:46:54 - INFO - __main__ - Step: 2020 Loss: 10.4892 LR: 0.000300
+12/09/2025 14:47:02 - INFO - __main__ - Step: 2030 Loss: 10.5157 LR: 0.000300
+12/09/2025 14:47:11 - INFO - __main__ - Step: 2040 Loss: 10.4690 LR: 0.000300
+12/09/2025 14:47:18 - INFO - __main__ - Step: 2050 Loss: 10.4872 LR: 0.000300
+12/09/2025 14:47:26 - INFO - __main__ - Step: 2060 Loss: 10.4257 LR: 0.000300
+12/09/2025 14:47:35 - INFO - __main__ - Step: 2070 Loss: 10.4276 LR: 0.000300
+12/09/2025 14:47:44 - INFO - __main__ - Step: 2080 Loss: 10.3805 LR: 0.000300
+12/09/2025 14:47:52 - INFO - __main__ - Step: 2090 Loss: 10.5010 LR: 0.000300
+12/09/2025 14:48:00 - INFO - __main__ - Step: 2100 Loss: 10.4214 LR: 0.000300
+12/09/2025 14:48:09 - INFO - __main__ - Step: 2110 Loss: 10.4174 LR: 0.000300
+12/09/2025 14:48:16 - INFO - __main__ - Step: 2120 Loss: 10.4200 LR: 0.000300
+12/09/2025 14:48:24 - INFO - __main__ - Step: 2130 Loss: 10.3379 LR: 0.000300
+12/09/2025 14:48:32 - INFO - __main__ - Step: 2140 Loss: 10.4045 LR: 0.000300
+12/09/2025 14:48:41 - INFO - __main__ - Step: 2150 Loss: 10.4128 LR: 0.000300
+12/09/2025 14:48:49 - INFO - __main__ - Step: 2160 Loss: 10.3600 LR: 0.000300
+12/09/2025 14:48:58 - INFO - __main__ - Step: 2170 Loss: 10.4262 LR: 0.000300
+12/09/2025 14:49:06 - INFO - __main__ - Step: 2180 Loss: 10.3869 LR: 0.000300
+12/09/2025 14:49:15 - INFO - __main__ - Step: 2190 Loss: 10.3994 LR: 0.000300
+12/09/2025 14:49:23 - INFO - __main__ - Step: 2200 Loss: 10.3750 LR: 0.000300
+12/09/2025 14:49:31 - INFO - __main__ - Step: 2210 Loss: 10.4021 LR: 0.000300
+12/09/2025 14:49:40 - INFO - __main__ - Step: 2220 Loss: 10.3652 LR: 0.000300
+12/09/2025 14:49:49 - INFO - __main__ - Step: 2230 Loss: 10.5028 LR: 0.000300
+12/09/2025 14:49:58 - INFO - __main__ - Step: 2240 Loss: 10.4137 LR: 0.000300
+12/09/2025 14:50:07 - INFO - __main__ - Step: 2250 Loss: 10.4091 LR: 0.000300
+12/09/2025 14:50:16 - INFO - __main__ - Step: 2260 Loss: 10.5426 LR: 0.000300
+12/09/2025 14:50:24 - INFO - __main__ - Step: 2270 Loss: 10.4749 LR: 0.000300
+12/09/2025 14:50:32 - INFO - __main__ - Step: 2280 Loss: 10.4904 LR: 0.000300
+12/09/2025 14:50:40 - INFO - __main__ - Step: 2290 Loss: 10.3283 LR: 0.000300
+12/09/2025 14:50:49 - INFO - __main__ - Step: 2300 Loss: 10.4035 LR: 0.000300
+12/09/2025 14:50:57 - INFO - __main__ - Step: 2310 Loss: 10.4176 LR: 0.000300
+12/09/2025 14:51:06 - INFO - __main__ - Step: 2320 Loss: 10.5085 LR: 0.000300
+12/09/2025 14:51:15 - INFO - __main__ - Step: 2330 Loss: 10.3436 LR: 0.000300
+12/09/2025 14:51:23 - INFO - __main__ - Step: 2340 Loss: 10.4886 LR: 0.000300
+12/09/2025 14:51:32 - INFO - __main__ - Step: 2350 Loss: 10.4517 LR: 0.000300
+12/09/2025 14:51:40 - INFO - __main__ - Step: 2360 Loss: 10.3703 LR: 0.000300
+12/09/2025 14:51:47 - INFO - __main__ - Step: 2370 Loss: 10.4542 LR: 0.000300
+12/09/2025 14:51:56 - INFO - __main__ - Step: 2380 Loss: 10.2895 LR: 0.000300
+12/09/2025 14:52:04 - INFO - __main__ - Step: 2390 Loss: 10.3339 LR: 0.000300
+12/09/2025 14:52:13 - INFO - __main__ - Step: 2400 Loss: 10.3490 LR: 0.000300
+12/09/2025 14:52:21 - INFO - __main__ - Step: 2410 Loss: 10.4567 LR: 0.000300
+12/09/2025 14:52:29 - INFO - __main__ - Step: 2420 Loss: 10.5396 LR: 0.000300
+12/09/2025 14:52:37 - INFO - __main__ - Step: 2430 Loss: 10.5307 LR: 0.000300
+12/09/2025 14:52:45 - INFO - __main__ - Step: 2440 Loss: 10.4733 LR: 0.000300
+12/09/2025 14:52:53 - INFO - __main__ - Step: 2450 Loss: 10.3741 LR: 0.000300
+12/09/2025 14:53:01 - INFO - __main__ - Step: 2460 Loss: 10.4488 LR: 0.000300
+12/09/2025 14:53:10 - INFO - __main__ - Step: 2470 Loss: 10.4334 LR: 0.000300
+12/09/2025 14:53:18 - INFO - __main__ - Step: 2480 Loss: 10.4863 LR: 0.000300
+12/09/2025 14:53:26 - INFO - __main__ - Step: 2490 Loss: 10.4204 LR: 0.000300
+12/09/2025 14:53:34 - INFO - __main__ - Step: 2500 Loss: 10.4145 LR: 0.000300
+12/09/2025 14:53:34 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500
+12/09/2025 14:53:42 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500/optimizer.bin
+12/09/2025 14:53:42 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500/scheduler.bin
+12/09/2025 14:53:42 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500/sampler.bin
+12/09/2025 14:53:42 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500/random_states_0.pkl
+12/09/2025 14:53:42 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-2500
+12/09/2025 14:53:42 - INFO - __main__ - Generating videos for validation...
+12/09/2025 14:53:42 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.74it/s]
+12/09/2025 14:53:49 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 14:53:56 - INFO - __main__ - Step: 2510 Loss: 10.4580 LR: 0.000300
+12/09/2025 14:54:04 - INFO - __main__ - Step: 2520 Loss: 10.4518 LR: 0.000300
+12/09/2025 14:54:12 - INFO - __main__ - Step: 2530 Loss: 10.5546 LR: 0.000300
+12/09/2025 14:54:20 - INFO - __main__ - Step: 2540 Loss: 10.4131 LR: 0.000300
+12/09/2025 14:54:29 - INFO - __main__ - Step: 2550 Loss: 10.3408 LR: 0.000300
+12/09/2025 14:54:37 - INFO - __main__ - Step: 2560 Loss: 10.3797 LR: 0.000300
+12/09/2025 14:54:45 - INFO - __main__ - Step: 2570 Loss: 10.4836 LR: 0.000300
+12/09/2025 14:54:53 - INFO - __main__ - Step: 2580 Loss: 10.2595 LR: 0.000300
+12/09/2025 14:55:00 - INFO - __main__ - Step: 2590 Loss: 10.3303 LR: 0.000300
+12/09/2025 14:55:07 - INFO - __main__ - Step: 2600 Loss: 10.3411 LR: 0.000300
+12/09/2025 14:55:16 - INFO - __main__ - Step: 2610 Loss: 10.4018 LR: 0.000300
+12/09/2025 14:55:24 - INFO - __main__ - Step: 2620 Loss: 10.4771 LR: 0.000300
+12/09/2025 14:55:32 - INFO - __main__ - Step: 2630 Loss: 10.3311 LR: 0.000300
+12/09/2025 14:55:40 - INFO - __main__ - Step: 2640 Loss: 10.3900 LR: 0.000300
+12/09/2025 14:55:49 - INFO - __main__ - Step: 2650 Loss: 10.3524 LR: 0.000300
+12/09/2025 14:55:57 - INFO - __main__ - Step: 2660 Loss: 10.3557 LR: 0.000300
+12/09/2025 14:56:05 - INFO - __main__ - Step: 2670 Loss: 10.3889 LR: 0.000300
+12/09/2025 14:56:13 - INFO - __main__ - Step: 2680 Loss: 10.3013 LR: 0.000300
+12/09/2025 14:56:21 - INFO - __main__ - Step: 2690 Loss: 10.4000 LR: 0.000300
+12/09/2025 14:56:29 - INFO - __main__ - Step: 2700 Loss: 10.4197 LR: 0.000300
+12/09/2025 14:56:37 - INFO - __main__ - Step: 2710 Loss: 10.4226 LR: 0.000300
+12/09/2025 14:56:46 - INFO - __main__ - Step: 2720 Loss: 10.3753 LR: 0.000300
+12/09/2025 14:56:54 - INFO - __main__ - Step: 2730 Loss: 10.4958 LR: 0.000300
+12/09/2025 14:57:02 - INFO - __main__ - Step: 2740 Loss: 10.3689 LR: 0.000300
+12/09/2025 14:57:10 - INFO - __main__ - Step: 2750 Loss: 10.4282 LR: 0.000300
+12/09/2025 14:57:18 - INFO - __main__ - Step: 2760 Loss: 10.4701 LR: 0.000300
+12/09/2025 14:57:26 - INFO - __main__ - Step: 2770 Loss: 10.4405 LR: 0.000300
+12/09/2025 14:57:35 - INFO - __main__ - Step: 2780 Loss: 10.4365 LR: 0.000300
+12/09/2025 14:57:43 - INFO - __main__ - Step: 2790 Loss: 10.4165 LR: 0.000300
+12/09/2025 14:57:52 - INFO - __main__ - Step: 2800 Loss: 10.4600 LR: 0.000300
+12/09/2025 14:58:00 - INFO - __main__ - Step: 2810 Loss: 10.3449 LR: 0.000300
+12/09/2025 14:58:08 - INFO - __main__ - Step: 2820 Loss: 10.3529 LR: 0.000300
+12/09/2025 14:58:16 - INFO - __main__ - Step: 2830 Loss: 10.3326 LR: 0.000300
+12/09/2025 14:58:25 - INFO - __main__ - Step: 2840 Loss: 10.5020 LR: 0.000300
+12/09/2025 14:58:33 - INFO - __main__ - Step: 2850 Loss: 10.4364 LR: 0.000300
+12/09/2025 14:58:41 - INFO - __main__ - Step: 2860 Loss: 10.3497 LR: 0.000300
+12/09/2025 14:58:49 - INFO - __main__ - Step: 2870 Loss: 10.4029 LR: 0.000300
+12/09/2025 14:58:58 - INFO - __main__ - Step: 2880 Loss: 10.3246 LR: 0.000300
+12/09/2025 14:59:06 - INFO - __main__ - Step: 2890 Loss: 10.4158 LR: 0.000300
+12/09/2025 14:59:14 - INFO - __main__ - Step: 2900 Loss: 10.3602 LR: 0.000300
+12/09/2025 14:59:22 - INFO - __main__ - Step: 2910 Loss: 10.3202 LR: 0.000300
+12/09/2025 14:59:30 - INFO - __main__ - Step: 2920 Loss: 10.4786 LR: 0.000300
+12/09/2025 14:59:39 - INFO - __main__ - Step: 2930 Loss: 10.4029 LR: 0.000300
+12/09/2025 14:59:48 - INFO - __main__ - Step: 2940 Loss: 10.4191 LR: 0.000300
+12/09/2025 14:59:57 - INFO - __main__ - Step: 2950 Loss: 10.4091 LR: 0.000300
+12/09/2025 15:00:06 - INFO - __main__ - Step: 2960 Loss: 10.5146 LR: 0.000300
+12/09/2025 15:00:14 - INFO - __main__ - Step: 2970 Loss: 10.3983 LR: 0.000300
+12/09/2025 15:00:22 - INFO - __main__ - Step: 2980 Loss: 10.3034 LR: 0.000300
+12/09/2025 15:00:30 - INFO - __main__ - Step: 2990 Loss: 10.3820 LR: 0.000300
+12/09/2025 15:00:39 - INFO - __main__ - Step: 3000 Loss: 10.4916 LR: 0.000300
+12/09/2025 15:00:39 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000
+12/09/2025 15:00:46 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000/optimizer.bin
+12/09/2025 15:00:46 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000/scheduler.bin
+12/09/2025 15:00:46 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000/sampler.bin
+12/09/2025 15:00:46 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000/random_states_0.pkl
+12/09/2025 15:00:46 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3000
+12/09/2025 15:00:46 - INFO - __main__ - Generating videos for validation...
+12/09/2025 15:00:46 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.76it/s]
+12/09/2025 15:00:53 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 15:01:01 - INFO - __main__ - Step: 3010 Loss: 10.4128 LR: 0.000300
+12/09/2025 15:01:10 - INFO - __main__ - Step: 3020 Loss: 10.4473 LR: 0.000300
+12/09/2025 15:01:18 - INFO - __main__ - Step: 3030 Loss: 10.5300 LR: 0.000300
+12/09/2025 15:01:27 - INFO - __main__ - Step: 3040 Loss: 10.4155 LR: 0.000300
+12/09/2025 15:01:34 - INFO - __main__ - Step: 3050 Loss: 10.4354 LR: 0.000300
+12/09/2025 15:01:43 - INFO - __main__ - Step: 3060 Loss: 10.3338 LR: 0.000300
+12/09/2025 15:01:51 - INFO - __main__ - Step: 3070 Loss: 10.4649 LR: 0.000300
+12/09/2025 15:01:59 - INFO - __main__ - Step: 3080 Loss: 10.4024 LR: 0.000300
+12/09/2025 15:02:07 - INFO - __main__ - Step: 3090 Loss: 10.4597 LR: 0.000300
+12/09/2025 15:02:16 - INFO - __main__ - Step: 3100 Loss: 10.4229 LR: 0.000300
+12/09/2025 15:02:25 - INFO - __main__ - Step: 3110 Loss: 10.2890 LR: 0.000300
+12/09/2025 15:02:33 - INFO - __main__ - Step: 3120 Loss: 10.3721 LR: 0.000300
+12/09/2025 15:02:41 - INFO - __main__ - Step: 3130 Loss: 10.4633 LR: 0.000300
+12/09/2025 15:02:49 - INFO - __main__ - Step: 3140 Loss: 10.2339 LR: 0.000300
+12/09/2025 15:02:57 - INFO - __main__ - Step: 3150 Loss: 10.3505 LR: 0.000300
+12/09/2025 15:03:05 - INFO - __main__ - Step: 3160 Loss: 10.3656 LR: 0.000300
+12/09/2025 15:03:13 - INFO - __main__ - Step: 3170 Loss: 10.3447 LR: 0.000300
+12/09/2025 15:03:22 - INFO - __main__ - Step: 3180 Loss: 10.3553 LR: 0.000300
+12/09/2025 15:03:31 - INFO - __main__ - Step: 3190 Loss: 10.4069 LR: 0.000300
+12/09/2025 15:03:39 - INFO - __main__ - Step: 3200 Loss: 10.4120 LR: 0.000300
+12/09/2025 15:03:48 - INFO - __main__ - Step: 3210 Loss: 10.3601 LR: 0.000300
+12/09/2025 15:03:56 - INFO - __main__ - Step: 3220 Loss: 10.3693 LR: 0.000300
+12/09/2025 15:04:04 - INFO - __main__ - Step: 3230 Loss: 10.3294 LR: 0.000300
+12/09/2025 15:04:13 - INFO - __main__ - Step: 3240 Loss: 10.3824 LR: 0.000300
+12/09/2025 15:04:22 - INFO - __main__ - Step: 3250 Loss: 10.4374 LR: 0.000300
+12/09/2025 15:04:30 - INFO - __main__ - Step: 3260 Loss: 10.4087 LR: 0.000300
+12/09/2025 15:04:39 - INFO - __main__ - Step: 3270 Loss: 10.4182 LR: 0.000300
+12/09/2025 15:04:47 - INFO - __main__ - Step: 3280 Loss: 10.3291 LR: 0.000300
+12/09/2025 15:04:56 - INFO - __main__ - Step: 3290 Loss: 10.3474 LR: 0.000300
+12/09/2025 15:05:03 - INFO - __main__ - Step: 3300 Loss: 10.3270 LR: 0.000300
+12/09/2025 15:05:12 - INFO - __main__ - Step: 3310 Loss: 10.4398 LR: 0.000300
+12/09/2025 15:05:20 - INFO - __main__ - Step: 3320 Loss: 10.3624 LR: 0.000300
+12/09/2025 15:05:29 - INFO - __main__ - Step: 3330 Loss: 10.4954 LR: 0.000300
+12/09/2025 15:05:37 - INFO - __main__ - Step: 3340 Loss: 10.4612 LR: 0.000300
+12/09/2025 15:05:45 - INFO - __main__ - Step: 3350 Loss: 10.3814 LR: 0.000300
+12/09/2025 15:05:54 - INFO - __main__ - Step: 3360 Loss: 10.3554 LR: 0.000300
+12/09/2025 15:06:01 - INFO - __main__ - Step: 3370 Loss: 10.4198 LR: 0.000300
+12/09/2025 15:06:09 - INFO - __main__ - Step: 3380 Loss: 10.4446 LR: 0.000300
+12/09/2025 15:06:18 - INFO - __main__ - Step: 3390 Loss: 10.4236 LR: 0.000300
+12/09/2025 15:06:26 - INFO - __main__ - Step: 3400 Loss: 10.4261 LR: 0.000300
+12/09/2025 15:06:35 - INFO - __main__ - Step: 3410 Loss: 10.4843 LR: 0.000300
+12/09/2025 15:06:43 - INFO - __main__ - Step: 3420 Loss: 10.3067 LR: 0.000300
+12/09/2025 15:06:51 - INFO - __main__ - Step: 3430 Loss: 10.2871 LR: 0.000300
+12/09/2025 15:06:59 - INFO - __main__ - Step: 3440 Loss: 10.4699 LR: 0.000300
+12/09/2025 15:07:07 - INFO - __main__ - Step: 3450 Loss: 10.5213 LR: 0.000300
+12/09/2025 15:07:15 - INFO - __main__ - Step: 3460 Loss: 10.4694 LR: 0.000300
+12/09/2025 15:07:24 - INFO - __main__ - Step: 3470 Loss: 10.3963 LR: 0.000300
+12/09/2025 15:07:32 - INFO - __main__ - Step: 3480 Loss: 10.4730 LR: 0.000300
+12/09/2025 15:07:41 - INFO - __main__ - Step: 3490 Loss: 10.4303 LR: 0.000300
+12/09/2025 15:07:49 - INFO - __main__ - Step: 3500 Loss: 10.3538 LR: 0.000300
+12/09/2025 15:07:49 - INFO - accelerate.accelerator - Saving current state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500
+12/09/2025 15:07:57 - INFO - accelerate.checkpointing - Optimizer state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500/optimizer.bin
+12/09/2025 15:07:57 - INFO - accelerate.checkpointing - Scheduler state saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500/scheduler.bin
+12/09/2025 15:07:57 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500/sampler.bin
+12/09/2025 15:07:57 - INFO - accelerate.checkpointing - Random states saved in output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500/random_states_0.pkl
+12/09/2025 15:07:57 - INFO - __main__ - Saved state to output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp/checkpoint-3500
+12/09/2025 15:07:57 - INFO - __main__ - Generating videos for validation...
+12/09/2025 15:07:57 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:06<00:00, 7.62it/s]
+12/09/2025 15:08:03 - INFO - __main__ - Validation videos saved to ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+12/09/2025 15:08:11 - INFO - __main__ - Step: 3510 Loss: 10.3953 LR: 0.000300
+12/09/2025 15:08:19 - INFO - __main__ - Step: 3520 Loss: 10.3537 LR: 0.000300
+12/09/2025 15:08:27 - INFO - __main__ - Step: 3530 Loss: 10.5191 LR: 0.000300
+12/09/2025 15:08:35 - INFO - __main__ - Step: 3540 Loss: 10.3530 LR: 0.000300
+12/09/2025 15:08:43 - INFO - __main__ - Step: 3550 Loss: 10.3432 LR: 0.000300
+12/09/2025 15:08:52 - INFO - __main__ - Step: 3560 Loss: 10.4816 LR: 0.000300
+12/09/2025 15:09:01 - INFO - __main__ - Step: 3570 Loss: 10.4883 LR: 0.000300
+12/09/2025 15:09:09 - INFO - __main__ - Step: 3580 Loss: 10.2422 LR: 0.000300
+12/09/2025 15:09:17 - INFO - __main__ - Step: 3590 Loss: 10.4056 LR: 0.000300
+12/09/2025 15:09:25 - INFO - __main__ - Step: 3600 Loss: 10.4111 LR: 0.000300
+12/09/2025 15:09:33 - INFO - __main__ - Step: 3610 Loss: 10.4007 LR: 0.000300
+12/09/2025 15:09:41 - INFO - __main__ - Step: 3620 Loss: 10.4743 LR: 0.000300
+12/09/2025 15:09:49 - INFO - __main__ - Step: 3630 Loss: 10.4240 LR: 0.000300
+12/09/2025 15:09:58 - INFO - __main__ - Step: 3640 Loss: 10.4337 LR: 0.000300
+12/09/2025 15:10:06 - INFO - __main__ - Step: 3650 Loss: 10.4613 LR: 0.000300
+12/09/2025 15:10:15 - INFO - __main__ - Step: 3660 Loss: 10.3092 LR: 0.000300
+12/09/2025 15:10:23 - INFO - __main__ - Step: 3670 Loss: 10.4424 LR: 0.000300
+12/09/2025 15:10:31 - INFO - __main__ - Step: 3680 Loss: 10.3409 LR: 0.000300
+12/09/2025 15:10:39 - INFO - __main__ - Step: 3690 Loss: 10.3445 LR: 0.000300
+12/09/2025 15:10:47 - INFO - __main__ - Step: 3700 Loss: 10.4723 LR: 0.000300
+12/09/2025 15:10:56 - INFO - __main__ - Step: 3710 Loss: 10.4960 LR: 0.000300
+12/09/2025 15:11:05 - INFO - __main__ - Step: 3720 Loss: 10.4134 LR: 0.000300
+12/09/2025 15:11:13 - INFO - __main__ - Step: 3730 Loss: 10.2951 LR: 0.000300
+12/09/2025 15:11:22 - INFO - __main__ - Step: 3740 Loss: 10.3510 LR: 0.000300
+12/09/2025 15:11:29 - INFO - __main__ - Step: 3750 Loss: 10.3872 LR: 0.000300
+12/09/2025 15:11:36 - INFO - __main__ - Step: 3760 Loss: 10.3660 LR: 0.000300
+12/09/2025 15:11:45 - INFO - __main__ - Step: 3770 Loss: 10.4696 LR: 0.000300
+12/09/2025 15:11:53 - INFO - __main__ - Step: 3780 Loss: 10.5097 LR: 0.000300
+12/09/2025 15:12:02 - INFO - __main__ - Step: 3790 Loss: 10.3678 LR: 0.000300
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1422, in main
+ encoder_hidden_states, cond_embeds = encode_prompt(
+ File "/mnt/Meissonic/train/dataset_utils.py", line 79, in encode_prompt
+ outputs = text_encoder(input_ids=input_ids, return_dict=True)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1932, in forward
+ encoder_outputs = self.encoder(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1100, in forward
+ layer_outputs = layer_module(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/modeling_layers.py", line 94, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 687, in forward
+ self_attention_outputs = self.layer[0](
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 603, in forward
+ attention_output = self.SelfAttention(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 529, in forward
+ scores = torch.matmul(query_states, key_states.transpose(3, 2))
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1717, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1422, in main
+[rank0]: encoder_hidden_states, cond_embeds = encode_prompt(
+[rank0]: File "/mnt/Meissonic/train/dataset_utils.py", line 79, in encode_prompt
+[rank0]: outputs = text_encoder(input_ids=input_ids, return_dict=True)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1932, in forward
+[rank0]: encoder_outputs = self.encoder(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 1100, in forward
+[rank0]: layer_outputs = layer_module(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/modeling_layers.py", line 94, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 687, in forward
+[rank0]: self_attention_outputs = self.layer[0](
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 603, in forward
+[rank0]: attention_output = self.SelfAttention(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 172, in wrapped_func
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py", line 529, in forward
+[rank0]: scores = torch.matmul(query_states, key_states.transpose(3, 2))
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/requirements.txt b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..66f2e09c95c51d0cbe9ea65818c0d8d0982ff0d2
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T14:17:39.551844Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12093763186688"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "ca77dv9hnvwqayawxeokto06mdonx22t"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-core.log b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..44fd07d0bc8833f6606cc4492399ec75cac9cd76
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T14:17:39.622485456Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpv17shkgs/port-73281.txt","pid":73281,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T14:17:39.622942873Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":73281}
+{"time":"2025-12-09T14:17:39.622956653Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-73281-73540-1131859293/socket","Net":"unix"}}
+{"time":"2025-12-09T14:17:39.809839589Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T14:17:39.815956521Z","level":"INFO","msg":"handleInformInit: received","streamId":"fk5kdvzr","id":"1(@)"}
+{"time":"2025-12-09T14:17:39.993579753Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fk5kdvzr","id":"1(@)"}
+{"time":"2025-12-09T15:12:09.52393793Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-internal.log b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..772a69f2f3f3ff59b824f62b485a1eeeac9db259
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T14:17:39.816055654Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T14:17:39.993414647Z","level":"INFO","msg":"stream: created new stream","id":"fk5kdvzr"}
+{"time":"2025-12-09T14:17:39.993490315Z","level":"INFO","msg":"handler: started","stream_id":"fk5kdvzr"}
+{"time":"2025-12-09T14:17:39.993572987Z","level":"INFO","msg":"stream: started","id":"fk5kdvzr"}
+{"time":"2025-12-09T14:17:39.99358997Z","level":"INFO","msg":"writer: started","stream_id":"fk5kdvzr"}
+{"time":"2025-12-09T14:17:39.993593276Z","level":"INFO","msg":"sender: started","stream_id":"fk5kdvzr"}
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug.log b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..09232eecfb7e4555a3c1ffeef602046a3362ee1c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 14:17:39,554 INFO MainThread:73281 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 14:17:39,554 INFO MainThread:73281 [wandb_setup.py:_flush():80] Configure stats pid to 73281
+2025-12-09 14:17:39,554 INFO MainThread:73281 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 14:17:39,554 INFO MainThread:73281 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 14:17:39,554 INFO MainThread:73281 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 14:17:39,555 INFO MainThread:73281 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug.log
+2025-12-09 14:17:39,555 INFO MainThread:73281 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_141739-fk5kdvzr/logs/debug-internal.log
+2025-12-09 14:17:39,555 INFO MainThread:73281 [wandb_init.py:init():841] calling init triggers
+2025-12-09 14:17:39,555 INFO MainThread:73281 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 14:17:39,555 INFO MainThread:73281 [wandb_init.py:init():889] starting backend
+2025-12-09 14:17:39,810 INFO MainThread:73281 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 14:17:39,814 INFO MainThread:73281 [wandb_init.py:init():900] backend started and connected
+2025-12-09 14:17:39,815 INFO MainThread:73281 [wandb_init.py:init():970] updated telemetry
+2025-12-09 14:17:39,820 INFO MainThread:73281 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 14:17:40,361 INFO MainThread:73281 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 14:17:40,486 INFO MainThread:73281 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 14:17:40,486 INFO MainThread:73281 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 14:17:40,486 INFO MainThread:73281 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 14:17:40,487 INFO MainThread:73281 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 14:17:40,491 INFO MainThread:73281 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 14:17:40,492 INFO MainThread:73281 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_141739-fk5kdvzr/run-fk5kdvzr.wandb b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/run-fk5kdvzr.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..66bad48a331f13626a23d69e4c7418893cb91f22
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_141739-fk5kdvzr/run-fk5kdvzr.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5039d1255ddff8897a7b3a47e6ddc6c6b07ca052e55c347f09fd06fba0666dca
+size 1146880
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/files/config.yaml b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9ab3f705d4de82efd2f8538aae53b0802b6b0abe
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/config.yaml
@@ -0,0 +1,301 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ b1w9ewbbbbzrcttqa1g135sn919dk75f:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133355233280"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:05:23.017782Z"
+ writerId: b1w9ewbbbbzrcttqa1g135sn919dk75f
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/files/output.log b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..0ec9ee539add3dfa42a06f96cd221a9eb71bccca
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/output.log
@@ -0,0 +1,18 @@
+12/09/2025 16:05:23 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 16:05:23 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 16:05:23 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/09/2025 16:05:23 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 16:05:23 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 16:05:23 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1687, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 620, in main
+ if sample_path:
+NameError: name 'sample_path' is not defined
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1687, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 620, in main
+[rank0]: if sample_path:
+[rank0]: NameError: name 'sample_path' is not defined
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/files/requirements.txt b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..23823421ac4bca2c8ae3fc2d6bc66dd310ae8744
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:05:23.017782Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133355233280"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "b1w9ewbbbbzrcttqa1g135sn919dk75f"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-summary.json b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d476fc88692f959c7a899096787abbc21a55dbc
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":0,"_wandb":{"runtime":0}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-core.log b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..3c820d56547238c6f6aa51055df6985677e58904
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:05:23.166093053Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpmor8_uof/port-798744.txt","pid":798744,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:05:23.166524827Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":798744}
+{"time":"2025-12-09T16:05:23.166534234Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-798744-799009-3280574891/socket","Net":"unix"}}
+{"time":"2025-12-09T16:05:23.349730714Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.357078081Z","level":"INFO","msg":"handleInformInit: received","streamId":"sefi33g5","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.540330681Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"sefi33g5","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.939027367Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.939066018Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:05:23.939064052Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.939122022Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:05:23.939132926Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-798744-799009-3280574891/socket","Net":"unix"}}
+{"time":"2025-12-09T16:05:24.563153344Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:05:24.563181492Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:05:24.563190061Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-internal.log b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..ed7dac309c7049421005f6129fc60bd287c4e833
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:05:23.35718071Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:05:23.539982439Z","level":"INFO","msg":"stream: created new stream","id":"sefi33g5"}
+{"time":"2025-12-09T16:05:23.540208535Z","level":"INFO","msg":"handler: started","stream_id":"sefi33g5"}
+{"time":"2025-12-09T16:05:23.540324015Z","level":"INFO","msg":"stream: started","id":"sefi33g5"}
+{"time":"2025-12-09T16:05:23.540345419Z","level":"INFO","msg":"writer: started","stream_id":"sefi33g5"}
+{"time":"2025-12-09T16:05:23.540345031Z","level":"INFO","msg":"sender: started","stream_id":"sefi33g5"}
+{"time":"2025-12-09T16:05:23.939076805Z","level":"INFO","msg":"stream: closing","id":"sefi33g5"}
+{"time":"2025-12-09T16:05:24.440688479Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:05:24.558111562Z","level":"INFO","msg":"handler: closed","stream_id":"sefi33g5"}
+{"time":"2025-12-09T16:05:24.558188368Z","level":"INFO","msg":"sender: closed","stream_id":"sefi33g5"}
+{"time":"2025-12-09T16:05:24.558193939Z","level":"INFO","msg":"stream: closed","id":"sefi33g5"}
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug.log b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..526c3d73ac0171a010e8970451182a0e63181075
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_setup.py:_flush():80] Configure stats pid to 798744
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug.log
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_160523-sefi33g5/logs/debug-internal.log
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:05:23,020 INFO MainThread:798744 [wandb_init.py:init():889] starting backend
+2025-12-09 16:05:23,349 INFO MainThread:798744 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:05:23,355 INFO MainThread:798744 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:05:23,356 INFO MainThread:798744 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:05:23,361 INFO MainThread:798744 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:05:23,799 INFO MainThread:798744 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:05:23,924 INFO MainThread:798744 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:05:23,925 INFO MainThread:798744 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:05:23,925 INFO MainThread:798744 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:05:23,925 INFO MainThread:798744 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:05:23,929 INFO MainThread:798744 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:05:23,930 INFO MainThread:798744 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 16:05:23,939 INFO wandb-AsyncioManager-main:798744 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:05:23,939 INFO wandb-AsyncioManager-main:798744 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_160523-sefi33g5/run-sefi33g5.wandb b/Meissonic/wandb/run-20251209_160523-sefi33g5/run-sefi33g5.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..7fd549ac0baaa7a709f06e33567467d01a531670
Binary files /dev/null and b/Meissonic/wandb/run-20251209_160523-sefi33g5/run-sefi33g5.wandb differ
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/files/config.yaml b/Meissonic/wandb/run-20251209_160614-00otapa7/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0eac4fdea382e3a37c4543136575d402f9618b9e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/files/config.yaml
@@ -0,0 +1,301 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ udoaiv9x0g97untt15waac5tzywks3cq:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133355352064"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:06:14.295708Z"
+ writerId: udoaiv9x0g97untt15waac5tzywks3cq
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/files/output.log b/Meissonic/wandb/run-20251209_160614-00otapa7/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..69575f236ed6b937da5d3d7939b6aef16053b720
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/files/output.log
@@ -0,0 +1,54 @@
+12/09/2025 16:06:15 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 16:06:15 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 16:06:15 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/09/2025 16:06:15 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 16:06:15 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 16:06:15 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 16:06:15 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 16:06:15 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:06:15 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:06:31 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:06:31 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:06:34 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:06:36 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:06:36 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 16:06:36 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:06:36 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:06:36 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 16:06:36 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 16:06:36 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 16:06:36 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 16:06:36 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 16:06:36 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/09/2025 16:06:36 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:06:36 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:06:36 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:06:36 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:06:36 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:06:36 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:06:37 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 16:06:37 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 16:06:37 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:06:37 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:06:37 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 16:06:37 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 8
+12/09/2025 16:06:37 - INFO - __main__ - Gradient Accumulation steps = 1
+[DEBUG] video_tokens: shape=torch.Size([1, 5, 60, 106]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/09/2025 16:06:38 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1687, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1493, in main
+ loss = F.cross_entropy(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+ return torch._C._nn.cross_entropy_loss(
+NotImplementedError: "nll_loss_forward_reduce_cuda_kernel_2d_index" not implemented for 'Int'
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1687, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1493, in main
+[rank0]: loss = F.cross_entropy(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+[rank0]: return torch._C._nn.cross_entropy_loss(
+[rank0]: NotImplementedError: "nll_loss_forward_reduce_cuda_kernel_2d_index" not implemented for 'Int'
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/files/requirements.txt b/Meissonic/wandb/run-20251209_160614-00otapa7/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..e2fd3275739a2894d9a3710f50005819f7cb05a5
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:06:14.295708Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133355352064"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "udoaiv9x0g97untt15waac5tzywks3cq"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-summary.json b/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..6735d66125b699c0dbd8ec56cf36ff05c01fa59e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":68,"_wandb":{"runtime":68}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-core.log b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..a159cee46270a7fb58bfbb4cc369ab914f8edbbb
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:06:14.367769937Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpqm293gl8/port-799765.txt","pid":799765,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:06:14.368350355Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":799765}
+{"time":"2025-12-09T16:06:14.368359779Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-799765-800014-3124456346/socket","Net":"unix"}}
+{"time":"2025-12-09T16:06:14.554246528Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:06:14.560223528Z","level":"INFO","msg":"handleInformInit: received","streamId":"00otapa7","id":"1(@)"}
+{"time":"2025-12-09T16:06:14.727400643Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"00otapa7","id":"1(@)"}
+{"time":"2025-12-09T16:07:23.851592211Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:07:23.851675183Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:07:23.851702883Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:07:23.851817322Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:07:23.852027714Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-799765-800014-3124456346/socket","Net":"unix"}}
+{"time":"2025-12-09T16:07:24.186884958Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:07:24.186921655Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:07:24.186934204Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-internal.log b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..f8767d43597ab3f3ef82f1e1c5e47c3492367fb2
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:06:14.56032165Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:06:14.727184038Z","level":"INFO","msg":"stream: created new stream","id":"00otapa7"}
+{"time":"2025-12-09T16:06:14.727271225Z","level":"INFO","msg":"handler: started","stream_id":"00otapa7"}
+{"time":"2025-12-09T16:06:14.727394239Z","level":"INFO","msg":"stream: started","id":"00otapa7"}
+{"time":"2025-12-09T16:06:14.727432914Z","level":"INFO","msg":"sender: started","stream_id":"00otapa7"}
+{"time":"2025-12-09T16:06:14.727432154Z","level":"INFO","msg":"writer: started","stream_id":"00otapa7"}
+{"time":"2025-12-09T16:07:23.851661162Z","level":"INFO","msg":"stream: closing","id":"00otapa7"}
+{"time":"2025-12-09T16:07:24.081209759Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:07:24.183666884Z","level":"INFO","msg":"handler: closed","stream_id":"00otapa7"}
+{"time":"2025-12-09T16:07:24.183798562Z","level":"INFO","msg":"sender: closed","stream_id":"00otapa7"}
+{"time":"2025-12-09T16:07:24.183810463Z","level":"INFO","msg":"stream: closed","id":"00otapa7"}
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug.log b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..da47719c3e20217832fe8b45fe77e258a9adc5da
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_setup.py:_flush():80] Configure stats pid to 799765
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:06:14,298 INFO MainThread:799765 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug.log
+2025-12-09 16:06:14,299 INFO MainThread:799765 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_160614-00otapa7/logs/debug-internal.log
+2025-12-09 16:06:14,299 INFO MainThread:799765 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:06:14,299 INFO MainThread:799765 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:06:14,299 INFO MainThread:799765 [wandb_init.py:init():889] starting backend
+2025-12-09 16:06:14,554 INFO MainThread:799765 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:06:14,558 INFO MainThread:799765 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:06:14,560 INFO MainThread:799765 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:06:14,564 INFO MainThread:799765 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:06:14,955 INFO MainThread:799765 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:06:15,080 INFO MainThread:799765 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:06:15,081 INFO MainThread:799765 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:06:15,081 INFO MainThread:799765 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:06:15,081 INFO MainThread:799765 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:06:15,084 INFO MainThread:799765 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:06:15,085 INFO MainThread:799765 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 16:07:23,851 INFO wandb-AsyncioManager-main:799765 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:07:23,852 INFO wandb-AsyncioManager-main:799765 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_160614-00otapa7/run-00otapa7.wandb b/Meissonic/wandb/run-20251209_160614-00otapa7/run-00otapa7.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..d9c82880dd9ad2a6f4fc9eaa33db11173e79510d
Binary files /dev/null and b/Meissonic/wandb/run-20251209_160614-00otapa7/run-00otapa7.wandb differ
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/config.yaml b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ddf814eef927384674503e03a98ef97b062bae7e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/config.yaml
@@ -0,0 +1,301 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ jp7apqd2y7dvf73te9x78rw0nrf3puvr:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133355528192"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:10:19.255348Z"
+ writerId: jp7apqd2y7dvf73te9x78rw0nrf3puvr
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/output.log b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..1c025b9f2312f7d45591ff20a2cd310ccab23c7c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/output.log
@@ -0,0 +1,155 @@
+12/09/2025 16:10:20 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 16:10:20 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 16:10:20 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/09/2025 16:10:20 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 16:10:20 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 16:10:20 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 16:10:20 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 16:10:20 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:10:20 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:10:36 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:10:36 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:10:38 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:10:40 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:10:40 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 16:10:40 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:10:40 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:10:40 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 16:10:40 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 16:10:40 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 16:10:40 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 16:10:40 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 16:10:40 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/09/2025 16:10:40 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:10:40 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:10:40 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:10:40 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:10:40 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:10:40 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:10:42 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 16:10:42 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 16:10:42 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:10:42 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:10:42 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 16:10:42 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 8
+12/09/2025 16:10:42 - INFO - __main__ - Gradient Accumulation steps = 1
+[DEBUG] video_tokens: shape=torch.Size([1, 5, 60, 106]), dtype=torch.int64, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/09/2025 16:10:43 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1689, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1506, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1154, in unpack_hook
+ _run_fn_with_dynamo_disabled(frame.recompute_fn, *args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1124, in _run_fn_with_dynamo_disabled
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1555, in recompute_fn
+ fn(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+ x = cross_attn_ffn(x, context, context_lens, e)
+ File "/mnt/Meissonic/src/transformer_video.py", line 484, in cross_attn_ffn
+ y = self.ffn(ffn_input)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
+ input = module(input)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/activation.py", line 816, in forward
+ return F.gelu(input, approximate=self.approximate)
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 136.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 75.56 MiB is free. Including non-PyTorch memory, this process has 39.41 GiB memory in use. Of the allocated memory 38.15 GiB is allocated by PyTorch, and 155.83 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1689, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1506, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1154, in unpack_hook
+[rank0]: _run_fn_with_dynamo_disabled(frame.recompute_fn, *args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1124, in _run_fn_with_dynamo_disabled
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1555, in recompute_fn
+[rank0]: fn(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+[rank0]: x = cross_attn_ffn(x, context, context_lens, e)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 484, in cross_attn_ffn
+[rank0]: y = self.ffn(ffn_input)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
+[rank0]: input = module(input)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/activation.py", line 816, in forward
+[rank0]: return F.gelu(input, approximate=self.approximate)
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 136.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 75.56 MiB is free. Including non-PyTorch memory, this process has 39.41 GiB memory in use. Of the allocated memory 38.15 GiB is allocated by PyTorch, and 155.83 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/requirements.txt b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8d73b037961f425dbbc35e44b5dd3a9ef4facd3
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:10:19.255348Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133355528192"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "jp7apqd2y7dvf73te9x78rw0nrf3puvr"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-summary.json b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..c2c05989968aab8f861b59f70eec7de82b16820e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":52},"_runtime":52}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-core.log b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..a0a9163d675dd0d4341699e0397ce22914c33549
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:10:19.325772351Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmplk9fg4nn/port-804202.txt","pid":804202,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:10:19.326353613Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":804202}
+{"time":"2025-12-09T16:10:19.326328884Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-804202-804449-3058100521/socket","Net":"unix"}}
+{"time":"2025-12-09T16:10:19.512203624Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:10:19.518374741Z","level":"INFO","msg":"handleInformInit: received","streamId":"ca5tv9y1","id":"1(@)"}
+{"time":"2025-12-09T16:10:19.684904074Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ca5tv9y1","id":"1(@)"}
+{"time":"2025-12-09T16:11:12.802727993Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:11:12.802814833Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:11:12.802800761Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:11:12.802931646Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:11:12.802930719Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-804202-804449-3058100521/socket","Net":"unix"}}
+{"time":"2025-12-09T16:11:13.183150726Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:11:13.183170626Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:11:13.18317985Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-internal.log b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..cfa4cb962d701e9ec9fff00e28eeb19bc3126a76
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:10:19.518476001Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:10:19.684672449Z","level":"INFO","msg":"stream: created new stream","id":"ca5tv9y1"}
+{"time":"2025-12-09T16:10:19.684757371Z","level":"INFO","msg":"handler: started","stream_id":"ca5tv9y1"}
+{"time":"2025-12-09T16:10:19.684895948Z","level":"INFO","msg":"stream: started","id":"ca5tv9y1"}
+{"time":"2025-12-09T16:10:19.684911695Z","level":"INFO","msg":"writer: started","stream_id":"ca5tv9y1"}
+{"time":"2025-12-09T16:10:19.684914721Z","level":"INFO","msg":"sender: started","stream_id":"ca5tv9y1"}
+{"time":"2025-12-09T16:11:12.802806611Z","level":"INFO","msg":"stream: closing","id":"ca5tv9y1"}
+{"time":"2025-12-09T16:11:13.06038557Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:11:13.180327674Z","level":"INFO","msg":"handler: closed","stream_id":"ca5tv9y1"}
+{"time":"2025-12-09T16:11:13.180418814Z","level":"INFO","msg":"sender: closed","stream_id":"ca5tv9y1"}
+{"time":"2025-12-09T16:11:13.180425983Z","level":"INFO","msg":"stream: closed","id":"ca5tv9y1"}
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug.log b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..014320019c61c76e7fbd692d9b199538482b5751
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_setup.py:_flush():80] Configure stats pid to 804202
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug.log
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_161019-ca5tv9y1/logs/debug-internal.log
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:10:19,258 INFO MainThread:804202 [wandb_init.py:init():889] starting backend
+2025-12-09 16:10:19,512 INFO MainThread:804202 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:10:19,516 INFO MainThread:804202 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:10:19,518 INFO MainThread:804202 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:10:19,522 INFO MainThread:804202 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:10:20,086 INFO MainThread:804202 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:10:20,211 INFO MainThread:804202 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:10:20,211 INFO MainThread:804202 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:10:20,211 INFO MainThread:804202 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:10:20,211 INFO MainThread:804202 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:10:20,214 INFO MainThread:804202 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:10:20,215 INFO MainThread:804202 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 16:11:12,802 INFO wandb-AsyncioManager-main:804202 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:11:12,803 INFO wandb-AsyncioManager-main:804202 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_161019-ca5tv9y1/run-ca5tv9y1.wandb b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/run-ca5tv9y1.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..883786c305882e782e813676c57a238a64ce3ef8
Binary files /dev/null and b/Meissonic/wandb/run-20251209_161019-ca5tv9y1/run-ca5tv9y1.wandb differ
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/files/config.yaml b/Meissonic/wandb/run-20251209_161324-onokk16i/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..243c66905f46c26266b0a583226f49f43a54d59e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/files/config.yaml
@@ -0,0 +1,301 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ efbkiwa6x1upmhno4asmii8gb5l9pte0:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "1"
+ - --num_frames
+ - "4"
+ - --video_height
+ - "256"
+ - --video_width
+ - "448"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "1"
+ - --gradient_accumulation_steps
+ - "1"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133355716608"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:13:24.569020Z"
+ writerId: efbkiwa6x1upmhno4asmii8gb5l9pte0
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 1
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 4
+output_dir:
+ value: ./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 1
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 448
+wan_backbone_lr_ratio:
+ value: 1
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/files/output.log b/Meissonic/wandb/run-20251209_161324-onokk16i/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..008ac1dbee8a494dfbb8183d6c6c5c3bbc2a28d0
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/files/output.log
@@ -0,0 +1,142 @@
+12/09/2025 16:13:25 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/09/2025 16:13:25 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/09/2025 16:13:25 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/09/2025 16:13:25 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/09/2025 16:13:25 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/09/2025 16:13:25 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=60, W'=106
+12/09/2025 16:13:25 - INFO - __main__ - Got text_dim from metadata: 4096
+12/09/2025 16:13:25 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:13:26 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:13:42 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:13:42 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:13:44 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:13:46 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:13:46 - INFO - __main__ - Wan backbone lr = 0.000300 (base_lr * 1.0)
+12/09/2025 16:13:46 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:13:46 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:13:46 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/09/2025 16:13:46 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/09/2025 16:13:46 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/09/2025 16:13:46 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/09/2025 16:13:46 - INFO - train.dataset_utils - Index range: 0 to 127
+12/09/2025 16:13:46 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/09/2025 16:13:46 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:13:46 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:13:46 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:13:46 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:13:46 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:13:46 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:13:48 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/09/2025 16:13:48 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/09/2025 16:13:48 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:13:48 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:13:48 - INFO - __main__ - Instantaneous batch size per device = 1
+12/09/2025 16:13:48 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 8
+12/09/2025 16:13:48 - INFO - __main__ - Gradient Accumulation steps = 1
+[DEBUG] video_tokens: shape=torch.Size([1, 5, 60, 106]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/09/2025 16:13:49 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1154, in unpack_hook
+ _run_fn_with_dynamo_disabled(frame.recompute_fn, *args)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1124, in _run_fn_with_dynamo_disabled
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1555, in recompute_fn
+ fn(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+ x = cross_attn_ffn(x, context, context_lens, e)
+ File "/mnt/Meissonic/src/transformer_video.py", line 484, in cross_attn_ffn
+ y = self.ffn(ffn_input)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
+ input = module(input)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/activation.py", line 816, in forward
+ return F.gelu(input, approximate=self.approximate)
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 136.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 75.56 MiB is free. Including non-PyTorch memory, this process has 39.41 GiB memory in use. Of the allocated memory 38.15 GiB is allocated by PyTorch, and 156.07 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1154, in unpack_hook
+[rank0]: _run_fn_with_dynamo_disabled(frame.recompute_fn, *args)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1124, in _run_fn_with_dynamo_disabled
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 1555, in recompute_fn
+[rank0]: fn(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+[rank0]: x = cross_attn_ffn(x, context, context_lens, e)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 484, in cross_attn_ffn
+[rank0]: y = self.ffn(ffn_input)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward
+[rank0]: input = module(input)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/activation.py", line 816, in forward
+[rank0]: return F.gelu(input, approximate=self.approximate)
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 136.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 75.56 MiB is free. Including non-PyTorch memory, this process has 39.41 GiB memory in use. Of the allocated memory 38.15 GiB is allocated by PyTorch, and 156.07 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/files/requirements.txt b/Meissonic/wandb/run-20251209_161324-onokk16i/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..dd6f0336b28041ab148e75118b6586d05615d6b1
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:13:24.569020Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "1",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "1",
+ "--gradient_accumulation_steps",
+ "1",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133355716608"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "efbkiwa6x1upmhno4asmii8gb5l9pte0"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-summary.json b/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..6735d66125b699c0dbd8ec56cf36ff05c01fa59e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":68,"_wandb":{"runtime":68}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-core.log b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..325943e78d5efa6fb978d29aa2beb95833274567
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:13:24.638327764Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpqmklcgtg/port-807619.txt","pid":807619,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:13:24.639641115Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":807619}
+{"time":"2025-12-09T16:13:24.639636363Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-807619-807867-2491286150/socket","Net":"unix"}}
+{"time":"2025-12-09T16:13:24.824846418Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:13:24.831086373Z","level":"INFO","msg":"handleInformInit: received","streamId":"onokk16i","id":"1(@)"}
+{"time":"2025-12-09T16:13:25.002141481Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"onokk16i","id":"1(@)"}
+{"time":"2025-12-09T16:14:33.950717019Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:14:33.95095849Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:14:33.951019432Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:14:33.951026191Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:14:33.95109025Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-807619-807867-2491286150/socket","Net":"unix"}}
+{"time":"2025-12-09T16:14:34.306036996Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:14:34.306059112Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:14:34.306068898Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-internal.log b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..b50ed98ebd95a50e243e925ce9c0e22e10921834
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:13:24.831225512Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:13:25.001968424Z","level":"INFO","msg":"stream: created new stream","id":"onokk16i"}
+{"time":"2025-12-09T16:13:25.002047757Z","level":"INFO","msg":"handler: started","stream_id":"onokk16i"}
+{"time":"2025-12-09T16:13:25.002133131Z","level":"INFO","msg":"stream: started","id":"onokk16i"}
+{"time":"2025-12-09T16:13:25.002148909Z","level":"INFO","msg":"writer: started","stream_id":"onokk16i"}
+{"time":"2025-12-09T16:13:25.002150129Z","level":"INFO","msg":"sender: started","stream_id":"onokk16i"}
+{"time":"2025-12-09T16:14:33.950793808Z","level":"INFO","msg":"stream: closing","id":"onokk16i"}
+{"time":"2025-12-09T16:14:34.204038017Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:14:34.303757866Z","level":"INFO","msg":"handler: closed","stream_id":"onokk16i"}
+{"time":"2025-12-09T16:14:34.303882509Z","level":"INFO","msg":"sender: closed","stream_id":"onokk16i"}
+{"time":"2025-12-09T16:14:34.303890895Z","level":"INFO","msg":"stream: closed","id":"onokk16i"}
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug.log b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..c36c76d2a43212ba68b19039f15926c3ca751a10
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_setup.py:_flush():80] Configure stats pid to 807619
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug.log
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_161324-onokk16i/logs/debug-internal.log
+2025-12-09 16:13:24,571 INFO MainThread:807619 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:13:24,572 INFO MainThread:807619 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:13:24,572 INFO MainThread:807619 [wandb_init.py:init():889] starting backend
+2025-12-09 16:13:24,825 INFO MainThread:807619 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:13:24,829 INFO MainThread:807619 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:13:24,830 INFO MainThread:807619 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:13:24,835 INFO MainThread:807619 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:13:25,804 INFO MainThread:807619 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:13:25,926 INFO MainThread:807619 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:13:25,926 INFO MainThread:807619 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:13:25,926 INFO MainThread:807619 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:13:25,926 INFO MainThread:807619 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:13:25,929 INFO MainThread:807619 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:13:25,930 INFO MainThread:807619 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio_continue_tmp', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 1, 'gradient_accumulation_steps': 1, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 1.0, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-09 16:14:33,949 INFO wandb-AsyncioManager-main:807619 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:14:33,950 INFO wandb-AsyncioManager-main:807619 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_161324-onokk16i/run-onokk16i.wandb b/Meissonic/wandb/run-20251209_161324-onokk16i/run-onokk16i.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c0fc332fc4393eb42b75cd6642bf86b650bf0a4f
Binary files /dev/null and b/Meissonic/wandb/run-20251209_161324-onokk16i/run-onokk16i.wandb differ
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/files/output.log b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..743119193f60c61acd582f9cd7c04f59b3db82e1
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/output.log
@@ -0,0 +1,62 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 66.82it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 16:16:48 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4936.13it/s]
+12/09/2025 16:16:50 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=2, H'=32, W'=56
+12/09/2025 16:16:50 - INFO - __main__ - Theoretical dimensions: F'=1, H'=32, W'=56
+12/09/2025 16:16:50 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:16:50 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:17:06 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:17:06 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:17:07 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:17:10 - INFO - __main__ - Wan backbone parameters are frozen (requires_grad=False)
+12/09/2025 16:17:10 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:17:10 - INFO - __main__ - Wan backbone is frozen (lr=0)
+12/09/2025 16:17:10 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:17:10 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:17:17 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 16:17:17 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 16:17:17 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:17:17 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:17:17 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:17:17 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:17:17 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:17:17 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:17:34 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:17:34 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:17:34 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 16:17:34 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 16:17:34 - INFO - __main__ - Gradient Accumulation steps = 4
+12/09/2025 16:18:08 - INFO - __main__ - Step: 10 Loss: 11.1675 LR: 0.000000
+12/09/2025 16:18:33 - INFO - __main__ - Step: 20 Loss: 11.2165 LR: 0.000000
+12/09/2025 16:18:59 - INFO - __main__ - Step: 30 Loss: 11.2436 LR: 0.000000
+12/09/2025 16:19:25 - INFO - __main__ - Step: 40 Loss: 11.1650 LR: 0.000000
+12/09/2025 16:19:50 - INFO - __main__ - Step: 50 Loss: 11.2041 LR: 0.000000
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/files/requirements.txt b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..df3be0f2fac448749760ea44e3c51b2892ed13dd
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/files/wandb-metadata.json
@@ -0,0 +1,154 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:16:44.765090Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--freeze_wan_backbone",
+ "--wan_backbone_lr_ratio",
+ "0.0",
+ "--num_frames",
+ "4",
+ "--video_height",
+ "256",
+ "--video_width",
+ "448",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133355880448"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "3192wi3a06uq5u8scltoogaxc8idi9af"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-core.log b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..47d904de5f67ea31c9659907f58f11e6e155e3a7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T16:16:44.836517546Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpr2ku7q4m/port-811188.txt","pid":811188,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:16:44.837153974Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":811188}
+{"time":"2025-12-09T16:16:44.837131302Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-811188-811451-2759201103/socket","Net":"unix"}}
+{"time":"2025-12-09T16:16:45.022783266Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:16:45.028720923Z","level":"INFO","msg":"handleInformInit: received","streamId":"bp798sa7","id":"1(@)"}
+{"time":"2025-12-09T16:16:45.198649239Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bp798sa7","id":"1(@)"}
+{"time":"2025-12-09T16:20:14.001036771Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-internal.log b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..2ba5c55b7ea6b2d4ce753404d9887269b5e44f0b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T16:16:45.028818035Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:16:45.198406717Z","level":"INFO","msg":"stream: created new stream","id":"bp798sa7"}
+{"time":"2025-12-09T16:16:45.198506901Z","level":"INFO","msg":"handler: started","stream_id":"bp798sa7"}
+{"time":"2025-12-09T16:16:45.198641791Z","level":"INFO","msg":"stream: started","id":"bp798sa7"}
+{"time":"2025-12-09T16:16:45.1986524Z","level":"INFO","msg":"writer: started","stream_id":"bp798sa7"}
+{"time":"2025-12-09T16:16:45.198662453Z","level":"INFO","msg":"sender: started","stream_id":"bp798sa7"}
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug.log b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..7ae08460627b46617a54bdfdb270c3685edcd78c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_setup.py:_flush():80] Configure stats pid to 811188
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug.log
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_161644-bp798sa7/logs/debug-internal.log
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:16:44,768 INFO MainThread:811188 [wandb_init.py:init():889] starting backend
+2025-12-09 16:16:45,023 INFO MainThread:811188 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:16:45,027 INFO MainThread:811188 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:16:45,028 INFO MainThread:811188 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:16:45,032 INFO MainThread:811188 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:16:45,622 INFO MainThread:811188 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:16:45,751 INFO MainThread:811188 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:16:45,751 INFO MainThread:811188 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:16:45,752 INFO MainThread:811188 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:16:45,752 INFO MainThread:811188 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:16:45,755 INFO MainThread:811188 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:16:45,756 INFO MainThread:811188 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_256x448_4f_2bs_4*8*8vqvae_0_00_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 4, 'video_height': 256, 'video_width': 448, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': True, 'wan_backbone_lr_ratio': 0.0, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_161644-bp798sa7/run-bp798sa7.wandb b/Meissonic/wandb/run-20251209_161644-bp798sa7/run-bp798sa7.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..5412808f967bb22f92b499679d77de8484ad33d1
Binary files /dev/null and b/Meissonic/wandb/run-20251209_161644-bp798sa7/run-bp798sa7.wandb differ
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/files/config.yaml b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..901989dc1946d790939451b73f93185bb6722b8c
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/config.yaml
@@ -0,0 +1,298 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ p7qmb4hazkd40rqjeu38ugero566r4ft:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "16"
+ - --video_height
+ - "180"
+ - --video_width
+ - "320"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133356040192"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:21:04.467894Z"
+ writerId: p7qmb4hazkd40rqjeu38ugero566r4ft
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: null
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 16
+output_dir:
+ value: ./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 180
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 320
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/files/output.log b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..ecef062613a57610b4deda9404546704300884e2
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/output.log
@@ -0,0 +1,125 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 70.72it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 16:21:07 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4878.72it/s]
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 632, in main
+ dummy_tokens = video_tokenizer.encode(dummy_video) # [1, F', H', W']
+ File "/mnt/Meissonic/src/pipeline_video.py", line 181, in encode
+ result = self.encoder(video)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+RuntimeError: The following operation failed in the TorchScript interpreter.
+Traceback of TorchScript, serialized code (most recent call last):
+ File "code/__torch__/torch/nn/modules/container/___torch_mangle_13750.py", line 14, in forward
+ quant_conv = self.quant_conv
+ encoder = self.encoder
+ _0 = (quant_conv).forward((encoder).forward(input, ), )
+ ~~~~~~~~~~~~~~~~ <--- HERE
+ _1, _2, _3, = (quantizer).forward(_0, )
+ return (_1, _2, _3)
+ File "code/__torch__/projects/edify_tokenizer/v1/module/layers3d/___torch_mangle_13744.py", line 56, in forward
+ _5 = (conv_in).forward((patcher3d).forward(input, ), )
+ _6 = (_14).forward((_04).forward(_5, ), )
+ _7 = (_00).forward((downsample0).forward(_6, ), )
+ ~~~~~~~~~~~~~~~~~~~~ <--- HERE
+ _3 = (_12).forward(_7, )
+ _4 = (downsample).forward()
+ File "code/__torch__/projects/edify_tokenizer/v1/module/layers3d/___torch_mangle_13597.py", line 16, in forward
+ _0 = (conv1).forward(x, )
+ x2 = torch.avg_pool3d(x, [1, 2, 2], [1, 2, 2], [0, 0, 0])
+ x0 = torch.add(_0, x2)
+ ~~~~~~~~~ <--- HERE
+ return (conv3).forward(x0, )
+
+Traceback of TorchScript, original code (most recent call last):
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/module/layers3d.py(193): forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/module/layers3d.py(680): forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py(218): forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(1274): trace_module
+/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(694): _trace_impl
+/usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(999): trace
+/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py(574): _fn
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/checkpointer.py(270): _get_ema_jit
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/checkpointer.py(94): save
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/trainer.py(139): train
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/scripts/train.py(44): launch
+/usr/local/lib/python3.10/dist-packages/loguru/_logger.py(1277): catch_wrapper
+/lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/scripts/train.py(80):
+/usr/lib/python3.10/runpy.py(86): _run_code
+/usr/lib/python3.10/runpy.py(196): _run_module_as_main
+RuntimeError: The size of tensor a (22) must match the size of tensor b (23) at non-singleton dimension 3
+
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 632, in main
+[rank0]: dummy_tokens = video_tokenizer.encode(dummy_video) # [1, F', H', W']
+[rank0]: File "/mnt/Meissonic/src/pipeline_video.py", line 181, in encode
+[rank0]: result = self.encoder(video)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: RuntimeError: The following operation failed in the TorchScript interpreter.
+[rank0]: Traceback of TorchScript, serialized code (most recent call last):
+[rank0]: File "code/__torch__/torch/nn/modules/container/___torch_mangle_13750.py", line 14, in forward
+[rank0]: quant_conv = self.quant_conv
+[rank0]: encoder = self.encoder
+[rank0]: _0 = (quant_conv).forward((encoder).forward(input, ), )
+[rank0]: ~~~~~~~~~~~~~~~~ <--- HERE
+[rank0]: _1, _2, _3, = (quantizer).forward(_0, )
+[rank0]: return (_1, _2, _3)
+[rank0]: File "code/__torch__/projects/edify_tokenizer/v1/module/layers3d/___torch_mangle_13744.py", line 56, in forward
+[rank0]: _5 = (conv_in).forward((patcher3d).forward(input, ), )
+[rank0]: _6 = (_14).forward((_04).forward(_5, ), )
+[rank0]: _7 = (_00).forward((downsample0).forward(_6, ), )
+[rank0]: ~~~~~~~~~~~~~~~~~~~~ <--- HERE
+[rank0]: _3 = (_12).forward(_7, )
+[rank0]: _4 = (downsample).forward()
+[rank0]: File "code/__torch__/projects/edify_tokenizer/v1/module/layers3d/___torch_mangle_13597.py", line 16, in forward
+[rank0]: _0 = (conv1).forward(x, )
+[rank0]: x2 = torch.avg_pool3d(x, [1, 2, 2], [1, 2, 2], [0, 0, 0])
+[rank0]: x0 = torch.add(_0, x2)
+[rank0]: ~~~~~~~~~ <--- HERE
+[rank0]: return (conv3).forward(x0, )
+
+[rank0]: Traceback of TorchScript, original code (most recent call last):
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/module/layers3d.py(193): forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/module/layers3d.py(680): forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/container.py(218): forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1542): _slow_forward
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1561): _call_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py(1552): _wrapped_call_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(1274): trace_module
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(694): _trace_impl
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/jit/_trace.py(999): trace
+[rank0]: /usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py(574): _fn
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/checkpointer.py(270): _get_ema_jit
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/checkpointer.py(94): save
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/projects/edify_tokenizer/v1/trainer.py(139): train
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/scripts/train.py(44): launch
+[rank0]: /usr/local/lib/python3.10/dist-packages/loguru/_logger.py(1277): catch_wrapper
+[rank0]: /lustre/fs3/portfolios/nvr/users/jinweig/edify_tokenizer1/cosmos/DV720_Causal_FSQ_T17_f4x8x8/scripts/train.py(80):
+[rank0]: /usr/lib/python3.10/runpy.py(86): _run_code
+[rank0]: /usr/lib/python3.10/runpy.py(196): _run_module_as_main
+[rank0]: RuntimeError: The size of tensor a (22) must match the size of tensor b (23) at non-singleton dimension 3
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/files/requirements.txt b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c277a624c43877f9766a41027cc20e93775ea73
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:21:04.467894Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "16",
+ "--video_height",
+ "180",
+ "--video_width",
+ "320",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133356040192"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "p7qmb4hazkd40rqjeu38ugero566r4ft"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-summary.json b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..090134c0bee209a0eb6edb4ebf8ccb739b16b3f9
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":3,"_wandb":{"runtime":3}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-core.log b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..b51bac218e7130961e35337c6eed5c7e9a55462b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:21:04.538365948Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp52270int/port-886533.txt","pid":886533,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:21:04.538886732Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":886533}
+{"time":"2025-12-09T16:21:04.538872677Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-886533-886801-2796040377/socket","Net":"unix"}}
+{"time":"2025-12-09T16:21:04.72471488Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:21:04.730804303Z","level":"INFO","msg":"handleInformInit: received","streamId":"bumjum6e","id":"1(@)"}
+{"time":"2025-12-09T16:21:04.901056649Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bumjum6e","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.118999185Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.119058085Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:21:09.119054219Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.119113772Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.119178717Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-886533-886801-2796040377/socket","Net":"unix"}}
+{"time":"2025-12-09T16:21:09.629171629Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.629191641Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:21:09.629200445Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-internal.log b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c77e4535266b1aed21bcc51de67297eba08f8cfc
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:21:04.730922438Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:21:04.900827158Z","level":"INFO","msg":"stream: created new stream","id":"bumjum6e"}
+{"time":"2025-12-09T16:21:04.900930312Z","level":"INFO","msg":"handler: started","stream_id":"bumjum6e"}
+{"time":"2025-12-09T16:21:04.901050385Z","level":"INFO","msg":"stream: started","id":"bumjum6e"}
+{"time":"2025-12-09T16:21:04.901059991Z","level":"INFO","msg":"writer: started","stream_id":"bumjum6e"}
+{"time":"2025-12-09T16:21:04.901067049Z","level":"INFO","msg":"sender: started","stream_id":"bumjum6e"}
+{"time":"2025-12-09T16:21:09.119061953Z","level":"INFO","msg":"stream: closing","id":"bumjum6e"}
+{"time":"2025-12-09T16:21:09.520970004Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:21:09.627220431Z","level":"INFO","msg":"handler: closed","stream_id":"bumjum6e"}
+{"time":"2025-12-09T16:21:09.627324636Z","level":"INFO","msg":"sender: closed","stream_id":"bumjum6e"}
+{"time":"2025-12-09T16:21:09.627331413Z","level":"INFO","msg":"stream: closed","id":"bumjum6e"}
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug.log b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..5dca6b8a923039808b7a4c91e731883ab398c788
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:21:04,470 INFO MainThread:886533 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_setup.py:_flush():80] Configure stats pid to 886533
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug.log
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_162104-bumjum6e/logs/debug-internal.log
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:21:04,471 INFO MainThread:886533 [wandb_init.py:init():889] starting backend
+2025-12-09 16:21:04,724 INFO MainThread:886533 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:21:04,729 INFO MainThread:886533 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:21:04,731 INFO MainThread:886533 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:21:04,736 INFO MainThread:886533 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:21:05,183 INFO MainThread:886533 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:21:05,307 INFO MainThread:886533 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:21:05,307 INFO MainThread:886533 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:21:05,308 INFO MainThread:886533 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:21:05,308 INFO MainThread:886533 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:21:05,310 INFO MainThread:886533 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:21:05,311 INFO MainThread:886533 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 180, 'video_width': 320, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
+2025-12-09 16:21:09,118 INFO wandb-AsyncioManager-main:886533 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:21:09,118 INFO wandb-AsyncioManager-main:886533 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_162104-bumjum6e/run-bumjum6e.wandb b/Meissonic/wandb/run-20251209_162104-bumjum6e/run-bumjum6e.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..5d89b4ae5c12f106ef197a35529f3433371d0ba3
Binary files /dev/null and b/Meissonic/wandb/run-20251209_162104-bumjum6e/run-bumjum6e.wandb differ
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/config.yaml b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..10eb95f5814eadbcafadabdda05da9559013549e
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/config.yaml
@@ -0,0 +1,298 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 81327e92g5gbekeuqlzq67d6vv3i4wr5:
+ args:
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "16"
+ - --video_height
+ - "256"
+ - --video_width
+ - "256"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12133356142592"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-09T16:21:54.995447Z"
+ writerId: 81327e92g5gbekeuqlzq67d6vv3i4wr5
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: null
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 16
+output_dir:
+ value: ./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: false
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 256
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 256
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/output.log b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d7700d99ed807c48078de496272620cb2565dc23
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/output.log
@@ -0,0 +1,69 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 67.51it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 16:21:58 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6184.99it/s]
+12/09/2025 16:22:00 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=5, H'=32, W'=32
+12/09/2025 16:22:00 - INFO - __main__ - Theoretical dimensions: F'=4, H'=32, W'=32
+12/09/2025 16:22:00 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:22:00 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:22:15 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:22:15 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:22:17 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:22:19 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:22:19 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/09/2025 16:22:19 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:22:19 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:22:26 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 16:22:26 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 16:22:26 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:22:26 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:22:26 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:22:26 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:22:26 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:22:26 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:22:45 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:22:45 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:22:45 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 16:22:45 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 16:22:45 - INFO - __main__ - Gradient Accumulation steps = 4
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.44 GiB is free. Process 887552 has 414.00 MiB memory in use. Process 887551 has 414.00 MiB memory in use. Process 887549 has 414.00 MiB memory in use. Process 887553 has 414.00 MiB memory in use. Process 887555 has 414.00 MiB memory in use. Process 887554 has 414.00 MiB memory in use. Process 887550 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 35.16 GiB memory in use. Of the allocated memory 26.13 GiB is allocated by PyTorch, and 7.93 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.44 GiB is free. Process 887552 has 414.00 MiB memory in use. Process 887551 has 414.00 MiB memory in use. Process 887549 has 414.00 MiB memory in use. Process 887553 has 414.00 MiB memory in use. Process 887555 has 414.00 MiB memory in use. Process 887554 has 414.00 MiB memory in use. Process 887550 has 414.00 MiB memory in use. Including non-PyTorch memory, this process has 35.16 GiB memory in use. Of the allocated memory 26.13 GiB is allocated by PyTorch, and 7.93 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/requirements.txt b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..c34dc3d00e9fbad98f96e4cda81008a82ddb3e77
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:21:54.995447Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "16",
+ "--video_height",
+ "256",
+ "--video_width",
+ "256",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133356142592"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "81327e92g5gbekeuqlzq67d6vv3i4wr5"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-summary.json b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..c90edda74f6e5a2a5eb008d0c2f1dd0de08aa4f7
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":82,"_wandb":{"runtime":82}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-core.log b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..08670b71a5df31d16801fb26dd09b43889e9af47
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-09T16:21:55.090725913Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpmbxuxjg6/port-887548.txt","pid":887548,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:21:55.09117577Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":887548}
+{"time":"2025-12-09T16:21:55.091170115Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-887548-887812-3055066833/socket","Net":"unix"}}
+{"time":"2025-12-09T16:21:55.277788729Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:21:55.285108101Z","level":"INFO","msg":"handleInformInit: received","streamId":"cecoyxuk","id":"1(@)"}
+{"time":"2025-12-09T16:21:55.453430312Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"cecoyxuk","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.157064432Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.157157066Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.157213001Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.157169765Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-09T16:23:18.157292313Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-887548-887812-3055066833/socket","Net":"unix"}}
+{"time":"2025-12-09T16:23:18.608002595Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.60802021Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-09T16:23:18.6080287Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-internal.log b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..4e156dc6815f802f12a974277437c03b73be9e66
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-09T16:21:55.285210268Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:21:55.453210882Z","level":"INFO","msg":"stream: created new stream","id":"cecoyxuk"}
+{"time":"2025-12-09T16:21:55.453305081Z","level":"INFO","msg":"handler: started","stream_id":"cecoyxuk"}
+{"time":"2025-12-09T16:21:55.453422234Z","level":"INFO","msg":"stream: started","id":"cecoyxuk"}
+{"time":"2025-12-09T16:21:55.453440215Z","level":"INFO","msg":"writer: started","stream_id":"cecoyxuk"}
+{"time":"2025-12-09T16:21:55.453443538Z","level":"INFO","msg":"sender: started","stream_id":"cecoyxuk"}
+{"time":"2025-12-09T16:23:18.157185906Z","level":"INFO","msg":"stream: closing","id":"cecoyxuk"}
+{"time":"2025-12-09T16:23:18.459780648Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-09T16:23:18.605017145Z","level":"INFO","msg":"handler: closed","stream_id":"cecoyxuk"}
+{"time":"2025-12-09T16:23:18.605075297Z","level":"INFO","msg":"sender: closed","stream_id":"cecoyxuk"}
+{"time":"2025-12-09T16:23:18.605081119Z","level":"INFO","msg":"stream: closed","id":"cecoyxuk"}
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug.log b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..3660bb64e0d22dfffb516098d4c640ffc7b439eb
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_setup.py:_flush():80] Configure stats pid to 887548
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:21:55,000 INFO MainThread:887548 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug.log
+2025-12-09 16:21:55,001 INFO MainThread:887548 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_162154-cecoyxuk/logs/debug-internal.log
+2025-12-09 16:21:55,001 INFO MainThread:887548 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:21:55,001 INFO MainThread:887548 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:21:55,001 INFO MainThread:887548 [wandb_init.py:init():889] starting backend
+2025-12-09 16:21:55,278 INFO MainThread:887548 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:21:55,283 INFO MainThread:887548 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:21:55,285 INFO MainThread:887548 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:21:55,291 INFO MainThread:887548 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:21:55,636 INFO MainThread:887548 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:21:55,819 INFO MainThread:887548 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:21:55,819 INFO MainThread:887548 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:21:55,819 INFO MainThread:887548 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:21:55,819 INFO MainThread:887548 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:21:55,822 INFO MainThread:887548 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:21:55,823 INFO MainThread:887548 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 256, 'video_width': 256, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
+2025-12-09 16:23:18,157 INFO wandb-AsyncioManager-main:887548 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-09 16:23:18,157 INFO wandb-AsyncioManager-main:887548 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251209_162154-cecoyxuk/run-cecoyxuk.wandb b/Meissonic/wandb/run-20251209_162154-cecoyxuk/run-cecoyxuk.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..7d813548e478df7f4f2d787ed24ccb9fcd519a26
Binary files /dev/null and b/Meissonic/wandb/run-20251209_162154-cecoyxuk/run-cecoyxuk.wandb differ
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/files/output.log b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d1a84bc288c1c67690fe139d73be7c59287cf6bf
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/output.log
@@ -0,0 +1,47 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 68.94it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/09/2025 16:23:40 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4862.56it/s]
+12/09/2025 16:23:42 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=5, H'=16, W'=16
+12/09/2025 16:23:42 - INFO - __main__ - Theoretical dimensions: F'=4, H'=16, W'=16
+12/09/2025 16:23:42 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:23:42 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/09/2025 16:23:58 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:23:58 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/09/2025 16:23:59 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/09/2025 16:24:02 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/09/2025 16:24:02 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/09/2025 16:24:02 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/09/2025 16:24:02 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/09/2025 16:24:09 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/09/2025 16:24:09 - INFO - train.dataset_utils - Using decord for video loading
+12/09/2025 16:24:09 - INFO - __main__ - Dataloader configuration:
+12/09/2025 16:24:09 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/09/2025 16:24:09 - INFO - __main__ - - prefetch_factor: 2
+12/09/2025 16:24:09 - INFO - __main__ - - persistent_workers: True
+12/09/2025 16:24:09 - INFO - __main__ - - pin_memory: True
+12/09/2025 16:24:09 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/09/2025 16:24:26 - INFO - __main__ - ***** Running training *****
+12/09/2025 16:24:26 - INFO - __main__ - Num training steps = 10000
+12/09/2025 16:24:26 - INFO - __main__ - Instantaneous batch size per device = 2
+12/09/2025 16:24:26 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/09/2025 16:24:26 - INFO - __main__ - Gradient Accumulation steps = 4
+12/09/2025 16:25:16 - INFO - __main__ - Step: 10 Loss: 11.0758 LR: 0.000060
+12/09/2025 16:25:49 - INFO - __main__ - Step: 20 Loss: 11.0759 LR: 0.000060
+12/09/2025 16:26:23 - INFO - __main__ - Step: 30 Loss: 11.0732 LR: 0.000060
+12/09/2025 16:26:57 - INFO - __main__ - Step: 40 Loss: 11.0707 LR: 0.000060
+12/09/2025 16:27:31 - INFO - __main__ - Step: 50 Loss: 11.0721 LR: 0.000060
+12/09/2025 16:28:04 - INFO - __main__ - Step: 60 Loss: 11.0691 LR: 0.000060
+12/09/2025 16:28:39 - INFO - __main__ - Step: 70 Loss: 11.0651 LR: 0.000060
+12/09/2025 16:29:15 - INFO - __main__ - Step: 80 Loss: 11.0624 LR: 0.000060
+12/09/2025 16:29:51 - INFO - __main__ - Step: 90 Loss: 11.0556 LR: 0.000060
+12/09/2025 16:30:26 - INFO - __main__ - Step: 100 Loss: 11.0429 LR: 0.000060
+12/09/2025 16:31:02 - INFO - __main__ - Step: 110 Loss: 11.0275 LR: 0.000060
+12/09/2025 16:31:38 - INFO - __main__ - Step: 120 Loss: 11.0152 LR: 0.000060
+12/09/2025 16:32:13 - INFO - __main__ - Step: 130 Loss: 10.9992 LR: 0.000060
+12/09/2025 16:32:47 - INFO - __main__ - Step: 140 Loss: 10.9513 LR: 0.000060
+12/09/2025 16:33:22 - INFO - __main__ - Step: 150 Loss: 10.9467 LR: 0.000060
+12/09/2025 16:33:55 - INFO - __main__ - Step: 160 Loss: 10.9003 LR: 0.000060
+12/09/2025 16:34:29 - INFO - __main__ - Step: 170 Loss: 10.9009 LR: 0.000060
+12/09/2025 16:35:03 - INFO - __main__ - Step: 180 Loss: 10.8768 LR: 0.000060
+12/09/2025 16:35:36 - INFO - __main__ - Step: 190 Loss: 10.8298 LR: 0.000060
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/files/requirements.txt b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/files/wandb-metadata.json b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..636026fed6fa8311eb1aee1b7514b4fb81dbed63
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-09T16:23:37.188415Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "16",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133356269568"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "c4qpena2ayixc8utojqhldz85b5zaowt"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-core.log b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..8143ddfc546ad8098d314d3f0af17dbb3c68581a
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-09T16:23:37.257637154Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpi9umj7zm/port-896751.txt","pid":896751,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-09T16:23:37.258144523Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":896751}
+{"time":"2025-12-09T16:23:37.258155302Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-896751-897022-3529873079/socket","Net":"unix"}}
+{"time":"2025-12-09T16:23:37.444237022Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-09T16:23:37.450217059Z","level":"INFO","msg":"handleInformInit: received","streamId":"uv3abozu","id":"1(@)"}
+{"time":"2025-12-09T16:23:37.620567422Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"uv3abozu","id":"1(@)"}
+{"time":"2025-12-09T16:36:38.972940398Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-internal.log b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..2554b4b22c2202bda8508803b3f960f70cdc9ce1
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-09T16:23:37.450366978Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-09T16:23:37.620280029Z","level":"INFO","msg":"stream: created new stream","id":"uv3abozu"}
+{"time":"2025-12-09T16:23:37.620379333Z","level":"INFO","msg":"handler: started","stream_id":"uv3abozu"}
+{"time":"2025-12-09T16:23:37.620559096Z","level":"INFO","msg":"stream: started","id":"uv3abozu"}
+{"time":"2025-12-09T16:23:37.620573354Z","level":"INFO","msg":"sender: started","stream_id":"uv3abozu"}
+{"time":"2025-12-09T16:23:37.620576443Z","level":"INFO","msg":"writer: started","stream_id":"uv3abozu"}
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug.log b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8427b90899ae5ef2a1e6e1b83b7bc369791d8d7f
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_setup.py:_flush():80] Configure stats pid to 896751
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug.log
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251209_162337-uv3abozu/logs/debug-internal.log
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_init.py:init():841] calling init triggers
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-09 16:23:37,191 INFO MainThread:896751 [wandb_init.py:init():889] starting backend
+2025-12-09 16:23:37,444 INFO MainThread:896751 [wandb_init.py:init():892] sending inform_init request
+2025-12-09 16:23:37,448 INFO MainThread:896751 [wandb_init.py:init():900] backend started and connected
+2025-12-09 16:23:37,450 INFO MainThread:896751 [wandb_init.py:init():970] updated telemetry
+2025-12-09 16:23:37,454 INFO MainThread:896751 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-09 16:23:37,893 INFO MainThread:896751 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-09 16:23:38,016 INFO MainThread:896751 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-09 16:23:38,016 INFO MainThread:896751 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-09 16:23:38,016 INFO MainThread:896751 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-09 16:23:38,017 INFO MainThread:896751 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-09 16:23:38,019 INFO MainThread:896751 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-09 16:23:38,020 INFO MainThread:896751 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251209_162337-uv3abozu/run-uv3abozu.wandb b/Meissonic/wandb/run-20251209_162337-uv3abozu/run-uv3abozu.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..4d77d3854a3dde8c50e14d882d2d8fe7690dea6b
--- /dev/null
+++ b/Meissonic/wandb/run-20251209_162337-uv3abozu/run-uv3abozu.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46dd39463ae72c061a5c24857a9368679ddd358bdb34c5750ed1f99cfbb4e7b4
+size 196608
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/output.log b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..4f9c648dc629ce4ba5f330848987b53066014084
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/output.log
@@ -0,0 +1,47 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 66.77it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/10/2025 03:03:29 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 3535.66it/s]
+12/10/2025 03:03:31 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=5, H'=16, W'=16
+12/10/2025 03:03:31 - INFO - __main__ - Theoretical dimensions: F'=4, H'=16, W'=16
+12/10/2025 03:03:31 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:03:31 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 03:03:47 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:03:47 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:03:50 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 03:03:54 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 03:03:54 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 03:03:54 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 03:03:54 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 03:04:01 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/10/2025 03:04:01 - INFO - train.dataset_utils - Using decord for video loading
+12/10/2025 03:04:01 - INFO - __main__ - Dataloader configuration:
+12/10/2025 03:04:01 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 03:04:01 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 03:04:01 - INFO - __main__ - - persistent_workers: True
+12/10/2025 03:04:01 - INFO - __main__ - - pin_memory: True
+12/10/2025 03:04:01 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 03:04:16 - INFO - __main__ - ***** Running training *****
+12/10/2025 03:04:16 - INFO - __main__ - Num training steps = 10000
+12/10/2025 03:04:16 - INFO - __main__ - Instantaneous batch size per device = 2
+12/10/2025 03:04:16 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/10/2025 03:04:16 - INFO - __main__ - Gradient Accumulation steps = 4
+12/10/2025 03:05:04 - INFO - __main__ - Step: 10 Loss: 11.0757 LR: 0.000060
+12/10/2025 03:05:37 - INFO - __main__ - Step: 20 Loss: 11.0758 LR: 0.000060
+12/10/2025 03:06:10 - INFO - __main__ - Step: 30 Loss: 11.0730 LR: 0.000060
+12/10/2025 03:06:43 - INFO - __main__ - Step: 40 Loss: 11.0716 LR: 0.000060
+12/10/2025 03:07:18 - INFO - __main__ - Step: 50 Loss: 11.0721 LR: 0.000060
+12/10/2025 03:07:51 - INFO - __main__ - Step: 60 Loss: 11.0697 LR: 0.000060
+12/10/2025 03:08:26 - INFO - __main__ - Step: 70 Loss: 11.0646 LR: 0.000060
+12/10/2025 03:09:00 - INFO - __main__ - Step: 80 Loss: 11.0623 LR: 0.000060
+12/10/2025 03:09:35 - INFO - __main__ - Step: 90 Loss: 11.0554 LR: 0.000060
+12/10/2025 03:10:10 - INFO - __main__ - Step: 100 Loss: 11.0420 LR: 0.000060
+12/10/2025 03:10:44 - INFO - __main__ - Step: 110 Loss: 11.0280 LR: 0.000060
+12/10/2025 03:11:19 - INFO - __main__ - Step: 120 Loss: 11.0138 LR: 0.000060
+12/10/2025 03:11:55 - INFO - __main__ - Step: 130 Loss: 11.0001 LR: 0.000060
+12/10/2025 03:12:29 - INFO - __main__ - Step: 140 Loss: 10.9526 LR: 0.000060
+12/10/2025 03:13:03 - INFO - __main__ - Step: 150 Loss: 10.9456 LR: 0.000060
+12/10/2025 03:13:38 - INFO - __main__ - Step: 160 Loss: 10.8995 LR: 0.000060
+12/10/2025 03:14:11 - INFO - __main__ - Step: 170 Loss: 10.9012 LR: 0.000060
+12/10/2025 03:14:46 - INFO - __main__ - Step: 180 Loss: 10.8755 LR: 0.000060
+12/10/2025 03:15:21 - INFO - __main__ - Step: 190 Loss: 10.8287 LR: 0.000060
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/requirements.txt b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..e467de2b6e445c266f6430acaf3c7ffef6c4bdaf
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T03:03:25.964354Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "16",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133356830720"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "ielhwboly7v6fm7l89mdg72fej61il8f"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-core.log b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..33894bd41e774fcf74f6ac85769a02956852c006
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-10T03:03:26.03254978Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpn0kc07t9/port-1508493.txt","pid":1508493,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T03:03:26.033003993Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1508493}
+{"time":"2025-12-10T03:03:26.033013765Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1508493-1508740-1211382152/socket","Net":"unix"}}
+{"time":"2025-12-10T03:03:26.21958791Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T03:03:26.225652825Z","level":"INFO","msg":"handleInformInit: received","streamId":"gkrz1ykg","id":"1(@)"}
+{"time":"2025-12-10T03:03:26.396922559Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gkrz1ykg","id":"1(@)"}
+{"time":"2025-12-10T03:16:24.528540964Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-internal.log b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..714399f6694aa425751aceb97f1071fc778de206
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-10T03:03:26.225797144Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T03:03:26.396690435Z","level":"INFO","msg":"stream: created new stream","id":"gkrz1ykg"}
+{"time":"2025-12-10T03:03:26.396798298Z","level":"INFO","msg":"handler: started","stream_id":"gkrz1ykg"}
+{"time":"2025-12-10T03:03:26.396913302Z","level":"INFO","msg":"stream: started","id":"gkrz1ykg"}
+{"time":"2025-12-10T03:03:26.396933074Z","level":"INFO","msg":"writer: started","stream_id":"gkrz1ykg"}
+{"time":"2025-12-10T03:03:26.396935917Z","level":"INFO","msg":"sender: started","stream_id":"gkrz1ykg"}
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug.log b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..85c4623183dd924370dbc28ef2bf38a2ba142c76
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_setup.py:_flush():80] Configure stats pid to 1508493
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug.log
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_030325-gkrz1ykg/logs/debug-internal.log
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_init.py:init():841] calling init triggers
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 03:03:25,967 INFO MainThread:1508493 [wandb_init.py:init():889] starting backend
+2025-12-10 03:03:26,219 INFO MainThread:1508493 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 03:03:26,224 INFO MainThread:1508493 [wandb_init.py:init():900] backend started and connected
+2025-12-10 03:03:26,225 INFO MainThread:1508493 [wandb_init.py:init():970] updated telemetry
+2025-12-10 03:03:26,229 INFO MainThread:1508493 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 03:03:26,692 INFO MainThread:1508493 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 03:03:26,875 INFO MainThread:1508493 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 03:03:26,875 INFO MainThread:1508493 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 03:03:26,875 INFO MainThread:1508493 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 03:03:26,875 INFO MainThread:1508493 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 03:03:26,878 INFO MainThread:1508493 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 03:03:26,879 INFO MainThread:1508493 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 16, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251210_030325-gkrz1ykg/run-gkrz1ykg.wandb b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/run-gkrz1ykg.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..e19bc005dc870403a3d14076efdda2679e154ce0
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_030325-gkrz1ykg/run-gkrz1ykg.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e29557a130e180ecbfc8f8b4eae69e163cce17814b944aa9a281bfd86671e3b
+size 196608
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/files/output.log b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..9117a0dc2579901f049330a882919a2b1e6dc7bc
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/output.log
@@ -0,0 +1,47 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 71.34it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/10/2025 03:27:48 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6659.14it/s]
+12/10/2025 03:27:50 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=5, H'=16, W'=16
+12/10/2025 03:27:50 - INFO - __main__ - Theoretical dimensions: F'=4, H'=16, W'=16
+12/10/2025 03:27:50 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:27:50 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 03:28:05 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:28:05 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:28:07 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 03:28:09 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 03:28:09 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 03:28:09 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 03:28:09 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 03:28:16 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/10/2025 03:28:16 - INFO - train.dataset_utils - Using decord for video loading
+12/10/2025 03:28:16 - INFO - __main__ - Dataloader configuration:
+12/10/2025 03:28:16 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 03:28:16 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 03:28:16 - INFO - __main__ - - persistent_workers: True
+12/10/2025 03:28:16 - INFO - __main__ - - pin_memory: True
+12/10/2025 03:28:16 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 03:28:34 - INFO - __main__ - ***** Running training *****
+12/10/2025 03:28:34 - INFO - __main__ - Num training steps = 10000
+12/10/2025 03:28:34 - INFO - __main__ - Instantaneous batch size per device = 2
+12/10/2025 03:28:34 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/10/2025 03:28:34 - INFO - __main__ - Gradient Accumulation steps = 4
+12/10/2025 03:29:23 - INFO - __main__ - Step: 10 Loss: 11.0751 LR: 0.000060
+12/10/2025 03:29:58 - INFO - __main__ - Step: 20 Loss: 11.0745 LR: 0.000060
+12/10/2025 03:30:33 - INFO - __main__ - Step: 30 Loss: 11.0720 LR: 0.000060
+12/10/2025 03:31:07 - INFO - __main__ - Step: 40 Loss: 11.0713 LR: 0.000060
+12/10/2025 03:31:42 - INFO - __main__ - Step: 50 Loss: 11.0713 LR: 0.000060
+12/10/2025 03:32:15 - INFO - __main__ - Step: 60 Loss: 11.0692 LR: 0.000060
+12/10/2025 03:32:49 - INFO - __main__ - Step: 70 Loss: 11.0663 LR: 0.000060
+12/10/2025 03:33:24 - INFO - __main__ - Step: 80 Loss: 11.0617 LR: 0.000060
+12/10/2025 03:33:59 - INFO - __main__ - Step: 90 Loss: 11.0555 LR: 0.000060
+12/10/2025 03:34:34 - INFO - __main__ - Step: 100 Loss: 11.0448 LR: 0.000060
+12/10/2025 03:35:08 - INFO - __main__ - Step: 110 Loss: 11.0327 LR: 0.000060
+12/10/2025 03:35:43 - INFO - __main__ - Step: 120 Loss: 11.0147 LR: 0.000060
+12/10/2025 03:36:19 - INFO - __main__ - Step: 130 Loss: 10.9994 LR: 0.000060
+12/10/2025 03:36:54 - INFO - __main__ - Step: 140 Loss: 10.9611 LR: 0.000060
+12/10/2025 03:37:28 - INFO - __main__ - Step: 150 Loss: 10.9458 LR: 0.000060
+12/10/2025 03:38:03 - INFO - __main__ - Step: 160 Loss: 10.9011 LR: 0.000060
+12/10/2025 03:38:37 - INFO - __main__ - Step: 170 Loss: 10.9101 LR: 0.000060
+12/10/2025 03:39:12 - INFO - __main__ - Step: 180 Loss: 10.8789 LR: 0.000060
+12/10/2025 03:39:47 - INFO - __main__ - Step: 190 Loss: 10.8317 LR: 0.000060
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/files/requirements.txt b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fa7d522f0916775f6e56b0f75b02c88b41403de
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T03:27:45.069546Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133357502464"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "az0nthtl1eiu5tz12udubozi6ufhx0at"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-core.log b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..404de629459ea1ec9dacd8fe83993b9e719723fe
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-10T03:27:45.136792287Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpo0exnb9c/port-1749619.txt","pid":1749619,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T03:27:45.13733044Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1749619}
+{"time":"2025-12-10T03:27:45.137306574Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1749619-1749884-2107733221/socket","Net":"unix"}}
+{"time":"2025-12-10T03:27:45.323961753Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T03:27:45.329819545Z","level":"INFO","msg":"handleInformInit: received","streamId":"o7so78o8","id":"1(@)"}
+{"time":"2025-12-10T03:27:45.496539195Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"o7so78o8","id":"1(@)"}
+{"time":"2025-12-10T03:40:49.793952726Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-internal.log b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c089e97c3d6ca645dfec69f355ea2c55f39b5e38
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-10T03:27:45.329946027Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T03:27:45.496337243Z","level":"INFO","msg":"stream: created new stream","id":"o7so78o8"}
+{"time":"2025-12-10T03:27:45.496427061Z","level":"INFO","msg":"handler: started","stream_id":"o7so78o8"}
+{"time":"2025-12-10T03:27:45.496531545Z","level":"INFO","msg":"stream: started","id":"o7so78o8"}
+{"time":"2025-12-10T03:27:45.496553803Z","level":"INFO","msg":"writer: started","stream_id":"o7so78o8"}
+{"time":"2025-12-10T03:27:45.496557065Z","level":"INFO","msg":"sender: started","stream_id":"o7so78o8"}
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug.log b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..d39907f0f9df11144958d668f0e58be33b3c034b
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_setup.py:_flush():80] Configure stats pid to 1749619
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug.log
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_032745-o7so78o8/logs/debug-internal.log
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_init.py:init():841] calling init triggers
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 03:27:45,072 INFO MainThread:1749619 [wandb_init.py:init():889] starting backend
+2025-12-10 03:27:45,324 INFO MainThread:1749619 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 03:27:45,328 INFO MainThread:1749619 [wandb_init.py:init():900] backend started and connected
+2025-12-10 03:27:45,329 INFO MainThread:1749619 [wandb_init.py:init():970] updated telemetry
+2025-12-10 03:27:45,334 INFO MainThread:1749619 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 03:27:45,776 INFO MainThread:1749619 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 03:27:45,899 INFO MainThread:1749619 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 03:27:45,899 INFO MainThread:1749619 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 03:27:45,899 INFO MainThread:1749619 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 03:27:45,899 INFO MainThread:1749619 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 03:27:45,902 INFO MainThread:1749619 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 03:27:45,903 INFO MainThread:1749619 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251210_032745-o7so78o8/run-o7so78o8.wandb b/Meissonic/wandb/run-20251210_032745-o7so78o8/run-o7so78o8.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..0dbbecbef1043d72338934caa2db97ec7ef5ddcd
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_032745-o7so78o8/run-o7so78o8.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a6554e67bd7fe4a111c314aea3dfac25180cbc585b55d30390eb0bb11251cda
+size 196608
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_081f6716317316010e23.png b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_081f6716317316010e23.png
new file mode 100644
index 0000000000000000000000000000000000000000..db98e10b357391b0c1aeac2daace217dee9d4ca2
Binary files /dev/null and b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_081f6716317316010e23.png differ
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_da9d6c1a97c444d47f3f.png b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_da9d6c1a97c444d47f3f.png
new file mode 100644
index 0000000000000000000000000000000000000000..3ffe531522b08c3914904d5d356469e67b7d63a7
Binary files /dev/null and b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/media/images/generated_videos_first_frame_500_da9d6c1a97c444d47f3f.png differ
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/output.log b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..43d2d89c79a656dccde7513fc072e25a0db8bee2
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/output.log
@@ -0,0 +1,90 @@
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 67.78it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+12/10/2025 03:53:39 - INFO - __main__ - Loaded text encoder: google/umt5-xxl (d_model=4096)
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5005.14it/s]
+12/10/2025 03:53:41 - INFO - __main__ - Actual compressed dimensions from tokenizer: F'=5, H'=16, W'=16
+12/10/2025 03:53:41 - INFO - __main__ - Theoretical dimensions: F'=4, H'=16, W'=16
+12/10/2025 03:53:41 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:53:41 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 03:53:57 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:53:57 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 03:53:59 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 03:54:01 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 03:54:01 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 03:54:01 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 03:54:01 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 03:54:08 - INFO - train.dataset_utils - Loaded 1019957 video entries from /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+12/10/2025 03:54:08 - INFO - train.dataset_utils - Using decord for video loading
+12/10/2025 03:54:08 - INFO - __main__ - Dataloader configuration:
+12/10/2025 03:54:08 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 03:54:08 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 03:54:08 - INFO - __main__ - - persistent_workers: True
+12/10/2025 03:54:08 - INFO - __main__ - - pin_memory: True
+12/10/2025 03:54:08 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 03:54:25 - INFO - __main__ - ***** Running training *****
+12/10/2025 03:54:25 - INFO - __main__ - Num training steps = 10000
+12/10/2025 03:54:25 - INFO - __main__ - Instantaneous batch size per device = 2
+12/10/2025 03:54:25 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/10/2025 03:54:25 - INFO - __main__ - Gradient Accumulation steps = 4
+12/10/2025 03:55:13 - INFO - __main__ - Step: 10 Loss: 11.0745 LR: 0.000060
+12/10/2025 03:55:48 - INFO - __main__ - Step: 20 Loss: 11.0750 LR: 0.000060
+12/10/2025 03:56:21 - INFO - __main__ - Step: 30 Loss: 11.0715 LR: 0.000060
+12/10/2025 03:56:55 - INFO - __main__ - Step: 40 Loss: 11.0719 LR: 0.000060
+12/10/2025 03:57:29 - INFO - __main__ - Step: 50 Loss: 11.0712 LR: 0.000060
+12/10/2025 03:58:03 - INFO - __main__ - Step: 60 Loss: 11.0689 LR: 0.000060
+12/10/2025 03:58:37 - INFO - __main__ - Step: 70 Loss: 11.0658 LR: 0.000060
+12/10/2025 03:59:11 - INFO - __main__ - Step: 80 Loss: 11.0614 LR: 0.000060
+12/10/2025 03:59:46 - INFO - __main__ - Step: 90 Loss: 11.0535 LR: 0.000060
+12/10/2025 04:00:20 - INFO - __main__ - Step: 100 Loss: 11.0433 LR: 0.000060
+12/10/2025 04:00:54 - INFO - __main__ - Step: 110 Loss: 11.0295 LR: 0.000060
+12/10/2025 04:01:29 - INFO - __main__ - Step: 120 Loss: 11.0128 LR: 0.000060
+12/10/2025 04:02:03 - INFO - __main__ - Step: 130 Loss: 10.9998 LR: 0.000060
+12/10/2025 04:02:37 - INFO - __main__ - Step: 140 Loss: 10.9615 LR: 0.000060
+12/10/2025 04:03:11 - INFO - __main__ - Step: 150 Loss: 10.9471 LR: 0.000060
+12/10/2025 04:03:47 - INFO - __main__ - Step: 160 Loss: 10.8990 LR: 0.000060
+12/10/2025 04:04:21 - INFO - __main__ - Step: 170 Loss: 10.9111 LR: 0.000060
+12/10/2025 04:04:55 - INFO - __main__ - Step: 180 Loss: 10.8777 LR: 0.000060
+12/10/2025 04:05:30 - INFO - __main__ - Step: 190 Loss: 10.8318 LR: 0.000060
+12/10/2025 04:06:05 - INFO - __main__ - Step: 200 Loss: 10.8156 LR: 0.000060
+12/10/2025 04:06:40 - INFO - __main__ - Step: 210 Loss: 10.7797 LR: 0.000060
+12/10/2025 04:07:15 - INFO - __main__ - Step: 220 Loss: 10.7180 LR: 0.000060
+12/10/2025 04:07:49 - INFO - __main__ - Step: 230 Loss: 10.7245 LR: 0.000060
+12/10/2025 04:08:25 - INFO - __main__ - Step: 240 Loss: 10.7025 LR: 0.000060
+12/10/2025 04:08:59 - INFO - __main__ - Step: 250 Loss: 10.6934 LR: 0.000060
+12/10/2025 04:09:35 - INFO - __main__ - Step: 260 Loss: 10.6302 LR: 0.000060
+12/10/2025 04:10:09 - INFO - __main__ - Step: 270 Loss: 10.6235 LR: 0.000060
+12/10/2025 04:10:44 - INFO - __main__ - Step: 280 Loss: 10.5938 LR: 0.000060
+12/10/2025 04:11:19 - INFO - __main__ - Step: 290 Loss: 10.6458 LR: 0.000060
+12/10/2025 04:11:54 - INFO - __main__ - Step: 300 Loss: 10.5509 LR: 0.000060
+12/10/2025 04:12:28 - INFO - __main__ - Step: 310 Loss: 10.5721 LR: 0.000060
+12/10/2025 04:13:03 - INFO - __main__ - Step: 320 Loss: 10.5578 LR: 0.000060
+12/10/2025 04:13:37 - INFO - __main__ - Step: 330 Loss: 10.5018 LR: 0.000060
+12/10/2025 04:14:12 - INFO - __main__ - Step: 340 Loss: 10.5305 LR: 0.000060
+12/10/2025 04:14:46 - INFO - __main__ - Step: 350 Loss: 10.5309 LR: 0.000060
+12/10/2025 04:15:20 - INFO - __main__ - Step: 360 Loss: 10.5047 LR: 0.000060
+12/10/2025 04:15:55 - INFO - __main__ - Step: 370 Loss: 10.4622 LR: 0.000060
+12/10/2025 04:16:30 - INFO - __main__ - Step: 380 Loss: 10.5089 LR: 0.000060
+12/10/2025 04:17:04 - INFO - __main__ - Step: 390 Loss: 10.4511 LR: 0.000060
+12/10/2025 04:17:39 - INFO - __main__ - Step: 400 Loss: 10.4623 LR: 0.000060
+12/10/2025 04:18:14 - INFO - __main__ - Step: 410 Loss: 10.5111 LR: 0.000060
+12/10/2025 04:18:47 - INFO - __main__ - Step: 420 Loss: 10.4446 LR: 0.000060
+12/10/2025 04:19:23 - INFO - __main__ - Step: 430 Loss: 10.4794 LR: 0.000060
+12/10/2025 04:19:57 - INFO - __main__ - Step: 440 Loss: 10.4818 LR: 0.000060
+12/10/2025 04:20:32 - INFO - __main__ - Step: 450 Loss: 10.4768 LR: 0.000060
+12/10/2025 04:21:07 - INFO - __main__ - Step: 460 Loss: 10.4552 LR: 0.000060
+12/10/2025 04:21:42 - INFO - __main__ - Step: 470 Loss: 10.4234 LR: 0.000060
+12/10/2025 04:22:17 - INFO - __main__ - Step: 480 Loss: 10.4355 LR: 0.000060
+12/10/2025 04:22:52 - INFO - __main__ - Step: 490 Loss: 10.4183 LR: 0.000060
+12/10/2025 04:23:26 - INFO - __main__ - Step: 500 Loss: 10.5091 LR: 0.000060
+12/10/2025 04:23:26 - INFO - accelerate.accelerator - Saving current state to output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 04:23:35 - INFO - accelerate.checkpointing - Optimizer state saved in output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 04:23:35 - INFO - accelerate.checkpointing - Scheduler state saved in output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 04:23:35 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 04:23:35 - INFO - accelerate.checkpointing - Random states saved in output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 04:23:35 - INFO - __main__ - Saved state to output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 04:23:35 - INFO - __main__ - Generating videos for validation...
+12/10/2025 04:23:35 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.04it/s]
+12/10/2025 04:23:42 - INFO - __main__ - Validation videos saved to ./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio
+12/10/2025 04:24:19 - INFO - __main__ - Step: 510 Loss: 10.3847 LR: 0.000060
+12/10/2025 04:24:54 - INFO - __main__ - Step: 520 Loss: 10.4455 LR: 0.000060
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/requirements.txt b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a9049e1a096feb538e1e49f30cee5b508b7e2be
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/files/wandb-metadata.json
@@ -0,0 +1,153 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T03:53:36.383134Z",
+ "args": [
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12133357961216"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "pbm2fbj3vcjuj9huqsanqzcnp62wayv1"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-core.log b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..d6ae206e1dba1ca64d735aa67296609f98bfb5e9
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-10T03:53:36.452090255Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpte7_g0so/port-1991718.txt","pid":1991718,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T03:53:36.452554454Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1991718}
+{"time":"2025-12-10T03:53:36.452557571Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1991718-1991975-1642945423/socket","Net":"unix"}}
+{"time":"2025-12-10T03:53:36.638650796Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T03:53:36.644680584Z","level":"INFO","msg":"handleInformInit: received","streamId":"u8db4xs3","id":"1(@)"}
+{"time":"2025-12-10T03:53:36.814688663Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"u8db4xs3","id":"1(@)"}
+{"time":"2025-12-10T04:25:22.68593366Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-internal.log b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..82cac581960431a2f84898ea3cc0f7642f996eb9
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-10T03:53:36.644840796Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T03:53:36.814471544Z","level":"INFO","msg":"stream: created new stream","id":"u8db4xs3"}
+{"time":"2025-12-10T03:53:36.814562621Z","level":"INFO","msg":"handler: started","stream_id":"u8db4xs3"}
+{"time":"2025-12-10T03:53:36.814680984Z","level":"INFO","msg":"stream: started","id":"u8db4xs3"}
+{"time":"2025-12-10T03:53:36.814691619Z","level":"INFO","msg":"writer: started","stream_id":"u8db4xs3"}
+{"time":"2025-12-10T03:53:36.814703071Z","level":"INFO","msg":"sender: started","stream_id":"u8db4xs3"}
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug.log b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..cd5b0371dcce89beab71f1b761b790a78e4b34ef
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_setup.py:_flush():80] Configure stats pid to 1991718
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug.log
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_035336-u8db4xs3/logs/debug-internal.log
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_init.py:init():841] calling init triggers
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 03:53:36,386 INFO MainThread:1991718 [wandb_init.py:init():889] starting backend
+2025-12-10 03:53:36,638 INFO MainThread:1991718 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 03:53:36,643 INFO MainThread:1991718 [wandb_init.py:init():900] backend started and connected
+2025-12-10 03:53:36,644 INFO MainThread:1991718 [wandb_init.py:init():970] updated telemetry
+2025-12-10 03:53:36,648 INFO MainThread:1991718 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 03:53:37,192 INFO MainThread:1991718 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 03:53:37,316 INFO MainThread:1991718 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 03:53:37,316 INFO MainThread:1991718 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 03:53:37,316 INFO MainThread:1991718 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 03:53:37,316 INFO MainThread:1991718 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 03:53:37,319 INFO MainThread:1991718 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 03:53:37,319 INFO MainThread:1991718 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_180x320_16f_2bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': False, 'features_dir': None}
diff --git a/Meissonic/wandb/run-20251210_035336-u8db4xs3/run-u8db4xs3.wandb b/Meissonic/wandb/run-20251210_035336-u8db4xs3/run-u8db4xs3.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..9ec9fdb344bf578e41379c0e5beda1d214952351
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_035336-u8db4xs3/run-u8db4xs3.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65adf28c4399ff3bde9d3f9930b1b289b78a3873e5979acb6f690fb5a2002b12
+size 491520
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/files/config.yaml b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..71664e191559314ce38d248ef90ce733267b6830
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/config.yaml
@@ -0,0 +1,303 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ fpeewl20f52v8o7trkfvthix59lgdpw0:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "2"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12139077726208"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T04:27:47.773230Z"
+ writerId: fpeewl20f52v8o7trkfvthix59lgdpw0
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 2
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/files/output.log b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..65584761c8dfee33cd1f4bb4610c1407b5523e53
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/output.log
@@ -0,0 +1,250 @@
+12/10/2025 04:27:48 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 04:27:48 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 04:27:48 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 04:27:48 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 04:27:48 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 04:27:48 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 04:27:48 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 04:27:48 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:27:48 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 04:28:04 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:28:04 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:28:06 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 04:28:08 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 04:28:08 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 04:28:08 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 04:28:08 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 04:28:08 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 04:28:08 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 04:28:08 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 04:28:08 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 04:28:08 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 04:28:08 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 04:28:08 - INFO - __main__ - Dataloader configuration:
+12/10/2025 04:28:08 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 04:28:08 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 04:28:08 - INFO - __main__ - - persistent_workers: True
+12/10/2025 04:28:08 - INFO - __main__ - - pin_memory: True
+12/10/2025 04:28:08 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 04:28:10 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 04:28:10 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/10/2025 04:28:10 - INFO - __main__ - ***** Running training *****
+12/10/2025 04:28:10 - INFO - __main__ - Num training steps = 10000
+12/10/2025 04:28:10 - INFO - __main__ - Instantaneous batch size per device = 2
+12/10/2025 04:28:10 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 64
+12/10/2025 04:28:10 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([2, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([2, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 04:28:11 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+[DEBUG] video_tokens: shape=torch.Size([2, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([2, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 04:28:15 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+[DEBUG] video_tokens: shape=torch.Size([2, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([2, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 04:28:15 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+[DEBUG] video_tokens: shape=torch.Size([2, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([2, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 04:28:16 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:17 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:18 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:18 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:19 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:19 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:20 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:20 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:21 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:21 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:22 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:23 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:23 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:24 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:24 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:25 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:25 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:26 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:26 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:27 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:27 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:28 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:29 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:29 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:30 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:30 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:31 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:31 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:32 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:32 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:33 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:34 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:34 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:35 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:35 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:36 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:36 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:37 - INFO - __main__ - Step: 10 Loss: 11.0752 LR: 0.000060
+12/10/2025 04:28:37 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:37 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:38 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:38 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:39 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:39 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:40 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:40 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:41 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:42 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:42 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:43 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:43 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:44 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:44 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:45 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:46 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:46 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:46 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:47 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:48 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:48 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:49 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:49 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:50 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:50 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:51 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:51 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:52 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:52 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:53 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:54 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:54 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:55 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:55 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:56 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:56 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:57 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:57 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:57 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:58 - INFO - __main__ - Step: 20 Loss: 11.0732 LR: 0.000060
+12/10/2025 04:28:58 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:59 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:28:59 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:00 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:00 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:01 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:01 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:02 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:02 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:03 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:03 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:04 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:04 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:05 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:06 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:06 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:07 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:07 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:08 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:08 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:09 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:09 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:10 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:10 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:11 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:11 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:12 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:12 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:13 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:13 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:14 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:14 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:15 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:15 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:16 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:16 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:17 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:18 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:18 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:19 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:19 - INFO - __main__ - Step: 30 Loss: 11.0719 LR: 0.000060
+12/10/2025 04:29:19 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:20 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:20 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:21 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:21 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:22 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:22 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:23 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:24 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:24 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:25 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:25 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:26 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:26 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:27 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:27 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:28 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:28 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:29 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:29 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:30 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:31 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:31 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:31 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:32 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:33 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:33 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:34 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:34 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:35 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:35 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:36 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:37 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:37 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:38 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:38 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:39 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:39 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:40 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:40 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:41 - INFO - __main__ - Step: 40 Loss: 11.0682 LR: 0.000060
+12/10/2025 04:29:41 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:42 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:42 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:43 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:44 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:44 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:45 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:45 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:46 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:46 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:47 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:47 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:48 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:48 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:49 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+12/10/2025 04:29:50 - WARNING - __main__ - cond_dropout_prob > 0.0 is not supported with precomputed features. Skipping cond_dropout.
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+ if __name__ == "__main__":
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1692, in
+[rank0]: if __name__ == "__main__":
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1509, in main
+[rank0]:
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/files/requirements.txt b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3a9728ca0269de1a707fb32b43b607a6b6db4b0
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T04:27:47.773230Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "2",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12139077726208"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "fpeewl20f52v8o7trkfvthix59lgdpw0"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-summary.json b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7693a059d00dfae50a8041a986a4278b7de3e7c
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_timestamp":1.7653409814908886e+09,"_step":40,"step_loss":11.068217277526855,"_wandb":{"runtime":122},"_runtime":122.718655158,"lr":5.9999999999999995e-05,"avg_masking_rate":0.8838287591934204}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-core.log b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..5dd0db9cb88e93a1a09c358e93ad9d46f4fc4aec
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T04:27:47.841707886Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpfheya3w_/port-2616859.txt","pid":2616859,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T04:27:47.842181416Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2616859}
+{"time":"2025-12-10T04:27:47.842192259Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2616859-2617110-619976077/socket","Net":"unix"}}
+{"time":"2025-12-10T04:27:48.028473168Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T04:27:48.034580582Z","level":"INFO","msg":"handleInformInit: received","streamId":"ig7zestl","id":"1(@)"}
+{"time":"2025-12-10T04:27:48.205561021Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ig7zestl","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.145346547Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.14542004Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T04:29:51.145409417Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.1454706Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.145521073Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2616859-2617110-619976077/socket","Net":"unix"}}
+{"time":"2025-12-10T04:29:51.502096225Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.50213062Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T04:29:51.502146483Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-internal.log b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..8266798a6ae9a41786d6e14559537affba0d8814
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T04:27:48.034729115Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T04:27:48.205351521Z","level":"INFO","msg":"stream: created new stream","id":"ig7zestl"}
+{"time":"2025-12-10T04:27:48.205450319Z","level":"INFO","msg":"handler: started","stream_id":"ig7zestl"}
+{"time":"2025-12-10T04:27:48.205554006Z","level":"INFO","msg":"stream: started","id":"ig7zestl"}
+{"time":"2025-12-10T04:27:48.205570824Z","level":"INFO","msg":"writer: started","stream_id":"ig7zestl"}
+{"time":"2025-12-10T04:27:48.205572875Z","level":"INFO","msg":"sender: started","stream_id":"ig7zestl"}
+{"time":"2025-12-10T04:29:51.1454205Z","level":"INFO","msg":"stream: closing","id":"ig7zestl"}
+{"time":"2025-12-10T04:29:51.383718617Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T04:29:51.499051314Z","level":"INFO","msg":"handler: closed","stream_id":"ig7zestl"}
+{"time":"2025-12-10T04:29:51.499164357Z","level":"INFO","msg":"sender: closed","stream_id":"ig7zestl"}
+{"time":"2025-12-10T04:29:51.499175978Z","level":"INFO","msg":"stream: closed","id":"ig7zestl"}
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug.log b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8d0c5d1d762fb0c41201333c2bc5c6f03d5609e4
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_setup.py:_flush():80] Configure stats pid to 2616859
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug.log
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_042747-ig7zestl/logs/debug-internal.log
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_init.py:init():841] calling init triggers
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 04:27:47,776 INFO MainThread:2616859 [wandb_init.py:init():889] starting backend
+2025-12-10 04:27:48,028 INFO MainThread:2616859 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 04:27:48,032 INFO MainThread:2616859 [wandb_init.py:init():900] backend started and connected
+2025-12-10 04:27:48,034 INFO MainThread:2616859 [wandb_init.py:init():970] updated telemetry
+2025-12-10 04:27:48,038 INFO MainThread:2616859 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 04:27:48,426 INFO MainThread:2616859 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 04:27:48,549 INFO MainThread:2616859 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 04:27:48,549 INFO MainThread:2616859 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 04:27:48,549 INFO MainThread:2616859 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 04:27:48,549 INFO MainThread:2616859 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 04:27:48,551 INFO MainThread:2616859 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 04:27:48,552 INFO MainThread:2616859 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 2, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-10 04:29:51,145 INFO wandb-AsyncioManager-main:2616859 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 04:29:51,145 INFO wandb-AsyncioManager-main:2616859 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_042747-ig7zestl/run-ig7zestl.wandb b/Meissonic/wandb/run-20251210_042747-ig7zestl/run-ig7zestl.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..94934f37db15ad63dcac62be0934a50ec76c344a
Binary files /dev/null and b/Meissonic/wandb/run-20251210_042747-ig7zestl/run-ig7zestl.wandb differ
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/config.yaml b/Meissonic/wandb/run-20251210_043009-5878wpml/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..612d285a19c5ffbc6316038fb6bab4955ae1ac13
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/files/config.yaml
@@ -0,0 +1,303 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ u29b1oh9n7s9k872fah1kpv4yjmrs1rq:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12139077951488"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T04:30:09.598769Z"
+ writerId: u29b1oh9n7s9k872fah1kpv4yjmrs1rq
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_94b9a8ac13767e53dfe4.png b/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_94b9a8ac13767e53dfe4.png
new file mode 100644
index 0000000000000000000000000000000000000000..8bff0fa2febd24fc4933bcf5f68c41d8455e50f0
Binary files /dev/null and b/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_94b9a8ac13767e53dfe4.png differ
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_ac0cddc93fd7d890391c.png b/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_ac0cddc93fd7d890391c.png
new file mode 100644
index 0000000000000000000000000000000000000000..2b6b9a760874b145e4222018368bef6ff46e215d
Binary files /dev/null and b/Meissonic/wandb/run-20251210_043009-5878wpml/files/media/images/generated_videos_first_frame_500_ac0cddc93fd7d890391c.png differ
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/output.log b/Meissonic/wandb/run-20251210_043009-5878wpml/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..d179946fda1932aeb103a9b8f1a47ea816256f38
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/files/output.log
@@ -0,0 +1,145 @@
+12/10/2025 04:30:10 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 04:30:10 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 04:30:10 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 04:30:10 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 04:30:10 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 04:30:10 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 04:30:10 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 04:30:10 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:30:10 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 04:30:26 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:30:26 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:30:28 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 04:30:30 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 04:30:30 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 04:30:30 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 04:30:30 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 04:30:30 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 04:30:30 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 04:30:30 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 04:30:30 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 04:30:30 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 04:30:30 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 04:30:30 - INFO - __main__ - Dataloader configuration:
+12/10/2025 04:30:30 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 04:30:30 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 04:30:30 - INFO - __main__ - - persistent_workers: True
+12/10/2025 04:30:30 - INFO - __main__ - - pin_memory: True
+12/10/2025 04:30:30 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 04:30:31 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 04:30:31 - INFO - __main__ - Skipping empty_embeds creation - using precomputed features
+12/10/2025 04:30:31 - INFO - __main__ - ***** Running training *****
+12/10/2025 04:30:31 - INFO - __main__ - Num training steps = 10000
+12/10/2025 04:30:31 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 04:30:31 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 04:30:31 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 04:30:55 - INFO - __main__ - Step: 10 Loss: 11.0747 LR: 0.000060
+12/10/2025 04:31:12 - INFO - __main__ - Step: 20 Loss: 11.0729 LR: 0.000060
+12/10/2025 04:31:29 - INFO - __main__ - Step: 30 Loss: 11.0707 LR: 0.000060
+12/10/2025 04:31:46 - INFO - __main__ - Step: 40 Loss: 11.0673 LR: 0.000060
+12/10/2025 04:32:04 - INFO - __main__ - Step: 50 Loss: 11.0643 LR: 0.000060
+12/10/2025 04:32:21 - INFO - __main__ - Step: 60 Loss: 11.0605 LR: 0.000060
+12/10/2025 04:32:39 - INFO - __main__ - Step: 70 Loss: 11.0540 LR: 0.000060
+12/10/2025 04:32:56 - INFO - __main__ - Step: 80 Loss: 11.0399 LR: 0.000060
+12/10/2025 04:33:13 - INFO - __main__ - Step: 90 Loss: 11.0209 LR: 0.000060
+12/10/2025 04:33:31 - INFO - __main__ - Step: 100 Loss: 10.9853 LR: 0.000060
+12/10/2025 04:33:48 - INFO - __main__ - Step: 110 Loss: 10.9407 LR: 0.000060
+12/10/2025 04:34:06 - INFO - __main__ - Step: 120 Loss: 10.8765 LR: 0.000060
+12/10/2025 04:34:24 - INFO - __main__ - Step: 130 Loss: 10.8079 LR: 0.000060
+12/10/2025 04:34:41 - INFO - __main__ - Step: 140 Loss: 10.7455 LR: 0.000060
+12/10/2025 04:34:58 - INFO - __main__ - Step: 150 Loss: 10.6594 LR: 0.000060
+12/10/2025 04:35:15 - INFO - __main__ - Step: 160 Loss: 10.6105 LR: 0.000060
+12/10/2025 04:35:33 - INFO - __main__ - Step: 170 Loss: 10.5386 LR: 0.000060
+12/10/2025 04:35:51 - INFO - __main__ - Step: 180 Loss: 10.4894 LR: 0.000060
+12/10/2025 04:36:10 - INFO - __main__ - Step: 190 Loss: 10.4253 LR: 0.000060
+12/10/2025 04:36:30 - INFO - __main__ - Step: 200 Loss: 10.3921 LR: 0.000060
+12/10/2025 04:36:48 - INFO - __main__ - Step: 210 Loss: 10.3110 LR: 0.000060
+12/10/2025 04:37:07 - INFO - __main__ - Step: 220 Loss: 10.2477 LR: 0.000060
+12/10/2025 04:37:26 - INFO - __main__ - Step: 230 Loss: 10.1233 LR: 0.000060
+12/10/2025 04:37:45 - INFO - __main__ - Step: 240 Loss: 9.8485 LR: 0.000060
+12/10/2025 04:38:04 - INFO - __main__ - Step: 250 Loss: 9.7038 LR: 0.000060
+12/10/2025 04:38:23 - INFO - __main__ - Step: 260 Loss: 9.1299 LR: 0.000060
+12/10/2025 04:38:42 - INFO - __main__ - Step: 270 Loss: 9.1336 LR: 0.000060
+12/10/2025 04:39:00 - INFO - __main__ - Step: 280 Loss: 8.3852 LR: 0.000060
+12/10/2025 04:39:18 - INFO - __main__ - Step: 290 Loss: 7.4781 LR: 0.000060
+12/10/2025 04:39:35 - INFO - __main__ - Step: 300 Loss: 7.1441 LR: 0.000060
+12/10/2025 04:39:52 - INFO - __main__ - Step: 310 Loss: 7.0994 LR: 0.000060
+12/10/2025 04:40:09 - INFO - __main__ - Step: 320 Loss: 6.8467 LR: 0.000060
+12/10/2025 04:40:28 - INFO - __main__ - Step: 330 Loss: 4.6846 LR: 0.000060
+12/10/2025 04:40:46 - INFO - __main__ - Step: 340 Loss: 6.1597 LR: 0.000060
+12/10/2025 04:41:04 - INFO - __main__ - Step: 350 Loss: 4.1510 LR: 0.000060
+12/10/2025 04:41:22 - INFO - __main__ - Step: 360 Loss: 3.1741 LR: 0.000060
+12/10/2025 04:41:39 - INFO - __main__ - Step: 370 Loss: 2.4839 LR: 0.000060
+12/10/2025 04:41:57 - INFO - __main__ - Step: 380 Loss: 1.9491 LR: 0.000060
+12/10/2025 04:42:14 - INFO - __main__ - Step: 390 Loss: 1.6005 LR: 0.000060
+12/10/2025 04:42:32 - INFO - __main__ - Step: 400 Loss: 1.2793 LR: 0.000060
+12/10/2025 04:42:49 - INFO - __main__ - Step: 410 Loss: 1.0339 LR: 0.000060
+12/10/2025 04:43:06 - INFO - __main__ - Step: 420 Loss: 0.8393 LR: 0.000060
+12/10/2025 04:43:24 - INFO - __main__ - Step: 430 Loss: 0.7320 LR: 0.000060
+12/10/2025 04:43:41 - INFO - __main__ - Step: 440 Loss: 0.6767 LR: 0.000060
+12/10/2025 04:43:58 - INFO - __main__ - Step: 450 Loss: 2.8503 LR: 0.000060
+12/10/2025 04:44:16 - INFO - __main__ - Step: 460 Loss: 10.3635 LR: 0.000060
+12/10/2025 04:44:34 - INFO - __main__ - Step: 470 Loss: 9.7815 LR: 0.000060
+12/10/2025 04:44:53 - INFO - __main__ - Step: 480 Loss: 8.7190 LR: 0.000060
+12/10/2025 04:45:12 - INFO - __main__ - Step: 490 Loss: 6.0148 LR: 0.000060
+12/10/2025 04:45:31 - INFO - __main__ - Step: 500 Loss: 2.5093 LR: 0.000060
+12/10/2025 04:45:31 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 04:45:38 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 04:45:38 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 04:45:38 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 04:45:38 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 04:45:38 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 04:45:38 - INFO - __main__ - Generating videos for validation...
+12/10/2025 04:45:38 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 75.39it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 7138.37it/s]
+12/10/2025 04:45:55 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 04:45:55 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.24it/s]
+12/10/2025 04:46:02 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1693, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1510, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1015.56 MiB is free. Including non-PyTorch memory, this process has 38.49 GiB memory in use. Of the allocated memory 28.91 GiB is allocated by PyTorch, and 8.47 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1693, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1510, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1015.56 MiB is free. Including non-PyTorch memory, this process has 38.49 GiB memory in use. Of the allocated memory 28.91 GiB is allocated by PyTorch, and 8.47 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/requirements.txt b/Meissonic/wandb/run-20251210_043009-5878wpml/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7a3ba58586d478faa4c009fd7f1e00d2d2a46ae
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T04:30:09.598769Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12139077951488"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "u29b1oh9n7s9k872fah1kpv4yjmrs1rq"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-summary.json b/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..569aef916d0f809203a931f426ea046f75f32249
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/files/wandb-summary.json
@@ -0,0 +1 @@
+{"avg_masking_rate":0.6932775974273682,"_timestamp":1.7653419622041483e+09,"_step":500,"step_loss":2.509253978729248,"generated_videos_first_frame":{"count":2,"filenames":["media/images/generated_videos_first_frame_500_ac0cddc93fd7d890391c.png","media/images/generated_videos_first_frame_500_94b9a8ac13767e53dfe4.png"],"captions":["a cat playing","a girl walking"],"_type":"images/separated","width":128,"height":128,"format":"png"},"_wandb":{"runtime":974},"_runtime":974.663843316,"lr":5.9999999999999995e-05}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-core.log b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..d91454d8cfdfd3d67b00bb7c4c2502c6acf075e9
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T04:30:09.698193881Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpxqlj0nzs/port-2620362.txt","pid":2620362,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T04:30:09.698667007Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2620362}
+{"time":"2025-12-10T04:30:09.698677597Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2620362-2620607-330668019/socket","Net":"unix"}}
+{"time":"2025-12-10T04:30:09.884629696Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T04:30:09.891548314Z","level":"INFO","msg":"handleInformInit: received","streamId":"5878wpml","id":"1(@)"}
+{"time":"2025-12-10T04:30:10.061099346Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"5878wpml","id":"1(@)"}
+{"time":"2025-12-10T04:46:24.942928232Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T04:46:24.943002572Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T04:46:24.942991389Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T04:46:24.943075829Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T04:46:24.943091418Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2620362-2620607-330668019/socket","Net":"unix"}}
+{"time":"2025-12-10T04:46:25.484502998Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T04:46:25.484570442Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T04:46:25.484589679Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-internal.log b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..ceadcd586173e2d4f22d279c6af2444a8093e599
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T04:30:09.891653528Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T04:30:10.060886723Z","level":"INFO","msg":"stream: created new stream","id":"5878wpml"}
+{"time":"2025-12-10T04:30:10.060977449Z","level":"INFO","msg":"handler: started","stream_id":"5878wpml"}
+{"time":"2025-12-10T04:30:10.061092102Z","level":"INFO","msg":"stream: started","id":"5878wpml"}
+{"time":"2025-12-10T04:30:10.061107932Z","level":"INFO","msg":"writer: started","stream_id":"5878wpml"}
+{"time":"2025-12-10T04:30:10.061111572Z","level":"INFO","msg":"sender: started","stream_id":"5878wpml"}
+{"time":"2025-12-10T04:46:24.942997072Z","level":"INFO","msg":"stream: closing","id":"5878wpml"}
+{"time":"2025-12-10T04:46:25.196277775Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T04:46:25.39496968Z","level":"INFO","msg":"handler: closed","stream_id":"5878wpml"}
+{"time":"2025-12-10T04:46:25.395193205Z","level":"INFO","msg":"sender: closed","stream_id":"5878wpml"}
+{"time":"2025-12-10T04:46:25.39520522Z","level":"INFO","msg":"stream: closed","id":"5878wpml"}
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug.log b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..fa09297566893f55a42fe8cd6739d55d124252fb
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 04:30:09,603 INFO MainThread:2620362 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_setup.py:_flush():80] Configure stats pid to 2620362
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug.log
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_043009-5878wpml/logs/debug-internal.log
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_init.py:init():841] calling init triggers
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 04:30:09,604 INFO MainThread:2620362 [wandb_init.py:init():889] starting backend
+2025-12-10 04:30:09,884 INFO MainThread:2620362 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 04:30:09,890 INFO MainThread:2620362 [wandb_init.py:init():900] backend started and connected
+2025-12-10 04:30:09,891 INFO MainThread:2620362 [wandb_init.py:init():970] updated telemetry
+2025-12-10 04:30:09,897 INFO MainThread:2620362 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 04:30:10,278 INFO MainThread:2620362 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 04:30:10,458 INFO MainThread:2620362 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 04:30:10,458 INFO MainThread:2620362 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 04:30:10,458 INFO MainThread:2620362 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 04:30:10,458 INFO MainThread:2620362 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 04:30:10,461 INFO MainThread:2620362 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 04:30:10,462 INFO MainThread:2620362 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features'}
+2025-12-10 04:46:24,943 INFO wandb-AsyncioManager-main:2620362 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 04:46:24,943 INFO wandb-AsyncioManager-main:2620362 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_043009-5878wpml/run-5878wpml.wandb b/Meissonic/wandb/run-20251210_043009-5878wpml/run-5878wpml.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..b770ff4ed8dc5dc4e99d3e2d23329f14ef28136e
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_043009-5878wpml/run-5878wpml.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d32516d6df52a8fb2f9ec2a69ea11b7f80157f8295a2db93de1a7370464134d0
+size 296403
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/files/config.yaml b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..166a63f31ea943fcb6f7fd9f38085093b6c82fb8
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/config.yaml
@@ -0,0 +1,303 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ b3z1vhpf74rln28qyknvlicz64uo75mz:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12144812875776"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T04:55:07.059107Z"
+ writerId: b3z1vhpf74rln28qyknvlicz64uo75mz
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/files/output.log b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..58ad0ed8c33de6dc8ce4533fd8c2a118bbf481d7
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/output.log
@@ -0,0 +1,18 @@
+12/10/2025 04:55:07 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 04:55:07 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 04:55:07 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 04:55:07 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 04:55:07 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 04:55:07 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1744, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 647, in main
+ if sample_path:
+NameError: name 'sample_path' is not defined
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1744, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 647, in main
+[rank0]: if sample_path:
+[rank0]: NameError: name 'sample_path' is not defined
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/files/requirements.txt b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..6e98bcdff8d61543ae37132ef5cb4ba8566b1649
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T04:55:07.059107Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12144812875776"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "b3z1vhpf74rln28qyknvlicz64uo75mz"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-summary.json b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0a620d0c1047a4dd8a400939b6da246ed8063a7
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-core.log b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..b8c2057f1fe19f31d351f20925c276726d7dc369
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T04:55:07.132956152Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpdygqy9pg/port-2642308.txt","pid":2642308,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T04:55:07.133486865Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2642308}
+{"time":"2025-12-10T04:55:07.133496718Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2642308-2642553-4210231117/socket","Net":"unix"}}
+{"time":"2025-12-10T04:55:07.318948519Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.325090184Z","level":"INFO","msg":"handleInformInit: received","streamId":"n9avnzup","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.494164812Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n9avnzup","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.811427792Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.811469711Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T04:55:07.811463462Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.811505274Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T04:55:07.811552197Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2642308-2642553-4210231117/socket","Net":"unix"}}
+{"time":"2025-12-10T04:55:08.341428152Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T04:55:08.341448567Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T04:55:08.341461221Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-internal.log b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..176a83f298b6a2c8cc5f623e8ec8184dfdeca4fc
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T04:55:07.325196653Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T04:55:07.493938833Z","level":"INFO","msg":"stream: created new stream","id":"n9avnzup"}
+{"time":"2025-12-10T04:55:07.494040848Z","level":"INFO","msg":"handler: started","stream_id":"n9avnzup"}
+{"time":"2025-12-10T04:55:07.494157208Z","level":"INFO","msg":"stream: started","id":"n9avnzup"}
+{"time":"2025-12-10T04:55:07.494184586Z","level":"INFO","msg":"writer: started","stream_id":"n9avnzup"}
+{"time":"2025-12-10T04:55:07.49418817Z","level":"INFO","msg":"sender: started","stream_id":"n9avnzup"}
+{"time":"2025-12-10T04:55:07.811473161Z","level":"INFO","msg":"stream: closing","id":"n9avnzup"}
+{"time":"2025-12-10T04:55:08.201048352Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T04:55:08.336943071Z","level":"INFO","msg":"handler: closed","stream_id":"n9avnzup"}
+{"time":"2025-12-10T04:55:08.337078003Z","level":"INFO","msg":"sender: closed","stream_id":"n9avnzup"}
+{"time":"2025-12-10T04:55:08.337085632Z","level":"INFO","msg":"stream: closed","id":"n9avnzup"}
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug.log b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..2406c5fc2017cc1ef8f1b730eeef95ee67b727f2
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_setup.py:_flush():80] Configure stats pid to 2642308
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug.log
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_045507-n9avnzup/logs/debug-internal.log
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_init.py:init():841] calling init triggers
+2025-12-10 04:55:07,062 INFO MainThread:2642308 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 04:55:07,063 INFO MainThread:2642308 [wandb_init.py:init():889] starting backend
+2025-12-10 04:55:07,319 INFO MainThread:2642308 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 04:55:07,323 INFO MainThread:2642308 [wandb_init.py:init():900] backend started and connected
+2025-12-10 04:55:07,324 INFO MainThread:2642308 [wandb_init.py:init():970] updated telemetry
+2025-12-10 04:55:07,329 INFO MainThread:2642308 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 04:55:07,672 INFO MainThread:2642308 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 04:55:07,798 INFO MainThread:2642308 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 04:55:07,798 INFO MainThread:2642308 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 04:55:07,798 INFO MainThread:2642308 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 04:55:07,798 INFO MainThread:2642308 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 04:55:07,802 INFO MainThread:2642308 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 04:55:07,803 INFO MainThread:2642308 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 04:55:07,811 INFO wandb-AsyncioManager-main:2642308 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 04:55:07,811 INFO wandb-AsyncioManager-main:2642308 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_045507-n9avnzup/run-n9avnzup.wandb b/Meissonic/wandb/run-20251210_045507-n9avnzup/run-n9avnzup.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..54c7be523a3533afb0080b821ddedce5e308b112
Binary files /dev/null and b/Meissonic/wandb/run-20251210_045507-n9avnzup/run-n9avnzup.wandb differ
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/config.yaml b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..88fe3fde022d55be4147605405e56fdf06c5bd90
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/config.yaml
@@ -0,0 +1,303 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ qqpknxczyu7ian1xhqsu8gzi7ixhu7s2:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12144813121536"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T04:57:17.679940Z"
+ writerId: qqpknxczyu7ian1xhqsu8gzi7ixhu7s2
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/output.log b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..e3a909c10891a42c28df0b838ad6668812fb4b42
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/output.log
@@ -0,0 +1,68 @@
+12/10/2025 04:57:18 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 04:57:18 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 04:57:18 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 04:57:18 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 04:57:18 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 04:57:18 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 04:57:18 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 04:57:18 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:57:18 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 04:57:35 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:57:35 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:57:37 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 04:57:39 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 04:57:39 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 04:57:39 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 04:57:39 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 04:57:39 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 04:57:39 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 04:57:39 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 04:57:39 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 04:57:39 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 04:57:39 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 04:57:39 - INFO - __main__ - Dataloader configuration:
+12/10/2025 04:57:39 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 04:57:39 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 04:57:39 - INFO - __main__ - - persistent_workers: True
+12/10/2025 04:57:39 - INFO - __main__ - - pin_memory: True
+12/10/2025 04:57:39 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 04:57:41 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 04:57:41 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 04:57:41 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 04:57:41 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 04:57:41 - INFO - __main__ - ***** Running training *****
+12/10/2025 04:57:41 - INFO - __main__ - Num training steps = 10000
+12/10/2025 04:57:41 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 04:57:41 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 04:57:41 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1744, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1550, in main
+ loss = F.cross_entropy(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+ return torch._C._nn.cross_entropy_loss(
+NotImplementedError: "nll_loss_forward_reduce_cuda_kernel_2d_index" not implemented for 'Int'
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1744, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1550, in main
+[rank0]: loss = F.cross_entropy(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/functional.py", line 3458, in cross_entropy
+[rank0]: return torch._C._nn.cross_entropy_loss(
+[rank0]: NotImplementedError: "nll_loss_forward_reduce_cuda_kernel_2d_index" not implemented for 'Int'
+Exception ignored in atexit callback:
+Traceback (most recent call last):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1648, in _clean_up_worker
+ w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/process.py", line 149, in join
+ res = self._popen.wait(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/popen_fork.py", line 40, in wait
+ if not wait([self.sentinel], timeout):
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/multiprocessing/connection.py", line 931, in wait
+ ready = selector.select(timeout)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/selectors.py", line 416, in select
+ fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt:
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/requirements.txt b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..85eded996083791a44bc231bcb0ff2b87aae26a4
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T04:57:17.679940Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12144813121536"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "qqpknxczyu7ian1xhqsu8gzi7ixhu7s2"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-summary.json b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..519453af2693820c461935aa95c333853a63586f
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_runtime":37,"_wandb":{"runtime":37}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-core.log b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..2031670e93e4b52608c7fe1b8c130fb8fb0d78c3
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T04:57:17.749501651Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp9zt3zwee/port-2645064.txt","pid":2645064,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T04:57:17.750776461Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2645064}
+{"time":"2025-12-10T04:57:17.750766186Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2645064-2645236-1292604464/socket","Net":"unix"}}
+{"time":"2025-12-10T04:57:17.935921782Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T04:57:17.942016424Z","level":"INFO","msg":"handleInformInit: received","streamId":"nkp4mvju","id":"1(@)"}
+{"time":"2025-12-10T04:57:18.108561105Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"nkp4mvju","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.198239991Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.198455801Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.198521874Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.198472608Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T04:57:56.198634459Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2645064-2645236-1292604464/socket","Net":"unix"}}
+{"time":"2025-12-10T04:57:56.620920763Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.620942501Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T04:57:56.620956696Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-internal.log b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..b3e6e62e93078e68590e096e5f85b3bca407a9f3
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T04:57:17.942109813Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T04:57:18.108374946Z","level":"INFO","msg":"stream: created new stream","id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:18.108460354Z","level":"INFO","msg":"handler: started","stream_id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:18.108554304Z","level":"INFO","msg":"stream: started","id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:18.108573013Z","level":"INFO","msg":"writer: started","stream_id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:18.108573172Z","level":"INFO","msg":"sender: started","stream_id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:56.198455836Z","level":"INFO","msg":"stream: closing","id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:56.506764749Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T04:57:56.617832646Z","level":"INFO","msg":"handler: closed","stream_id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:56.617951022Z","level":"INFO","msg":"sender: closed","stream_id":"nkp4mvju"}
+{"time":"2025-12-10T04:57:56.617962222Z","level":"INFO","msg":"stream: closed","id":"nkp4mvju"}
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug.log b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..29d0e0b67b791f29c353d9dcdba7ec7e8531ca5c
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_setup.py:_flush():80] Configure stats pid to 2645064
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug.log
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_045717-nkp4mvju/logs/debug-internal.log
+2025-12-10 04:57:17,682 INFO MainThread:2645064 [wandb_init.py:init():841] calling init triggers
+2025-12-10 04:57:17,683 INFO MainThread:2645064 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 04:57:17,683 INFO MainThread:2645064 [wandb_init.py:init():889] starting backend
+2025-12-10 04:57:17,936 INFO MainThread:2645064 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 04:57:17,940 INFO MainThread:2645064 [wandb_init.py:init():900] backend started and connected
+2025-12-10 04:57:17,941 INFO MainThread:2645064 [wandb_init.py:init():970] updated telemetry
+2025-12-10 04:57:17,946 INFO MainThread:2645064 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 04:57:18,321 INFO MainThread:2645064 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 04:57:18,447 INFO MainThread:2645064 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 04:57:18,447 INFO MainThread:2645064 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 04:57:18,447 INFO MainThread:2645064 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 04:57:18,447 INFO MainThread:2645064 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 04:57:18,450 INFO MainThread:2645064 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 04:57:18,451 INFO MainThread:2645064 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 04:57:56,198 INFO wandb-AsyncioManager-main:2645064 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 04:57:56,198 INFO wandb-AsyncioManager-main:2645064 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_045717-nkp4mvju/run-nkp4mvju.wandb b/Meissonic/wandb/run-20251210_045717-nkp4mvju/run-nkp4mvju.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..403090bd8caf4f543ac1e04115c879ceeb255ae2
Binary files /dev/null and b/Meissonic/wandb/run-20251210_045717-nkp4mvju/run-nkp4mvju.wandb differ
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/config.yaml b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..89db082b4599d94656c2e28e01b5359cd4549c56
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/config.yaml
@@ -0,0 +1,305 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ jxbyukuh3yklib8f49yg9pa95z492jqn:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12144813256704"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T04:59:34.205846Z"
+ writerId: jxbyukuh3yklib8f49yg9pa95z492jqn
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_bb0a0ba4ea3567f7f45a.png b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_bb0a0ba4ea3567f7f45a.png
new file mode 100644
index 0000000000000000000000000000000000000000..9182bc4387a0fb657a3ea4b2340bd014a1ecdf89
Binary files /dev/null and b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_bb0a0ba4ea3567f7f45a.png differ
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_e5bd64a3ce99f0946981.png b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_e5bd64a3ce99f0946981.png
new file mode 100644
index 0000000000000000000000000000000000000000..01e3df6f09803f3af7ddfcc4acfb9f12267803a8
Binary files /dev/null and b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/media/images/generated_videos_first_frame_500_e5bd64a3ce99f0946981.png differ
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/output.log b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..07d8c65500b96b639ec94154b3a5b4bafba25998
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/output.log
@@ -0,0 +1,134 @@
+12/10/2025 04:59:34 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 04:59:34 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 04:59:34 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 04:59:34 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 04:59:34 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 04:59:34 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 04:59:34 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 04:59:34 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:59:35 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 04:59:50 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:59:50 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 04:59:52 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 04:59:54 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 04:59:54 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 04:59:54 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 04:59:54 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 04:59:54 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 04:59:54 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 04:59:54 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 04:59:54 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 04:59:54 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 04:59:54 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 04:59:54 - INFO - __main__ - Dataloader configuration:
+12/10/2025 04:59:54 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 04:59:54 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 04:59:54 - INFO - __main__ - - persistent_workers: True
+12/10/2025 04:59:54 - INFO - __main__ - - pin_memory: True
+12/10/2025 04:59:54 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 04:59:56 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 04:59:56 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 04:59:56 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 04:59:56 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 04:59:56 - INFO - __main__ - ***** Running training *****
+12/10/2025 04:59:56 - INFO - __main__ - Num training steps = 10000
+12/10/2025 04:59:56 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 04:59:56 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 04:59:56 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 05:00:18 - INFO - __main__ - Step: 10 Loss: 11.0753 LR: 0.000060
+12/10/2025 05:00:34 - INFO - __main__ - Step: 20 Loss: 11.0730 LR: 0.000060
+12/10/2025 05:00:51 - INFO - __main__ - Step: 30 Loss: 11.0694 LR: 0.000060
+12/10/2025 05:01:09 - INFO - __main__ - Step: 40 Loss: 11.0665 LR: 0.000060
+12/10/2025 05:01:26 - INFO - __main__ - Step: 50 Loss: 11.0638 LR: 0.000060
+12/10/2025 05:01:43 - INFO - __main__ - Step: 60 Loss: 11.0581 LR: 0.000060
+12/10/2025 05:01:59 - INFO - __main__ - Step: 70 Loss: 11.0504 LR: 0.000060
+12/10/2025 05:02:16 - INFO - __main__ - Step: 80 Loss: 11.0334 LR: 0.000060
+12/10/2025 05:02:32 - INFO - __main__ - Step: 90 Loss: 11.0099 LR: 0.000060
+12/10/2025 05:02:51 - INFO - __main__ - Step: 100 Loss: 10.9695 LR: 0.000060
+12/10/2025 05:03:09 - INFO - __main__ - Step: 110 Loss: 10.9227 LR: 0.000060
+12/10/2025 05:03:27 - INFO - __main__ - Step: 120 Loss: 10.8649 LR: 0.000060
+12/10/2025 05:03:45 - INFO - __main__ - Step: 130 Loss: 10.7969 LR: 0.000060
+12/10/2025 05:04:01 - INFO - __main__ - Step: 140 Loss: 10.7392 LR: 0.000060
+12/10/2025 05:04:18 - INFO - __main__ - Step: 150 Loss: 10.6720 LR: 0.000060
+12/10/2025 05:04:35 - INFO - __main__ - Step: 160 Loss: 10.6267 LR: 0.000060
+12/10/2025 05:04:51 - INFO - __main__ - Step: 170 Loss: 10.5702 LR: 0.000060
+12/10/2025 05:05:08 - INFO - __main__ - Step: 180 Loss: 10.5286 LR: 0.000060
+12/10/2025 05:05:25 - INFO - __main__ - Step: 190 Loss: 10.4715 LR: 0.000060
+12/10/2025 05:05:42 - INFO - __main__ - Step: 200 Loss: 10.4428 LR: 0.000060
+12/10/2025 05:05:58 - INFO - __main__ - Step: 210 Loss: 10.4134 LR: 0.000060
+12/10/2025 05:06:14 - INFO - __main__ - Step: 220 Loss: 10.3862 LR: 0.000060
+12/10/2025 05:06:31 - INFO - __main__ - Step: 230 Loss: 10.3619 LR: 0.000060
+12/10/2025 05:06:47 - INFO - __main__ - Step: 240 Loss: 10.3476 LR: 0.000060
+12/10/2025 05:07:03 - INFO - __main__ - Step: 250 Loss: 10.3287 LR: 0.000060
+12/10/2025 05:07:20 - INFO - __main__ - Step: 260 Loss: 10.3108 LR: 0.000060
+12/10/2025 05:07:37 - INFO - __main__ - Step: 270 Loss: 10.3068 LR: 0.000060
+12/10/2025 05:07:53 - INFO - __main__ - Step: 280 Loss: 10.3050 LR: 0.000060
+12/10/2025 05:08:09 - INFO - __main__ - Step: 290 Loss: 10.2792 LR: 0.000060
+12/10/2025 05:08:26 - INFO - __main__ - Step: 300 Loss: 10.2944 LR: 0.000060
+12/10/2025 05:08:43 - INFO - __main__ - Step: 310 Loss: 10.2807 LR: 0.000060
+12/10/2025 05:08:59 - INFO - __main__ - Step: 320 Loss: 10.2740 LR: 0.000060
+12/10/2025 05:09:16 - INFO - __main__ - Step: 330 Loss: 10.2760 LR: 0.000060
+12/10/2025 05:09:32 - INFO - __main__ - Step: 340 Loss: 10.2531 LR: 0.000060
+12/10/2025 05:09:48 - INFO - __main__ - Step: 350 Loss: 10.2740 LR: 0.000060
+12/10/2025 05:10:05 - INFO - __main__ - Step: 360 Loss: 10.2565 LR: 0.000060
+12/10/2025 05:10:22 - INFO - __main__ - Step: 370 Loss: 10.2202 LR: 0.000060
+12/10/2025 05:10:39 - INFO - __main__ - Step: 380 Loss: 10.2410 LR: 0.000060
+12/10/2025 05:10:55 - INFO - __main__ - Step: 390 Loss: 10.2119 LR: 0.000060
+12/10/2025 05:11:12 - INFO - __main__ - Step: 400 Loss: 10.2321 LR: 0.000060
+12/10/2025 05:11:29 - INFO - __main__ - Step: 410 Loss: 10.2193 LR: 0.000060
+12/10/2025 05:11:46 - INFO - __main__ - Step: 420 Loss: 10.2282 LR: 0.000060
+12/10/2025 05:12:02 - INFO - __main__ - Step: 430 Loss: 10.2097 LR: 0.000060
+12/10/2025 05:12:20 - INFO - __main__ - Step: 440 Loss: 10.2135 LR: 0.000060
+12/10/2025 05:12:36 - INFO - __main__ - Step: 450 Loss: 10.2036 LR: 0.000060
+12/10/2025 05:12:53 - INFO - __main__ - Step: 460 Loss: 10.1900 LR: 0.000060
+12/10/2025 05:13:10 - INFO - __main__ - Step: 470 Loss: 10.1852 LR: 0.000060
+12/10/2025 05:13:27 - INFO - __main__ - Step: 480 Loss: 10.1773 LR: 0.000060
+12/10/2025 05:13:44 - INFO - __main__ - Step: 490 Loss: 10.1748 LR: 0.000060
+12/10/2025 05:14:00 - INFO - __main__ - Step: 500 Loss: 10.1598 LR: 0.000060
+12/10/2025 05:14:00 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 05:14:49 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 05:14:50 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 05:14:50 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 05:14:50 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 05:14:50 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 05:14:50 - INFO - __main__ - Generating videos for validation...
+12/10/2025 05:14:50 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 73.06it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 5638.59it/s]
+12/10/2025 05:15:05 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 05:15:05 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.31it/s]
+12/10/2025 05:15:12 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1748, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1565, in main
+ accelerator.backward(loss)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+ loss.backward(**kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+ torch.autograd.backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+ _engine_run_backward(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.19 GiB is free. Including non-PyTorch memory, this process has 38.29 GiB memory in use. Of the allocated memory 28.93 GiB is allocated by PyTorch, and 8.25 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1748, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1565, in main
+[rank0]: accelerator.backward(loss)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/accelerator.py", line 2852, in backward
+[rank0]: loss.backward(**kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_tensor.py", line 625, in backward
+[rank0]: torch.autograd.backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]: _engine_run_backward(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
+[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.44 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.19 GiB is free. Including non-PyTorch memory, this process has 38.29 GiB memory in use. Of the allocated memory 28.93 GiB is allocated by PyTorch, and 8.25 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/requirements.txt b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7c204e361c34ad6b83f3e585b5c6922bcdf7d47
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T04:59:34.205846Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12144813256704"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "jxbyukuh3yklib8f49yg9pa95z492jqn"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-summary.json b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..dbd0ff66613c79f59dbf3d07b10733fb02d8e656
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/files/wandb-summary.json
@@ -0,0 +1 @@
+{"avg_masking_rate":0.6341594457626343,"_timestamp":1.7653437126120093e+09,"_step":500,"step_loss":10.159809112548828,"generated_videos_first_frame":{"_type":"images/separated","width":128,"height":128,"format":"png","count":2,"filenames":["media/images/generated_videos_first_frame_500_e5bd64a3ce99f0946981.png","media/images/generated_videos_first_frame_500_bb0a0ba4ea3567f7f45a.png"],"captions":["a cat playing","a girl walking"]},"_wandb":{"runtime":979},"_runtime":979.936700086,"lr":5.9999999999999995e-05}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-core.log b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..c694bf51c9ba042046a5d2702f7c8cbcaca1b608
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T04:59:34.274806638Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpab_coscc/port-2648638.txt","pid":2648638,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T04:59:34.275309337Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2648638}
+{"time":"2025-12-10T04:59:34.275320056Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2648638-2648807-3055656174/socket","Net":"unix"}}
+{"time":"2025-12-10T04:59:34.461477196Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T04:59:34.467604005Z","level":"INFO","msg":"handleInformInit: received","streamId":"tcqz8xbx","id":"1(@)"}
+{"time":"2025-12-10T04:59:34.640293994Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tcqz8xbx","id":"1(@)"}
+{"time":"2025-12-10T05:15:54.748146784Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T05:15:54.74821762Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T05:15:54.748208689Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T05:15:54.748281193Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T05:15:54.7483001Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2648638-2648807-3055656174/socket","Net":"unix"}}
+{"time":"2025-12-10T05:15:55.146957014Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T05:15:55.146980292Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T05:15:55.146992919Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-internal.log b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..582f6b0b69c0d4fac87b70a965c249f9d9f48770
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T04:59:34.4677486Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T04:59:34.640069359Z","level":"INFO","msg":"stream: created new stream","id":"tcqz8xbx"}
+{"time":"2025-12-10T04:59:34.640154163Z","level":"INFO","msg":"handler: started","stream_id":"tcqz8xbx"}
+{"time":"2025-12-10T04:59:34.64028702Z","level":"INFO","msg":"stream: started","id":"tcqz8xbx"}
+{"time":"2025-12-10T04:59:34.640304652Z","level":"INFO","msg":"writer: started","stream_id":"tcqz8xbx"}
+{"time":"2025-12-10T04:59:34.640328965Z","level":"INFO","msg":"sender: started","stream_id":"tcqz8xbx"}
+{"time":"2025-12-10T05:15:54.748218828Z","level":"INFO","msg":"stream: closing","id":"tcqz8xbx"}
+{"time":"2025-12-10T05:15:54.994339802Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T05:15:55.14391639Z","level":"INFO","msg":"handler: closed","stream_id":"tcqz8xbx"}
+{"time":"2025-12-10T05:15:55.143984436Z","level":"INFO","msg":"sender: closed","stream_id":"tcqz8xbx"}
+{"time":"2025-12-10T05:15:55.14399513Z","level":"INFO","msg":"stream: closed","id":"tcqz8xbx"}
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug.log b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..a805d58e9d48220efa68b4c4cea7e93b5b03a2fe
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 04:59:34,208 INFO MainThread:2648638 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 04:59:34,208 INFO MainThread:2648638 [wandb_setup.py:_flush():80] Configure stats pid to 2648638
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug.log
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_045934-tcqz8xbx/logs/debug-internal.log
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_init.py:init():841] calling init triggers
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 04:59:34,209 INFO MainThread:2648638 [wandb_init.py:init():889] starting backend
+2025-12-10 04:59:34,461 INFO MainThread:2648638 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 04:59:34,466 INFO MainThread:2648638 [wandb_init.py:init():900] backend started and connected
+2025-12-10 04:59:34,467 INFO MainThread:2648638 [wandb_init.py:init():970] updated telemetry
+2025-12-10 04:59:34,471 INFO MainThread:2648638 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 04:59:34,811 INFO MainThread:2648638 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 04:59:34,935 INFO MainThread:2648638 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 04:59:34,935 INFO MainThread:2648638 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 04:59:34,935 INFO MainThread:2648638 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 04:59:34,935 INFO MainThread:2648638 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 04:59:34,938 INFO MainThread:2648638 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 04:59:34,939 INFO MainThread:2648638 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 05:15:54,748 INFO wandb-AsyncioManager-main:2648638 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 05:15:54,748 INFO wandb-AsyncioManager-main:2648638 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_045934-tcqz8xbx/run-tcqz8xbx.wandb b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/run-tcqz8xbx.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..82030cd98f704d2574f0a08fab810d71ef8a9252
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_045934-tcqz8xbx/run-tcqz8xbx.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:155e81b68779a620ce7ccdf1a999ba679e38fbd50f24d16923fba4bb6f09b2a7
+size 297961
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/config.yaml b/Meissonic/wandb/run-20251210_065438-svzut638/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..16869a19666c20c98292d5cac2dd17d0b2a5377b
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/files/config.yaml
@@ -0,0 +1,305 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ allmhscr418v3bp5824ixjkvvxtxqjh3:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12144814084096"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T06:54:38.912502Z"
+ writerId: allmhscr418v3bp5824ixjkvvxtxqjh3
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_72143d6818e0e98395f3.png b/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_72143d6818e0e98395f3.png
new file mode 100644
index 0000000000000000000000000000000000000000..8d1f063ce4b0a2f0455285ce26dc0ff4b19dafba
Binary files /dev/null and b/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_72143d6818e0e98395f3.png differ
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_c83cb9544c132f5fbda1.png b/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_c83cb9544c132f5fbda1.png
new file mode 100644
index 0000000000000000000000000000000000000000..1386ae7093ab1d45484bdbf6b3fa6a823461f94c
Binary files /dev/null and b/Meissonic/wandb/run-20251210_065438-svzut638/files/media/images/generated_videos_first_frame_500_c83cb9544c132f5fbda1.png differ
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/output.log b/Meissonic/wandb/run-20251210_065438-svzut638/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..306d8c4d5672843aeada44c0ad0a08d0ad5a6bc5
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/files/output.log
@@ -0,0 +1,119 @@
+12/10/2025 06:54:39 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 06:54:39 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 06:54:39 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 06:54:39 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 06:54:39 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 06:54:39 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 06:54:39 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 06:54:39 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 06:54:39 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 06:54:56 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 06:54:56 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 06:54:57 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 06:54:59 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 06:54:59 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 06:54:59 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 06:54:59 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 06:54:59 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 06:54:59 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 06:54:59 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 06:54:59 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 06:54:59 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 06:54:59 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 06:54:59 - INFO - __main__ - Dataloader configuration:
+12/10/2025 06:54:59 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 06:54:59 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 06:54:59 - INFO - __main__ - - persistent_workers: True
+12/10/2025 06:54:59 - INFO - __main__ - - pin_memory: True
+12/10/2025 06:54:59 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 06:55:01 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 06:55:01 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 06:55:01 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 06:55:01 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 06:55:01 - INFO - __main__ - ***** Running training *****
+12/10/2025 06:55:01 - INFO - __main__ - Num training steps = 10000
+12/10/2025 06:55:01 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 06:55:01 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 06:55:01 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 06:55:23 - INFO - __main__ - Step: 10 Loss: 11.0753 LR: 0.000060
+12/10/2025 06:55:39 - INFO - __main__ - Step: 20 Loss: 11.0730 LR: 0.000060
+12/10/2025 06:55:56 - INFO - __main__ - Step: 30 Loss: 11.0694 LR: 0.000060
+12/10/2025 06:56:13 - INFO - __main__ - Step: 40 Loss: 11.0665 LR: 0.000060
+12/10/2025 06:56:32 - INFO - __main__ - Step: 50 Loss: 11.0638 LR: 0.000060
+12/10/2025 06:56:49 - INFO - __main__ - Step: 60 Loss: 11.0581 LR: 0.000060
+12/10/2025 06:57:05 - INFO - __main__ - Step: 70 Loss: 11.0504 LR: 0.000060
+12/10/2025 06:57:22 - INFO - __main__ - Step: 80 Loss: 11.0334 LR: 0.000060
+12/10/2025 06:57:39 - INFO - __main__ - Step: 90 Loss: 11.0098 LR: 0.000060
+12/10/2025 06:57:55 - INFO - __main__ - Step: 100 Loss: 10.9694 LR: 0.000060
+12/10/2025 06:58:12 - INFO - __main__ - Step: 110 Loss: 10.9226 LR: 0.000060
+12/10/2025 06:58:29 - INFO - __main__ - Step: 120 Loss: 10.8645 LR: 0.000060
+12/10/2025 06:58:46 - INFO - __main__ - Step: 130 Loss: 10.7966 LR: 0.000060
+12/10/2025 06:59:03 - INFO - __main__ - Step: 140 Loss: 10.7388 LR: 0.000060
+12/10/2025 06:59:20 - INFO - __main__ - Step: 150 Loss: 10.6719 LR: 0.000060
+12/10/2025 06:59:38 - INFO - __main__ - Step: 160 Loss: 10.6265 LR: 0.000060
+12/10/2025 06:59:54 - INFO - __main__ - Step: 170 Loss: 10.5702 LR: 0.000060
+12/10/2025 07:00:11 - INFO - __main__ - Step: 180 Loss: 10.5286 LR: 0.000060
+12/10/2025 07:00:27 - INFO - __main__ - Step: 190 Loss: 10.4710 LR: 0.000060
+12/10/2025 07:00:44 - INFO - __main__ - Step: 200 Loss: 10.4425 LR: 0.000060
+12/10/2025 07:01:00 - INFO - __main__ - Step: 210 Loss: 10.4138 LR: 0.000060
+12/10/2025 07:01:16 - INFO - __main__ - Step: 220 Loss: 10.3857 LR: 0.000060
+12/10/2025 07:01:33 - INFO - __main__ - Step: 230 Loss: 10.3620 LR: 0.000060
+12/10/2025 07:01:49 - INFO - __main__ - Step: 240 Loss: 10.3475 LR: 0.000060
+12/10/2025 07:02:06 - INFO - __main__ - Step: 250 Loss: 10.3281 LR: 0.000060
+12/10/2025 07:02:23 - INFO - __main__ - Step: 260 Loss: 10.3099 LR: 0.000060
+12/10/2025 07:02:41 - INFO - __main__ - Step: 270 Loss: 10.3059 LR: 0.000060
+12/10/2025 07:02:59 - INFO - __main__ - Step: 280 Loss: 10.3050 LR: 0.000060
+12/10/2025 07:03:18 - INFO - __main__ - Step: 290 Loss: 10.2775 LR: 0.000060
+12/10/2025 07:03:36 - INFO - __main__ - Step: 300 Loss: 10.2930 LR: 0.000060
+12/10/2025 07:03:55 - INFO - __main__ - Step: 310 Loss: 10.2803 LR: 0.000060
+12/10/2025 07:04:12 - INFO - __main__ - Step: 320 Loss: 10.2736 LR: 0.000060
+12/10/2025 07:04:29 - INFO - __main__ - Step: 330 Loss: 10.2748 LR: 0.000060
+12/10/2025 07:04:46 - INFO - __main__ - Step: 340 Loss: 10.2525 LR: 0.000060
+12/10/2025 07:05:03 - INFO - __main__ - Step: 350 Loss: 10.2732 LR: 0.000060
+12/10/2025 07:05:20 - INFO - __main__ - Step: 360 Loss: 10.2562 LR: 0.000060
+12/10/2025 07:05:36 - INFO - __main__ - Step: 370 Loss: 10.2201 LR: 0.000060
+12/10/2025 07:05:52 - INFO - __main__ - Step: 380 Loss: 10.2399 LR: 0.000060
+12/10/2025 07:06:09 - INFO - __main__ - Step: 390 Loss: 10.2113 LR: 0.000060
+12/10/2025 07:06:25 - INFO - __main__ - Step: 400 Loss: 10.2313 LR: 0.000060
+12/10/2025 07:06:43 - INFO - __main__ - Step: 410 Loss: 10.2194 LR: 0.000060
+12/10/2025 07:07:00 - INFO - __main__ - Step: 420 Loss: 10.2263 LR: 0.000060
+12/10/2025 07:07:16 - INFO - __main__ - Step: 430 Loss: 10.2091 LR: 0.000060
+12/10/2025 07:07:33 - INFO - __main__ - Step: 440 Loss: 10.2121 LR: 0.000060
+12/10/2025 07:07:50 - INFO - __main__ - Step: 450 Loss: 10.2016 LR: 0.000060
+12/10/2025 07:08:07 - INFO - __main__ - Step: 460 Loss: 10.1867 LR: 0.000060
+12/10/2025 07:08:24 - INFO - __main__ - Step: 470 Loss: 10.1775 LR: 0.000060
+12/10/2025 07:08:41 - INFO - __main__ - Step: 480 Loss: 10.1756 LR: 0.000060
+12/10/2025 07:08:57 - INFO - __main__ - Step: 490 Loss: 10.1692 LR: 0.000060
+12/10/2025 07:09:13 - INFO - __main__ - Step: 500 Loss: 10.1555 LR: 0.000060
+12/10/2025 07:09:13 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 07:10:03 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 07:10:03 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 07:10:03 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 07:10:03 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 07:10:03 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 07:10:03 - INFO - __main__ - Generating videos for validation...
+12/10/2025 07:10:03 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 71.50it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4328.49it/s]
+12/10/2025 07:10:20 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 07:10:20 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.11it/s]
+12/10/2025 07:10:27 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 07:10:27 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1765, in
+ main(parse_args())
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1466, in main
+ mask_id = video_tokenizer.mask_token_id # codebook_size
+UnboundLocalError: local variable 'video_tokenizer' referenced before assignment
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1765, in
+[rank0]: main(parse_args())
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1466, in main
+[rank0]: mask_id = video_tokenizer.mask_token_id # codebook_size
+[rank0]: UnboundLocalError: local variable 'video_tokenizer' referenced before assignment
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/requirements.txt b/Meissonic/wandb/run-20251210_065438-svzut638/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..45ccb5ecaa4d08fdf6a5d21e01b5355286f87696
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T06:54:38.912502Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12144814084096"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "allmhscr418v3bp5824ixjkvvxtxqjh3"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-summary.json b/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c42cd11727d450d62991df63dfb701d1506021f
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/files/wandb-summary.json
@@ -0,0 +1 @@
+{"step_loss":10.155487060546875,"lr":5.9999999999999995e-05,"generated_videos_first_frame":{"captions":["a cat playing","a girl walking"],"_type":"images/separated","width":128,"height":128,"format":"png","count":2,"filenames":["media/images/generated_videos_first_frame_500_c83cb9544c132f5fbda1.png","media/images/generated_videos_first_frame_500_72143d6818e0e98395f3.png"]},"_wandb":{"runtime":988},"_runtime":988.319789275,"avg_masking_rate":0.6341594457626343,"_timestamp":1.7653506274207036e+09,"_step":500}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-core.log b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..b51c39b563cdc414c99b8950fd02e53f7b5cb1a2
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T06:54:38.982036417Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpupo0lc_m/port-2759789.txt","pid":2759789,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T06:54:38.982493461Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2759789}
+{"time":"2025-12-10T06:54:38.982502882Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2759789-2760029-2313296116/socket","Net":"unix"}}
+{"time":"2025-12-10T06:54:39.168820915Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T06:54:39.174838218Z","level":"INFO","msg":"handleInformInit: received","streamId":"svzut638","id":"1(@)"}
+{"time":"2025-12-10T06:54:39.339741536Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"svzut638","id":"1(@)"}
+{"time":"2025-12-10T07:11:07.910542343Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T07:11:07.910625646Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T07:11:07.910615044Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T07:11:07.910702456Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T07:11:07.910706025Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2759789-2760029-2313296116/socket","Net":"unix"}}
+{"time":"2025-12-10T07:11:08.290894957Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T07:11:08.290918847Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T07:11:08.290930553Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-internal.log b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..96bc03e758f51a41ee2ad8d6d85bac1f15006999
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T06:54:39.174951785Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T06:54:39.339516206Z","level":"INFO","msg":"stream: created new stream","id":"svzut638"}
+{"time":"2025-12-10T06:54:39.3396309Z","level":"INFO","msg":"handler: started","stream_id":"svzut638"}
+{"time":"2025-12-10T06:54:39.339734022Z","level":"INFO","msg":"stream: started","id":"svzut638"}
+{"time":"2025-12-10T06:54:39.339758383Z","level":"INFO","msg":"sender: started","stream_id":"svzut638"}
+{"time":"2025-12-10T06:54:39.339761537Z","level":"INFO","msg":"writer: started","stream_id":"svzut638"}
+{"time":"2025-12-10T07:11:07.910612265Z","level":"INFO","msg":"stream: closing","id":"svzut638"}
+{"time":"2025-12-10T07:11:08.148078474Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T07:11:08.287808812Z","level":"INFO","msg":"handler: closed","stream_id":"svzut638"}
+{"time":"2025-12-10T07:11:08.287886196Z","level":"INFO","msg":"sender: closed","stream_id":"svzut638"}
+{"time":"2025-12-10T07:11:08.287897958Z","level":"INFO","msg":"stream: closed","id":"svzut638"}
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug.log b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..24e08154508293316d84edef844701ec21b1f5d9
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_setup.py:_flush():80] Configure stats pid to 2759789
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug.log
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_065438-svzut638/logs/debug-internal.log
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_init.py:init():841] calling init triggers
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 06:54:38,915 INFO MainThread:2759789 [wandb_init.py:init():889] starting backend
+2025-12-10 06:54:39,168 INFO MainThread:2759789 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 06:54:39,173 INFO MainThread:2759789 [wandb_init.py:init():900] backend started and connected
+2025-12-10 06:54:39,174 INFO MainThread:2759789 [wandb_init.py:init():970] updated telemetry
+2025-12-10 06:54:39,179 INFO MainThread:2759789 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 06:54:39,590 INFO MainThread:2759789 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 06:54:39,717 INFO MainThread:2759789 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 06:54:39,718 INFO MainThread:2759789 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 06:54:39,718 INFO MainThread:2759789 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 06:54:39,718 INFO MainThread:2759789 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 06:54:39,721 INFO MainThread:2759789 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 06:54:39,722 INFO MainThread:2759789 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 07:11:07,910 INFO wandb-AsyncioManager-main:2759789 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 07:11:07,910 INFO wandb-AsyncioManager-main:2759789 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_065438-svzut638/run-svzut638.wandb b/Meissonic/wandb/run-20251210_065438-svzut638/run-svzut638.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..fd3f2d7f556e174a6f6e1edf314f4a2dc66ce2d9
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_065438-svzut638/run-svzut638.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffbf106f6012581154ef827e713d00f725e5d801d3e39c6b444e835feaf713a4
+size 295153
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/config.yaml b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..62880696d70f5ea20ae57745533f6a0eb6d7f8be
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/config.yaml
@@ -0,0 +1,305 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ ydwpsmj3zkrh2el3ienk1qod2cym9vjb:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12144814755840"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.10.19
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T07:17:16.641784Z"
+ writerId: ydwpsmj3zkrh2el3ienk1qod2cym9vjb
+ m: []
+ python_version: 3.10.19
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ "3":
+ - 61
+ "4": 3.10.19
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_0dab8879463923119ab0.png b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_0dab8879463923119ab0.png
new file mode 100644
index 0000000000000000000000000000000000000000..c5e53a182159759e474169502edacf59cbc13815
Binary files /dev/null and b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_0dab8879463923119ab0.png differ
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_79ca9194981b4a266799.png b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_79ca9194981b4a266799.png
new file mode 100644
index 0000000000000000000000000000000000000000..b882ccb65e9ff567b955e7b765f357236e0d9f1f
Binary files /dev/null and b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_1000_79ca9194981b4a266799.png differ
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_ca222b2e9040c4c4b096.png b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_ca222b2e9040c4c4b096.png
new file mode 100644
index 0000000000000000000000000000000000000000..765ed1fe60fdc94e92732ceee140f9372da13bf7
Binary files /dev/null and b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_ca222b2e9040c4c4b096.png differ
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_edd6b527675de3516a45.png b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_edd6b527675de3516a45.png
new file mode 100644
index 0000000000000000000000000000000000000000..2f6b95770e16b1495e2ca9e12d958b42ddcce55a
Binary files /dev/null and b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/media/images/generated_videos_first_frame_500_edd6b527675de3516a45.png differ
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/output.log b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..4bfbedcf9cbee7f17e3a856024c0299b304b0432
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/output.log
@@ -0,0 +1,344 @@
+12/10/2025 07:17:31 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 07:17:31 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 07:17:31 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 07:17:31 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 07:17:31 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 07:17:31 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 07:17:31 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 07:17:31 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 07:17:32 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 07:17:48 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 07:17:48 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 07:17:50 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 07:17:51 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 07:17:51 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 07:17:51 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 07:17:51 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 07:17:51 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 07:17:51 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 07:17:51 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 07:17:51 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 07:17:51 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 07:17:51 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 07:17:51 - INFO - __main__ - Dataloader configuration:
+12/10/2025 07:17:51 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 07:17:51 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 07:17:51 - INFO - __main__ - - persistent_workers: True
+12/10/2025 07:17:51 - INFO - __main__ - - pin_memory: True
+12/10/2025 07:17:51 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 07:17:53 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 07:17:53 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 07:17:53 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 07:17:53 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 07:17:53 - INFO - __main__ - ***** Running training *****
+12/10/2025 07:17:53 - INFO - __main__ - Num training steps = 10000
+12/10/2025 07:17:53 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 07:17:53 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 07:17:53 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 07:18:14 - INFO - __main__ - Step: 10 Loss: 11.0753 LR: 0.000060
+12/10/2025 07:18:31 - INFO - __main__ - Step: 20 Loss: 11.0730 LR: 0.000060
+12/10/2025 07:18:48 - INFO - __main__ - Step: 30 Loss: 11.0695 LR: 0.000060
+12/10/2025 07:19:05 - INFO - __main__ - Step: 40 Loss: 11.0665 LR: 0.000060
+12/10/2025 07:19:21 - INFO - __main__ - Step: 50 Loss: 11.0638 LR: 0.000060
+12/10/2025 07:19:38 - INFO - __main__ - Step: 60 Loss: 11.0581 LR: 0.000060
+12/10/2025 07:19:54 - INFO - __main__ - Step: 70 Loss: 11.0504 LR: 0.000060
+12/10/2025 07:20:11 - INFO - __main__ - Step: 80 Loss: 11.0334 LR: 0.000060
+12/10/2025 07:20:28 - INFO - __main__ - Step: 90 Loss: 11.0099 LR: 0.000060
+12/10/2025 07:20:45 - INFO - __main__ - Step: 100 Loss: 10.9695 LR: 0.000060
+12/10/2025 07:21:02 - INFO - __main__ - Step: 110 Loss: 10.9229 LR: 0.000060
+12/10/2025 07:21:20 - INFO - __main__ - Step: 120 Loss: 10.8647 LR: 0.000060
+12/10/2025 07:21:37 - INFO - __main__ - Step: 130 Loss: 10.7967 LR: 0.000060
+12/10/2025 07:21:54 - INFO - __main__ - Step: 140 Loss: 10.7392 LR: 0.000060
+12/10/2025 07:22:11 - INFO - __main__ - Step: 150 Loss: 10.6721 LR: 0.000060
+12/10/2025 07:22:29 - INFO - __main__ - Step: 160 Loss: 10.6267 LR: 0.000060
+12/10/2025 07:22:45 - INFO - __main__ - Step: 170 Loss: 10.5702 LR: 0.000060
+12/10/2025 07:23:01 - INFO - __main__ - Step: 180 Loss: 10.5286 LR: 0.000060
+12/10/2025 07:23:18 - INFO - __main__ - Step: 190 Loss: 10.4711 LR: 0.000060
+12/10/2025 07:23:34 - INFO - __main__ - Step: 200 Loss: 10.4432 LR: 0.000060
+12/10/2025 07:23:51 - INFO - __main__ - Step: 210 Loss: 10.4134 LR: 0.000060
+12/10/2025 07:24:07 - INFO - __main__ - Step: 220 Loss: 10.3854 LR: 0.000060
+12/10/2025 07:24:23 - INFO - __main__ - Step: 230 Loss: 10.3619 LR: 0.000060
+12/10/2025 07:24:40 - INFO - __main__ - Step: 240 Loss: 10.3476 LR: 0.000060
+12/10/2025 07:24:56 - INFO - __main__ - Step: 250 Loss: 10.3288 LR: 0.000060
+12/10/2025 07:25:13 - INFO - __main__ - Step: 260 Loss: 10.3103 LR: 0.000060
+12/10/2025 07:25:29 - INFO - __main__ - Step: 270 Loss: 10.3060 LR: 0.000060
+12/10/2025 07:25:45 - INFO - __main__ - Step: 280 Loss: 10.3030 LR: 0.000060
+12/10/2025 07:26:02 - INFO - __main__ - Step: 290 Loss: 10.2779 LR: 0.000060
+12/10/2025 07:26:18 - INFO - __main__ - Step: 300 Loss: 10.2935 LR: 0.000060
+12/10/2025 07:26:34 - INFO - __main__ - Step: 310 Loss: 10.2803 LR: 0.000060
+12/10/2025 07:26:51 - INFO - __main__ - Step: 320 Loss: 10.2737 LR: 0.000060
+12/10/2025 07:27:07 - INFO - __main__ - Step: 330 Loss: 10.2748 LR: 0.000060
+12/10/2025 07:27:23 - INFO - __main__ - Step: 340 Loss: 10.2528 LR: 0.000060
+12/10/2025 07:27:39 - INFO - __main__ - Step: 350 Loss: 10.2735 LR: 0.000060
+12/10/2025 07:27:56 - INFO - __main__ - Step: 360 Loss: 10.2561 LR: 0.000060
+12/10/2025 07:28:13 - INFO - __main__ - Step: 370 Loss: 10.2202 LR: 0.000060
+12/10/2025 07:28:29 - INFO - __main__ - Step: 380 Loss: 10.2401 LR: 0.000060
+12/10/2025 07:28:45 - INFO - __main__ - Step: 390 Loss: 10.2113 LR: 0.000060
+12/10/2025 07:29:02 - INFO - __main__ - Step: 400 Loss: 10.2319 LR: 0.000060
+12/10/2025 07:29:18 - INFO - __main__ - Step: 410 Loss: 10.2197 LR: 0.000060
+12/10/2025 07:29:35 - INFO - __main__ - Step: 420 Loss: 10.2274 LR: 0.000060
+12/10/2025 07:29:53 - INFO - __main__ - Step: 430 Loss: 10.2096 LR: 0.000060
+12/10/2025 07:30:11 - INFO - __main__ - Step: 440 Loss: 10.2125 LR: 0.000060
+12/10/2025 07:30:29 - INFO - __main__ - Step: 450 Loss: 10.2030 LR: 0.000060
+12/10/2025 07:30:46 - INFO - __main__ - Step: 460 Loss: 10.1881 LR: 0.000060
+12/10/2025 07:31:02 - INFO - __main__ - Step: 470 Loss: 10.1804 LR: 0.000060
+12/10/2025 07:31:19 - INFO - __main__ - Step: 480 Loss: 10.1765 LR: 0.000060
+12/10/2025 07:31:35 - INFO - __main__ - Step: 490 Loss: 10.1706 LR: 0.000060
+12/10/2025 07:31:51 - INFO - __main__ - Step: 500 Loss: 10.1511 LR: 0.000060
+12/10/2025 07:31:51 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 07:32:41 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 07:32:41 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 07:32:41 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 07:32:41 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 07:32:41 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 07:32:41 - INFO - __main__ - Generating videos for validation...
+12/10/2025 07:32:41 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 73.03it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 4336.80it/s]
+12/10/2025 07:32:57 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 07:32:57 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.06it/s]
+12/10/2025 07:33:04 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 07:33:04 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+12/10/2025 07:33:22 - INFO - __main__ - Step: 510 Loss: 10.1642 LR: 0.000060
+12/10/2025 07:33:39 - INFO - __main__ - Step: 520 Loss: 10.1155 LR: 0.000060
+12/10/2025 07:33:56 - INFO - __main__ - Step: 530 Loss: 10.1195 LR: 0.000060
+12/10/2025 07:34:14 - INFO - __main__ - Step: 540 Loss: 10.0654 LR: 0.000060
+12/10/2025 07:34:31 - INFO - __main__ - Step: 550 Loss: 10.0827 LR: 0.000060
+12/10/2025 07:34:48 - INFO - __main__ - Step: 560 Loss: 10.0255 LR: 0.000060
+12/10/2025 07:35:04 - INFO - __main__ - Step: 570 Loss: 10.0276 LR: 0.000060
+12/10/2025 07:35:21 - INFO - __main__ - Step: 580 Loss: 10.0056 LR: 0.000060
+12/10/2025 07:35:37 - INFO - __main__ - Step: 590 Loss: 9.9365 LR: 0.000060
+12/10/2025 07:35:54 - INFO - __main__ - Step: 600 Loss: 9.8320 LR: 0.000060
+12/10/2025 07:36:10 - INFO - __main__ - Step: 610 Loss: 10.0306 LR: 0.000060
+12/10/2025 07:36:27 - INFO - __main__ - Step: 620 Loss: 9.7723 LR: 0.000060
+12/10/2025 07:36:43 - INFO - __main__ - Step: 630 Loss: 9.6669 LR: 0.000060
+12/10/2025 07:36:59 - INFO - __main__ - Step: 640 Loss: 9.9538 LR: 0.000060
+12/10/2025 07:37:16 - INFO - __main__ - Step: 650 Loss: 9.8120 LR: 0.000060
+12/10/2025 07:37:33 - INFO - __main__ - Step: 660 Loss: 9.8283 LR: 0.000060
+12/10/2025 07:37:49 - INFO - __main__ - Step: 670 Loss: 9.5049 LR: 0.000060
+12/10/2025 07:38:06 - INFO - __main__ - Step: 680 Loss: 9.8073 LR: 0.000060
+12/10/2025 07:38:23 - INFO - __main__ - Step: 690 Loss: 9.7236 LR: 0.000060
+12/10/2025 07:38:40 - INFO - __main__ - Step: 700 Loss: 9.1967 LR: 0.000060
+12/10/2025 07:38:57 - INFO - __main__ - Step: 710 Loss: 9.3345 LR: 0.000060
+12/10/2025 07:39:14 - INFO - __main__ - Step: 720 Loss: 9.0384 LR: 0.000060
+12/10/2025 07:39:31 - INFO - __main__ - Step: 730 Loss: 9.6747 LR: 0.000060
+12/10/2025 07:39:49 - INFO - __main__ - Step: 740 Loss: 9.3858 LR: 0.000060
+12/10/2025 07:40:06 - INFO - __main__ - Step: 750 Loss: 8.8479 LR: 0.000060
+12/10/2025 07:40:23 - INFO - __main__ - Step: 760 Loss: 9.1746 LR: 0.000060
+12/10/2025 07:40:39 - INFO - __main__ - Step: 770 Loss: 8.8764 LR: 0.000060
+12/10/2025 07:40:56 - INFO - __main__ - Step: 780 Loss: 9.0416 LR: 0.000060
+12/10/2025 07:41:14 - INFO - __main__ - Step: 790 Loss: 8.6986 LR: 0.000060
+12/10/2025 07:41:32 - INFO - __main__ - Step: 800 Loss: 8.5523 LR: 0.000060
+12/10/2025 07:41:50 - INFO - __main__ - Step: 810 Loss: 7.5571 LR: 0.000060
+12/10/2025 07:42:08 - INFO - __main__ - Step: 820 Loss: 8.3334 LR: 0.000060
+12/10/2025 07:42:25 - INFO - __main__ - Step: 830 Loss: 7.9969 LR: 0.000060
+12/10/2025 07:42:42 - INFO - __main__ - Step: 840 Loss: 8.5891 LR: 0.000060
+12/10/2025 07:42:58 - INFO - __main__ - Step: 850 Loss: 7.8850 LR: 0.000060
+12/10/2025 07:43:14 - INFO - __main__ - Step: 860 Loss: 8.7087 LR: 0.000060
+12/10/2025 07:43:31 - INFO - __main__ - Step: 870 Loss: 8.1369 LR: 0.000060
+12/10/2025 07:43:48 - INFO - __main__ - Step: 880 Loss: 8.4226 LR: 0.000060
+12/10/2025 07:44:04 - INFO - __main__ - Step: 890 Loss: 8.6348 LR: 0.000060
+12/10/2025 07:44:21 - INFO - __main__ - Step: 900 Loss: 7.6271 LR: 0.000060
+12/10/2025 07:44:37 - INFO - __main__ - Step: 910 Loss: 8.4708 LR: 0.000060
+12/10/2025 07:44:53 - INFO - __main__ - Step: 920 Loss: 7.6171 LR: 0.000060
+12/10/2025 07:45:09 - INFO - __main__ - Step: 930 Loss: 7.9358 LR: 0.000060
+12/10/2025 07:45:27 - INFO - __main__ - Step: 940 Loss: 6.9904 LR: 0.000060
+12/10/2025 07:45:43 - INFO - __main__ - Step: 950 Loss: 7.5480 LR: 0.000060
+12/10/2025 07:46:00 - INFO - __main__ - Step: 960 Loss: 7.7005 LR: 0.000060
+12/10/2025 07:46:17 - INFO - __main__ - Step: 970 Loss: 4.9190 LR: 0.000060
+12/10/2025 07:46:36 - INFO - __main__ - Step: 980 Loss: 7.4162 LR: 0.000060
+12/10/2025 07:46:52 - INFO - __main__ - Step: 990 Loss: 7.0244 LR: 0.000060
+12/10/2025 07:47:10 - INFO - __main__ - Step: 1000 Loss: 6.2988 LR: 0.000060
+12/10/2025 07:47:10 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000
+12/10/2025 07:47:16 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/optimizer.bin
+12/10/2025 07:47:16 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/scheduler.bin
+12/10/2025 07:47:16 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/sampler.bin
+12/10/2025 07:47:16 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/random_states_0.pkl
+12/10/2025 07:47:16 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000
+12/10/2025 07:47:16 - INFO - __main__ - Generating videos for validation...
+12/10/2025 07:47:16 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 74.40it/s]
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 7616.12it/s]
+12/10/2025 07:47:32 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 07:47:32 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.15it/s]
+12/10/2025 07:47:38 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 07:47:38 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+12/10/2025 07:47:55 - INFO - __main__ - Step: 1010 Loss: 6.0339 LR: 0.000060
+12/10/2025 07:48:12 - INFO - __main__ - Step: 1020 Loss: 5.9514 LR: 0.000060
+12/10/2025 07:48:29 - INFO - __main__ - Step: 1030 Loss: 3.9624 LR: 0.000060
+12/10/2025 07:48:47 - INFO - __main__ - Step: 1040 Loss: 6.0093 LR: 0.000060
+12/10/2025 07:49:05 - INFO - __main__ - Step: 1050 Loss: 6.8974 LR: 0.000060
+12/10/2025 07:49:23 - INFO - __main__ - Step: 1060 Loss: 6.1530 LR: 0.000060
+12/10/2025 07:49:42 - INFO - __main__ - Step: 1070 Loss: 6.1358 LR: 0.000060
+12/10/2025 07:50:00 - INFO - __main__ - Step: 1080 Loss: 4.2982 LR: 0.000060
+12/10/2025 07:50:18 - INFO - __main__ - Step: 1090 Loss: 4.7812 LR: 0.000060
+12/10/2025 07:50:36 - INFO - __main__ - Step: 1100 Loss: 5.3707 LR: 0.000060
+12/10/2025 07:50:55 - INFO - __main__ - Step: 1110 Loss: 5.8028 LR: 0.000060
+12/10/2025 07:51:11 - INFO - __main__ - Step: 1120 Loss: 5.2905 LR: 0.000060
+12/10/2025 07:51:27 - INFO - __main__ - Step: 1130 Loss: 4.7033 LR: 0.000060
+12/10/2025 07:51:44 - INFO - __main__ - Step: 1140 Loss: 4.0230 LR: 0.000060
+12/10/2025 07:52:01 - INFO - __main__ - Step: 1150 Loss: 5.4948 LR: 0.000060
+12/10/2025 07:52:17 - INFO - __main__ - Step: 1160 Loss: 4.7230 LR: 0.000060
+12/10/2025 07:52:34 - INFO - __main__ - Step: 1170 Loss: 4.5157 LR: 0.000060
+12/10/2025 07:52:51 - INFO - __main__ - Step: 1180 Loss: 4.5091 LR: 0.000060
+12/10/2025 07:53:07 - INFO - __main__ - Step: 1190 Loss: 6.7011 LR: 0.000060
+12/10/2025 07:53:23 - INFO - __main__ - Step: 1200 Loss: 2.4988 LR: 0.000060
+12/10/2025 07:53:40 - INFO - __main__ - Step: 1210 Loss: 6.0789 LR: 0.000060
+12/10/2025 07:53:56 - INFO - __main__ - Step: 1220 Loss: 2.7961 LR: 0.000060
+12/10/2025 07:54:13 - INFO - __main__ - Step: 1230 Loss: 3.2556 LR: 0.000060
+12/10/2025 07:54:29 - INFO - __main__ - Step: 1240 Loss: 3.3652 LR: 0.000060
+12/10/2025 07:54:46 - INFO - __main__ - Step: 1250 Loss: 2.4001 LR: 0.000060
+12/10/2025 07:55:03 - INFO - __main__ - Step: 1260 Loss: 3.9875 LR: 0.000060
+12/10/2025 07:55:19 - INFO - __main__ - Step: 1270 Loss: 2.5632 LR: 0.000060
+12/10/2025 07:55:36 - INFO - __main__ - Step: 1280 Loss: 5.4715 LR: 0.000060
+12/10/2025 07:55:53 - INFO - __main__ - Step: 1290 Loss: 3.4570 LR: 0.000060
+12/10/2025 07:56:09 - INFO - __main__ - Step: 1300 Loss: 4.6496 LR: 0.000060
+12/10/2025 07:56:25 - INFO - __main__ - Step: 1310 Loss: 4.8271 LR: 0.000060
+12/10/2025 07:56:42 - INFO - __main__ - Step: 1320 Loss: 1.8501 LR: 0.000060
+12/10/2025 07:56:59 - INFO - __main__ - Step: 1330 Loss: 6.8129 LR: 0.000060
+12/10/2025 07:57:15 - INFO - __main__ - Step: 1340 Loss: 3.6670 LR: 0.000060
+12/10/2025 07:57:32 - INFO - __main__ - Step: 1350 Loss: 4.4606 LR: 0.000060
+12/10/2025 07:57:48 - INFO - __main__ - Step: 1360 Loss: 3.2188 LR: 0.000060
+12/10/2025 07:58:05 - INFO - __main__ - Step: 1370 Loss: 6.0791 LR: 0.000060
+12/10/2025 07:58:22 - INFO - __main__ - Step: 1380 Loss: 3.4297 LR: 0.000060
+12/10/2025 07:58:38 - INFO - __main__ - Step: 1390 Loss: 3.6190 LR: 0.000060
+12/10/2025 07:58:55 - INFO - __main__ - Step: 1400 Loss: 4.6799 LR: 0.000060
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1774, in
+ if accelerator.is_main_process:
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1525, in main
+ logits = model(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+ x = cross_attn_ffn(x, context, context_lens, e)
+ File "/mnt/Meissonic/src/transformer_video.py", line 477, in cross_attn_ffn
+ x = x + self.cross_attn(self.norm3(x), context, context_lens)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 389, in forward
+ k = self.norm_k(self.k(context)).view(b, -1, n, d)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+KeyboardInterrupt
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1774, in
+[rank0]: if accelerator.is_main_process:
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1525, in main
+[rank0]: logits = model(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 489, in forward
+[rank0]: x = cross_attn_ffn(x, context, context_lens, e)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 477, in cross_attn_ffn
+[rank0]: x = x + self.cross_attn(self.norm3(x), context, context_lens)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 389, in forward
+[rank0]: k = self.norm_k(self.k(context)).view(b, -1, n, d)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/matrix-game2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: KeyboardInterrupt
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/requirements.txt b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d01ecad871b6b3baba9900a3b3d370e9205a61d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/requirements.txt
@@ -0,0 +1,151 @@
+ImageIO==2.37.2
+typing-inspection==0.4.2
+av==16.0.1
+dill==0.4.0
+matplotlib==3.10.7
+xxhash==3.6.0
+tap==0.2
+mc_bin_client==1.0.1
+exceptiongroup==1.3.1
+cycler==0.12.1
+einops==0.8.1
+opencv-python==4.12.0.88
+scikit-image==0.25.2
+dashscope==1.25.2
+charset-normalizer==3.4.4
+filelock==3.19.1
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+lazy_loader==0.4
+kiwisolver==1.4.9
+Flask==3.1.2
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+setuptools==80.9.0
+websocket-client==1.9.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+itsdangerous==2.2.0
+pydantic_core==2.41.5
+matrix-game-2.0==0.0.1
+wsproto==1.3.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+nvidia-cusparselt-cu12==0.7.1
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+lightning==2.6.0
+Flask-SocketIO==5.5.1
+torchvision==0.24.1
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+fonttools==4.61.0
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+modelscope==1.32.0
+ftfy==6.3.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+beartype==0.22.8
+dominate==2.9.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+easydict==1.13
+networkx==3.3
+wheel==0.45.1
+timm==1.0.22
+pyparsing==3.2.5
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+pfzy==0.3.4
+httpcore==1.0.9
+multidict==6.7.0
+pycparser==2.23
+regex==2025.11.3
+importlib_metadata==8.7.0
+Werkzeug==3.1.4
+antlr4-python3-runtime==4.9.3
+sentry-sdk==2.46.0
+urllib3==2.5.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+torchmetrics==1.8.2
+cryptography==46.0.3
+omegaconf==2.3.0
+cffi==2.0.0
+packaging==25.0
+inquirerpy==0.3.4
+aiosignal==1.4.0
+MarkupSafe==2.1.5
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+decord==0.6.0
+async-timeout==5.0.1
+sympy==1.14.0
+numpy==2.1.2
+torch==2.9.1
+diffusers==0.35.2
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+tifffile==2025.5.10
+safetensors==0.7.0
+gitdb==4.0.12
+blinker==1.9.0
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+typer-slim==0.20.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+python-engineio==4.12.3
+lmdb==1.7.5
+nvidia-nvtx-cu12==12.8.90
+fsspec==2025.9.0
+markdown-it-py==4.0.0
+six==1.17.0
+platformdirs==4.5.0
+starlette==0.50.0
+scipy==1.15.3
+pycocotools==2.0.10
+accelerate==1.12.0
+zipp==3.23.0
+propcache==0.4.1
+bidict==0.23.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+simple-websocket==1.1.0
+nvidia-curand-cu12==10.3.9.90
+contourpy==1.3.2
+imageio-ffmpeg==0.6.0
+nvidia-cufft-cu12==11.3.3.83
+pydantic==2.12.5
+pip==25.3
+prompt_toolkit==3.0.52
+pillow==11.3.0
+protobuf==6.33.1
+yarl==1.22.0
+clip==1.0
+nvidia-cudnn-cu12==9.10.2.21
+python-socketio==5.15.0
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe02fb11161661d5644632029f93d8d387062b74
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.10.19",
+ "startedAt": "2025-12-10T07:17:16.641784Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/matrix-game2/bin/python3.10",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12144814755840"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "ydwpsmj3zkrh2el3ienk1qod2cym9vjb"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-summary.json b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..b4d84d9a4ea753766a191277000a4b01d5a32bc6
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":2493},"_runtime":2493.834201701,"avg_masking_rate":0.6237305402755737,"_timestamp":1.7653535354021459e+09,"_step":1400,"step_loss":4.679940700531006,"lr":5.9999999999999995e-05,"generated_videos_first_frame":{"count":2,"filenames":["media/images/generated_videos_first_frame_1000_79ca9194981b4a266799.png","media/images/generated_videos_first_frame_1000_0dab8879463923119ab0.png"],"captions":["a cat playing","a girl walking"],"_type":"images/separated","width":128,"height":128,"format":"png"}}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-core.log b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..15ea64394d59c02984a80e416c94e13d83df2b2f
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-core.log
@@ -0,0 +1,12 @@
+{"time":"2025-12-10T07:17:16.714578164Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpomdelq4a/port-2831568.txt","pid":2831568,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T07:17:16.715052713Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2831568}
+{"time":"2025-12-10T07:17:16.71505599Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2831568-2831806-3920032682/socket","Net":"unix"}}
+{"time":"2025-12-10T07:17:16.901170368Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T07:17:16.90805965Z","level":"INFO","msg":"handleInformInit: received","streamId":"kc9aapl4","id":"1(@)"}
+{"time":"2025-12-10T07:17:17.075066279Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"kc9aapl4","id":"1(@)"}
+{"time":"2025-12-10T07:59:05.681805405Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T07:59:05.681949364Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T07:59:05.682036992Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T07:59:05.681970304Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T07:59:05.6821211Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-2831568-2831806-3920032682/socket","Net":"unix"}}
+{"time":"2025-12-10T07:59:06.131796407Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-internal.log b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..876aa4da1059b7073ca50b93148f66c2853da60d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-internal.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-10T07:17:16.908162531Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T07:17:17.074836072Z","level":"INFO","msg":"stream: created new stream","id":"kc9aapl4"}
+{"time":"2025-12-10T07:17:17.074944098Z","level":"INFO","msg":"handler: started","stream_id":"kc9aapl4"}
+{"time":"2025-12-10T07:17:17.075059244Z","level":"INFO","msg":"stream: started","id":"kc9aapl4"}
+{"time":"2025-12-10T07:17:17.075075183Z","level":"INFO","msg":"writer: started","stream_id":"kc9aapl4"}
+{"time":"2025-12-10T07:17:17.075079961Z","level":"INFO","msg":"sender: started","stream_id":"kc9aapl4"}
+{"time":"2025-12-10T07:59:05.681907531Z","level":"INFO","msg":"stream: closing","id":"kc9aapl4"}
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug.log b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..2329603fb33a68c67234d108c9a0f1b81b04aeb2
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_setup.py:_flush():80] Configure stats pid to 2831568
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug.log
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_071716-kc9aapl4/logs/debug-internal.log
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_init.py:init():841] calling init triggers
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 07:17:16,645 INFO MainThread:2831568 [wandb_init.py:init():889] starting backend
+2025-12-10 07:17:16,901 INFO MainThread:2831568 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 07:17:16,906 INFO MainThread:2831568 [wandb_init.py:init():900] backend started and connected
+2025-12-10 07:17:16,908 INFO MainThread:2831568 [wandb_init.py:init():970] updated telemetry
+2025-12-10 07:17:16,912 INFO MainThread:2831568 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 07:17:31,847 INFO MainThread:2831568 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 07:17:31,970 INFO MainThread:2831568 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 07:17:31,970 INFO MainThread:2831568 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 07:17:31,970 INFO MainThread:2831568 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 07:17:31,970 INFO MainThread:2831568 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 07:17:31,973 INFO MainThread:2831568 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 07:17:31,974 INFO MainThread:2831568 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 07:59:05,682 INFO wandb-AsyncioManager-main:2831568 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 07:59:05,682 INFO wandb-AsyncioManager-main:2831568 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_071716-kc9aapl4/run-kc9aapl4.wandb b/Meissonic/wandb/run-20251210_071716-kc9aapl4/run-kc9aapl4.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..7a59cbcddb54563819bf66448b3463483bc02221
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_071716-kc9aapl4/run-kc9aapl4.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3e14587a993a57ae4b81ee401af9f90403ae0857025856ad896d75cda2d44c
+size 720896
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/config.yaml b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8b4ef07bc459810d940e0adb64777dbee0f98e99
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/config.yaml
@@ -0,0 +1,305 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ 4fj7z9cxfwc9zu6b03d9bmfdsnai8a5p:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12199603720192"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/test/bin/python3.13
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.13.10
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T10:36:42.008952Z"
+ writerId: 4fj7z9cxfwc9zu6b03d9bmfdsnai8a5p
+ m: []
+ python_version: 3.13.10
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ - 105
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ - 105
+ "4": 3.13.10
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/output.log b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..a937ee97e45916ab4ef0062b8e8e49acdfa71d2f
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/output.log
@@ -0,0 +1,211 @@
+12/10/2025 10:36:42 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 10:36:42 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 10:36:42 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 10:36:42 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 10:36:42 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 10:36:42 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 10:36:42 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 10:36:42 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:36:42 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 10:37:00 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:37:00 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:37:03 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 10:37:05 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 10:37:05 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 10:37:05 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 10:37:05 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 10:37:05 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 10:37:05 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 10:37:05 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 10:37:05 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 10:37:05 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 10:37:05 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 10:37:05 - INFO - __main__ - Dataloader configuration:
+12/10/2025 10:37:05 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 10:37:05 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 10:37:05 - INFO - __main__ - - persistent_workers: True
+12/10/2025 10:37:05 - INFO - __main__ - - pin_memory: True
+12/10/2025 10:37:05 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 10:37:06 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 10:37:06 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 10:37:06 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 10:37:06 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 10:37:06 - INFO - __main__ - ***** Running training *****
+12/10/2025 10:37:06 - INFO - __main__ - Num training steps = 10000
+12/10/2025 10:37:06 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 10:37:06 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 10:37:06 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1786, in
+ main(parse_args())
+ ~~~~^^^^^^^^^^^^^^
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1525, in main
+ logits = model(
+ tokens=input_ids, # [B, F', H', W']
+ ...<2 lines>...
+ y=None,
+ ) # Returns [B, vocab_size, F', H', W']
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+ else self._run_ddp_forward(*inputs, **kwargs)
+ ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+ return self.module(*inputs, **kwargs) # type: ignore[index]
+ ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+ return super().__call__(*args, **kwargs)
+ ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/accelerate/utils/operations.py", line 819, in forward
+ return model_forward(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/accelerate/utils/operations.py", line 807, in __call__
+ return convert_to_fp32(self.model_forward(*args, **kwargs))
+ ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+ return func(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+ out_list = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(self.backbone),
+ ...<5 lines>...
+ **ckpt_kwargs,
+ )
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_compile.py", line 53, in inner
+ return disable_fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+ return fn(*args, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+ ret = function(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+ return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+ x = block(x, **kwargs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+ y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+ return self._call_impl(*args, **kwargs)
+ ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+ return forward_call(*args, **kwargs)
+ File "/mnt/Meissonic/src/transformer_video.py", line 357, in forward
+ x = flash_attention(
+ q=rope_apply(q, grid_sizes, freqs),
+ ...<2 lines>...
+ k_lens=seq_lens,
+ window_size=self.window_size)
+ File "/mnt/Meissonic/src/transformer_video.py", line 124, in flash_attention
+ assert FLASH_ATTN_2_AVAILABLE
+ ^^^^^^^^^^^^^^^^^^^^^^
+AssertionError
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1786, in
+[rank0]: main(parse_args())
+[rank0]: ~~~~^^^^^^^^^^^^^^
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1525, in main
+[rank0]: logits = model(
+[rank0]: tokens=input_ids, # [B, F', H', W']
+[rank0]: ...<2 lines>...
+[rank0]: y=None,
+[rank0]: ) # Returns [B, vocab_size, F', H', W']
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1661, in forward
+[rank0]: else self._run_ddp_forward(*inputs, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/parallel/distributed.py", line 1487, in _run_ddp_forward
+[rank0]: return self.module(*inputs, **kwargs) # type: ignore[index]
+[rank0]: ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
+[rank0]: return super().__call__(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/accelerate/utils/operations.py", line 819, in forward
+[rank0]: return model_forward(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/accelerate/utils/operations.py", line 807, in __call__
+[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
+[rank0]: ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+[rank0]: return func(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1016, in forward
+[rank0]: out_list = torch.utils.checkpoint.checkpoint(
+[rank0]: create_custom_forward(self.backbone),
+[rank0]: ...<5 lines>...
+[rank0]: **ckpt_kwargs,
+[rank0]: )
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_compile.py", line 53, in inner
+[rank0]: return disable_fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]: return fn(*args, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+[rank0]: ret = function(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 1011, in custom_forward
+[rank0]: return module(x=x_in, t=t_in, context=context_in, seq_len=seq_len_in, y=y_in)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 732, in forward
+[rank0]: x = block(x, **kwargs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 471, in forward
+[rank0]: y = self.self_attn(attn_input, seq_lens, grid_sizes, freqs)
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]: return self._call_impl(*args, **kwargs)
+[rank0]: ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]: return forward_call(*args, **kwargs)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 357, in forward
+[rank0]: x = flash_attention(
+[rank0]: q=rope_apply(q, grid_sizes, freqs),
+[rank0]: ...<2 lines>...
+[rank0]: k_lens=seq_lens,
+[rank0]: window_size=self.window_size)
+[rank0]: File "/mnt/Meissonic/src/transformer_video.py", line 124, in flash_attention
+[rank0]: assert FLASH_ATTN_2_AVAILABLE
+[rank0]: ^^^^^^^^^^^^^^^^^^^^^^
+[rank0]: AssertionError
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/requirements.txt b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5d79dc78d38d4cd741be5be11dd4475290a154d2
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/requirements.txt
@@ -0,0 +1,121 @@
+typing-inspection==0.4.2
+dask==2025.11.0
+dill==0.4.0
+ffmpy==1.0.0
+xxhash==3.6.0
+partd==1.4.2
+brotli==1.2.0
+charset-normalizer==3.4.4
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+orjson==3.11.5
+pydantic_core==2.41.5
+groovy==0.1.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+MarkupSafe==3.0.3
+protobuf==6.33.2
+typer==0.20.0
+gradio==6.1.0
+sentry-sdk==2.47.0
+nvidia-cusparselt-cu12==0.7.1
+locket==1.0.0
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+pydub==0.25.1
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+torchvision==0.24.1
+fastapi==0.124.0
+cloudpickle==3.1.2
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+open_clip_torch==3.2.0
+mdurl==0.1.2
+pandas==2.3.3
+toolz==1.1.0
+ftfy==6.3.1
+platformdirs==4.5.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+wheel==0.45.1
+timm==1.0.22
+semantic-version==2.10.0
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+httpcore==1.0.9
+fsspec==2025.10.0
+multidict==6.7.0
+regex==2025.11.3
+gradio_client==2.0.1
+importlib_metadata==8.7.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+filelock==3.20.0
+torchmetrics==1.8.2
+numpy==2.3.5
+uvicorn==0.38.0
+packaging==25.0
+aiosignal==1.4.0
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+urllib3==2.6.1
+networkx==3.6.1
+setuptools==80.9.0
+sympy==1.14.0
+torch==2.9.1
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+safetensors==0.7.0
+gitdb==4.0.12
+safehttpx==0.1.7
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+pillow==12.0.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+nvidia-nvtx-cu12==12.8.90
+markdown-it-py==4.0.0
+six==1.17.0
+starlette==0.50.0
+audioop-lts==0.2.2
+accelerate==1.12.0
+diffusers==0.36.0
+annotated-doc==0.0.4
+zipp==3.23.0
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-curand-cu12==10.3.9.90
+python-multipart==0.0.20
+nvidia-cufft-cu12==11.3.3.83
+pip==25.3
+aiofiles==24.1.0
+yarl==1.22.0
+nvidia-cudnn-cu12==9.10.2.21
+tomlkit==0.13.3
+pydantic==2.12.4
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7439b4cb5ac48814cb0810c311356e386d2aefa
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.13.10",
+ "startedAt": "2025-12-10T10:36:42.008952Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/test/bin/python3.13",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12199603720192"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "4fj7z9cxfwc9zu6b03d9bmfdsnai8a5p"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-summary.json b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa34b295a76fbe8b47b19e3ac892049d460b7ba8
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/files/wandb-summary.json
@@ -0,0 +1 @@
+{"_wandb":{"runtime":68},"_runtime":68}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-core.log b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..5780d21d630c6b99da01913e2ac2da61aa1666db
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T10:36:42.091100269Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmpen4dnq09/port-1177490.txt","pid":1177490,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T10:36:42.091692568Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1177490}
+{"time":"2025-12-10T10:36:42.091708517Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1177490-1177781-2755904903/socket","Net":"unix"}}
+{"time":"2025-12-10T10:36:42.277704585Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T10:36:42.28458188Z","level":"INFO","msg":"handleInformInit: received","streamId":"tkz8w8yv","id":"1(@)"}
+{"time":"2025-12-10T10:36:42.455192024Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tkz8w8yv","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.456333068Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.456411118Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.456431784Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T10:37:51.456575184Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.458699605Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1177490-1177781-2755904903/socket","Net":"unix"}}
+{"time":"2025-12-10T10:37:51.77414998Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.774173687Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T10:37:51.774184046Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-internal.log b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..898c0bbd9bc23a4de40f4599f2a43b6d27ede1f7
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T10:36:42.284695178Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T10:36:42.454961318Z","level":"INFO","msg":"stream: created new stream","id":"tkz8w8yv"}
+{"time":"2025-12-10T10:36:42.455049148Z","level":"INFO","msg":"handler: started","stream_id":"tkz8w8yv"}
+{"time":"2025-12-10T10:36:42.455185135Z","level":"INFO","msg":"stream: started","id":"tkz8w8yv"}
+{"time":"2025-12-10T10:36:42.455198542Z","level":"INFO","msg":"writer: started","stream_id":"tkz8w8yv"}
+{"time":"2025-12-10T10:36:42.45520445Z","level":"INFO","msg":"sender: started","stream_id":"tkz8w8yv"}
+{"time":"2025-12-10T10:37:51.456438757Z","level":"INFO","msg":"stream: closing","id":"tkz8w8yv"}
+{"time":"2025-12-10T10:37:51.672305417Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T10:37:51.770950948Z","level":"INFO","msg":"handler: closed","stream_id":"tkz8w8yv"}
+{"time":"2025-12-10T10:37:51.77105342Z","level":"INFO","msg":"sender: closed","stream_id":"tkz8w8yv"}
+{"time":"2025-12-10T10:37:51.771061354Z","level":"INFO","msg":"stream: closed","id":"tkz8w8yv"}
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug.log b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..e6a588ee6aca52e9d4fecdab90bb641591b29821
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_setup.py:_flush():80] Configure stats pid to 1177490
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug.log
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_103642-tkz8w8yv/logs/debug-internal.log
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_init.py:init():841] calling init triggers
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 10:36:42,010 INFO MainThread:1177490 [wandb_init.py:init():889] starting backend
+2025-12-10 10:36:42,278 INFO MainThread:1177490 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 10:36:42,283 INFO MainThread:1177490 [wandb_init.py:init():900] backend started and connected
+2025-12-10 10:36:42,284 INFO MainThread:1177490 [wandb_init.py:init():970] updated telemetry
+2025-12-10 10:36:42,289 INFO MainThread:1177490 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 10:36:42,673 INFO MainThread:1177490 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 10:36:42,760 INFO MainThread:1177490 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 10:36:42,760 INFO MainThread:1177490 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 10:36:42,760 INFO MainThread:1177490 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 10:36:42,760 INFO MainThread:1177490 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 10:36:42,763 INFO MainThread:1177490 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 10:36:42,764 INFO MainThread:1177490 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 10:37:51,456 INFO wandb-AsyncioManager-main:1177490 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 10:37:51,456 INFO wandb-AsyncioManager-main:1177490 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_103642-tkz8w8yv/run-tkz8w8yv.wandb b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/run-tkz8w8yv.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..7ad45294dfeddac82165f803b0b3e0fc19dd33b5
Binary files /dev/null and b/Meissonic/wandb/run-20251210_103642-tkz8w8yv/run-tkz8w8yv.wandb differ
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/config.yaml b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b0bbd61f2d547d7828a6bade3e9d464eb29fd58d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/config.yaml
@@ -0,0 +1,307 @@
+_wandb:
+ value:
+ cli_version: 0.23.1
+ e:
+ gsh7dojjv7efi33ri5grdscpp81d6a5z:
+ args:
+ - --use_precomputed_features
+ - --features_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+ - --text_encoder_architecture
+ - umt5-xxl
+ - --wan_pretrained_path
+ - Wan-AI/Wan2.1-T2V-1.3B
+ - --training_from_scratch
+ - "True"
+ - --pretrained_model_name_or_path
+ - dummy
+ - --wan_backbone_lr_ratio
+ - "0.2"
+ - --num_frames
+ - "17"
+ - --video_height
+ - "128"
+ - --video_width
+ - "128"
+ - --dataloader_num_workers
+ - "8"
+ - --video_tokenizer_model_id
+ - Cosmos-0.1-Tokenizer-DV4x8x8
+ - --instance_dataset
+ - OpenVid1MDataset
+ - --instance_data_dir
+ - /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+ - --train_batch_size
+ - "8"
+ - --gradient_accumulation_steps
+ - "4"
+ - --learning_rate
+ - "3e-4"
+ - --max_train_steps
+ - "10000"
+ - --checkpointing_steps
+ - "500"
+ - --validation_steps
+ - "500"
+ - --logging_steps
+ - "10"
+ - --validation_prompts
+ - a cat playing
+ - a girl walking
+ - --output_dir
+ - ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+ - --mixed_precision
+ - bf16
+ - --lr_scheduler
+ - constant
+ - --lr_warmup_steps
+ - "0"
+ - --use_8bit_adam
+ - --gradient_checkpointing
+ - --min_masking_rate
+ - "0.0"
+ - --cond_dropout_prob
+ - "0.1"
+ - --split_vae_encode
+ - "1"
+ - --allow_tf32
+ - --seed
+ - "42"
+ - --report_to
+ - wandb
+ codePath: train/train_mei_video.py
+ codePathLocal: train/train_mei_video.py
+ cpu_count: 48
+ cpu_count_logical: 96
+ cudaVersion: "12.8"
+ disk:
+ /:
+ total: "16650112278528"
+ used: "12200011956224"
+ email: jinbin5bai@gmail.com
+ executable: /home/ubuntu/miniconda3/envs/test/bin/python3.13
+ git:
+ commit: 6819d374ef1b86bdedad373aab1121a89687e5cf
+ remote: https://github.com/viiika/Meissonic.git
+ gpu: NVIDIA A100-SXM4-40GB
+ gpu_count: 8
+ gpu_nvidia:
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-71102f28-cd17-57e7-6181-120bf743d23d
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-303ab142-3206-9a14-c758-58ab97d7510e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e
+ - architecture: Ampere
+ cudaCores: 6912
+ memoryTotal: "42949672960"
+ name: NVIDIA A100-SXM4-40GB
+ uuid: GPU-efb2d1fc-1eed-653d-ed51-5273085154ba
+ host: ip-172-31-91-136
+ memory:
+ total: "1204521451520"
+ os: Linux-6.8.0-1027-aws-x86_64-with-glibc2.35
+ program: /mnt/Meissonic/train/train_mei_video.py
+ python: CPython 3.13.10
+ root: /mnt/Meissonic
+ startedAt: "2025-12-10T10:58:33.251591Z"
+ writerId: gsh7dojjv7efi33ri5grdscpp81d6a5z
+ m: []
+ python_version: 3.13.10
+ t:
+ "1":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ - 105
+ "2":
+ - 1
+ - 11
+ - 41
+ - 49
+ - 51
+ - 71
+ - 83
+ - 98
+ - 105
+ "3":
+ - 61
+ "4": 3.13.10
+ "5": 0.23.1
+ "6": 4.57.3
+ "12": 0.23.1
+ "13": linux-x86_64
+adam_beta1:
+ value: 0.9
+adam_beta2:
+ value: 0.999
+adam_epsilon:
+ value: 1e-08
+adam_weight_decay:
+ value: 0.01
+allow_tf32:
+ value: true
+checkpointing_steps:
+ value: 500
+checkpoints_total_limit:
+ value: null
+cond_dropout_prob:
+ value: 0.1
+dataloader_num_workers:
+ value: 8
+dataloader_prefetch_factor:
+ value: 2
+ema_decay:
+ value: 0.9999
+ema_update_after_step:
+ value: 0
+empty_embeds_path:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+features_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+freeze_wan_backbone:
+ value: false
+gradient_accumulation_steps:
+ value: 4
+gradient_checkpointing:
+ value: true
+image_key:
+ value: null
+instance_data_dataset:
+ value: null
+instance_data_dir:
+ value: /mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv
+instance_data_image:
+ value: null
+instance_dataset:
+ value: OpenVid1MDataset
+learning_rate:
+ value: 0.0003
+logging_dir:
+ value: logs
+logging_steps:
+ value: 10
+lora_alpha:
+ value: 32
+lora_r:
+ value: 16
+lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+lr_scheduler:
+ value: constant
+lr_warmup_steps:
+ value: 0
+max_grad_norm:
+ value: 50
+max_train_steps:
+ value: 10000
+min_masking_rate:
+ value: 0
+mixed_precision:
+ value: bf16
+num_frames:
+ value: 17
+output_dir:
+ value: ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+pretrained_model_name_or_path:
+ value: dummy
+prompt_key:
+ value: null
+prompt_prefix:
+ value: null
+report_to:
+ value: wandb
+resolution:
+ value: 512
+resume_from_checkpoint:
+ value: null
+revision:
+ value: null
+scale_lr:
+ value: false
+seed:
+ value: 42
+split_vae_encode:
+ value: 1
+text_encoder_architecture:
+ value: umt5-xxl
+text_encoder_lora_alpha:
+ value: 32
+text_encoder_lora_r:
+ value: 16
+text_encoder_lora_target_modules:
+ value:
+ - to_q
+ - to_k
+ - to_v
+text_encoder_use_lora:
+ value: false
+train_batch_size:
+ value: 8
+train_text_encoder:
+ value: false
+training_from_scratch:
+ value: true
+use_8bit_adam:
+ value: true
+use_ema:
+ value: false
+use_lora:
+ value: false
+use_precomputed_features:
+ value: true
+validation_prompts:
+ value:
+ - a cat playing
+ - a girl walking
+validation_steps:
+ value: 500
+variant:
+ value: null
+video_height:
+ value: 128
+video_tokenizer_model_id:
+ value: Cosmos-0.1-Tokenizer-DV4x8x8
+video_width:
+ value: 128
+wan_backbone_lr_ratio:
+ value: 0.2
+wan_pretrained_path:
+ value: Wan-AI/Wan2.1-T2V-1.3B
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/output.log b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..aef3cb03f1fa8ed45f682c752a8ddc79d657ef91
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/output.log
@@ -0,0 +1,133 @@
+12/10/2025 10:58:34 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 10:58:34 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 10:58:34 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 10:58:34 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 10:58:34 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 10:58:34 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 10:58:34 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 10:58:34 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:58:34 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 10:58:50 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:58:50 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 10:58:52 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 10:58:54 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 10:58:54 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 10:58:54 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 10:58:54 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 10:58:54 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 10:58:54 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 10:58:54 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 10:58:54 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 10:58:54 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 10:58:54 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 10:58:54 - INFO - __main__ - Dataloader configuration:
+12/10/2025 10:58:54 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 10:58:54 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 10:58:54 - INFO - __main__ - - persistent_workers: True
+12/10/2025 10:58:54 - INFO - __main__ - - pin_memory: True
+12/10/2025 10:58:54 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 10:58:55 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 10:58:55 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 10:58:55 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 10:58:55 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 10:58:55 - INFO - __main__ - ***** Running training *****
+12/10/2025 10:58:55 - INFO - __main__ - Num training steps = 10000
+12/10/2025 10:58:55 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 10:58:55 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 10:58:55 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 10:59:21 - INFO - __main__ - Step: 10 Loss: 11.0753 LR: 0.000060
+12/10/2025 10:59:37 - INFO - __main__ - Step: 20 Loss: 11.0730 LR: 0.000060
+12/10/2025 10:59:52 - INFO - __main__ - Step: 30 Loss: 11.0695 LR: 0.000060
+12/10/2025 11:00:09 - INFO - __main__ - Step: 40 Loss: 11.0665 LR: 0.000060
+12/10/2025 11:00:27 - INFO - __main__ - Step: 50 Loss: 11.0638 LR: 0.000060
+12/10/2025 11:00:44 - INFO - __main__ - Step: 60 Loss: 11.0581 LR: 0.000060
+12/10/2025 11:01:03 - INFO - __main__ - Step: 70 Loss: 11.0504 LR: 0.000060
+12/10/2025 11:01:21 - INFO - __main__ - Step: 80 Loss: 11.0334 LR: 0.000060
+12/10/2025 11:01:39 - INFO - __main__ - Step: 90 Loss: 11.0099 LR: 0.000060
+12/10/2025 11:01:57 - INFO - __main__ - Step: 100 Loss: 10.9694 LR: 0.000060
+12/10/2025 11:02:15 - INFO - __main__ - Step: 110 Loss: 10.9228 LR: 0.000060
+12/10/2025 11:02:33 - INFO - __main__ - Step: 120 Loss: 10.8648 LR: 0.000060
+12/10/2025 11:02:51 - INFO - __main__ - Step: 130 Loss: 10.7970 LR: 0.000060
+12/10/2025 11:03:09 - INFO - __main__ - Step: 140 Loss: 10.7392 LR: 0.000060
+12/10/2025 11:03:25 - INFO - __main__ - Step: 150 Loss: 10.6722 LR: 0.000060
+12/10/2025 11:03:42 - INFO - __main__ - Step: 160 Loss: 10.6268 LR: 0.000060
+12/10/2025 11:04:00 - INFO - __main__ - Step: 170 Loss: 10.5703 LR: 0.000060
+12/10/2025 11:04:17 - INFO - __main__ - Step: 180 Loss: 10.5288 LR: 0.000060
+12/10/2025 11:04:33 - INFO - __main__ - Step: 190 Loss: 10.4711 LR: 0.000060
+12/10/2025 11:04:49 - INFO - __main__ - Step: 200 Loss: 10.4425 LR: 0.000060
+12/10/2025 11:05:05 - INFO - __main__ - Step: 210 Loss: 10.4134 LR: 0.000060
+12/10/2025 11:05:21 - INFO - __main__ - Step: 220 Loss: 10.3852 LR: 0.000060
+12/10/2025 11:05:37 - INFO - __main__ - Step: 230 Loss: 10.3618 LR: 0.000060
+12/10/2025 11:05:54 - INFO - __main__ - Step: 240 Loss: 10.3473 LR: 0.000060
+12/10/2025 11:06:09 - INFO - __main__ - Step: 250 Loss: 10.3280 LR: 0.000060
+12/10/2025 11:06:25 - INFO - __main__ - Step: 260 Loss: 10.3093 LR: 0.000060
+12/10/2025 11:06:41 - INFO - __main__ - Step: 270 Loss: 10.3049 LR: 0.000060
+12/10/2025 11:06:57 - INFO - __main__ - Step: 280 Loss: 10.3022 LR: 0.000060
+12/10/2025 11:07:13 - INFO - __main__ - Step: 290 Loss: 10.2769 LR: 0.000060
+12/10/2025 11:07:29 - INFO - __main__ - Step: 300 Loss: 10.2934 LR: 0.000060
+12/10/2025 11:07:46 - INFO - __main__ - Step: 310 Loss: 10.2800 LR: 0.000060
+12/10/2025 11:08:02 - INFO - __main__ - Step: 320 Loss: 10.2736 LR: 0.000060
+12/10/2025 11:08:20 - INFO - __main__ - Step: 330 Loss: 10.2749 LR: 0.000060
+12/10/2025 11:08:37 - INFO - __main__ - Step: 340 Loss: 10.2522 LR: 0.000060
+12/10/2025 11:08:56 - INFO - __main__ - Step: 350 Loss: 10.2730 LR: 0.000060
+12/10/2025 11:09:14 - INFO - __main__ - Step: 360 Loss: 10.2558 LR: 0.000060
+12/10/2025 11:09:32 - INFO - __main__ - Step: 370 Loss: 10.2199 LR: 0.000060
+12/10/2025 11:09:50 - INFO - __main__ - Step: 380 Loss: 10.2401 LR: 0.000060
+12/10/2025 11:10:06 - INFO - __main__ - Step: 390 Loss: 10.2107 LR: 0.000060
+12/10/2025 11:10:22 - INFO - __main__ - Step: 400 Loss: 10.2311 LR: 0.000060
+12/10/2025 11:10:39 - INFO - __main__ - Step: 410 Loss: 10.2198 LR: 0.000060
+12/10/2025 11:10:56 - INFO - __main__ - Step: 420 Loss: 10.2267 LR: 0.000060
+12/10/2025 11:11:13 - INFO - __main__ - Step: 430 Loss: 10.2086 LR: 0.000060
+12/10/2025 11:11:29 - INFO - __main__ - Step: 440 Loss: 10.2145 LR: 0.000060
+12/10/2025 11:11:46 - INFO - __main__ - Step: 450 Loss: 10.2026 LR: 0.000060
+12/10/2025 11:12:01 - INFO - __main__ - Step: 460 Loss: 10.1881 LR: 0.000060
+12/10/2025 11:12:18 - INFO - __main__ - Step: 470 Loss: 10.1810 LR: 0.000060
+12/10/2025 11:12:34 - INFO - __main__ - Step: 480 Loss: 10.1763 LR: 0.000060
+12/10/2025 11:12:50 - INFO - __main__ - Step: 490 Loss: 10.1732 LR: 0.000060
+12/10/2025 11:13:07 - INFO - __main__ - Step: 500 Loss: 10.1615 LR: 0.000060
+12/10/2025 11:13:07 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 11:13:56 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 11:13:56 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 11:13:56 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 11:13:56 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 11:13:56 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 11:13:56 - INFO - __main__ - Generating videos for validation...
+12/10/2025 11:13:56 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 17.24it/s]
+Traceback (most recent call last):
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1786, in
+ main(parse_args())
+ ~~~~^^^^^^^^^^^^^^
+ File "/mnt/Meissonic/train/train_mei_video.py", line 1624, in main
+ val_tokenizer = T5Tokenizer.from_pretrained(model_id)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2157, in __getattribute__
+ requires_backends(cls, cls._backends)
+ ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
+ File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2143, in requires_backends
+ raise ImportError("".join(failed))
+ImportError:
+T5Tokenizer requires the SentencePiece library but it was not found in your environment. Check out the instructions on the
+installation page of its repo: https://github.com/google/sentencepiece#installation and follow the ones
+that match your environment. Please note that you may need to restart your runtime after installation.
+
+[rank0]: Traceback (most recent call last):
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1786, in
+[rank0]: main(parse_args())
+[rank0]: ~~~~^^^^^^^^^^^^^^
+[rank0]: File "/mnt/Meissonic/train/train_mei_video.py", line 1624, in main
+[rank0]: val_tokenizer = T5Tokenizer.from_pretrained(model_id)
+[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2157, in __getattribute__
+[rank0]: requires_backends(cls, cls._backends)
+[rank0]: ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
+[rank0]: File "/home/ubuntu/miniconda3/envs/test/lib/python3.13/site-packages/transformers/utils/import_utils.py", line 2143, in requires_backends
+[rank0]: raise ImportError("".join(failed))
+[rank0]: ImportError:
+[rank0]: T5Tokenizer requires the SentencePiece library but it was not found in your environment. Check out the instructions on the
+[rank0]: installation page of its repo: https://github.com/google/sentencepiece#installation and follow the ones
+[rank0]: that match your environment. Please note that you may need to restart your runtime after installation.
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/requirements.txt b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5872439cda379024a8df7829cfc0a29905cc5d6a
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/requirements.txt
@@ -0,0 +1,123 @@
+typing-inspection==0.4.2
+dask==2025.11.0
+dill==0.4.0
+ffmpy==1.0.0
+xxhash==3.6.0
+partd==1.4.2
+brotli==1.2.0
+einops==0.8.1
+charset-normalizer==3.4.4
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+orjson==3.11.5
+pydantic_core==2.41.5
+groovy==0.1.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+MarkupSafe==3.0.3
+protobuf==6.33.2
+typer==0.20.0
+gradio==6.1.0
+sentry-sdk==2.47.0
+nvidia-cusparselt-cu12==0.7.1
+locket==1.0.0
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+pydub==0.25.1
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+torchvision==0.24.1
+fastapi==0.124.0
+cloudpickle==3.1.2
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+toolz==1.1.0
+ftfy==6.3.1
+platformdirs==4.5.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+wheel==0.45.1
+timm==1.0.22
+semantic-version==2.10.0
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+httpcore==1.0.9
+fsspec==2025.10.0
+multidict==6.7.0
+regex==2025.11.3
+gradio_client==2.0.1
+importlib_metadata==8.7.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+filelock==3.20.0
+torchmetrics==1.8.2
+numpy==2.3.5
+uvicorn==0.38.0
+packaging==25.0
+aiosignal==1.4.0
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+urllib3==2.6.1
+networkx==3.6.1
+setuptools==80.9.0
+sympy==1.14.0
+torch==2.9.1
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+safetensors==0.7.0
+gitdb==4.0.12
+safehttpx==0.1.7
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+pillow==12.0.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+nvidia-nvtx-cu12==12.8.90
+markdown-it-py==4.0.0
+six==1.17.0
+starlette==0.50.0
+audioop-lts==0.2.2
+accelerate==1.12.0
+diffusers==0.36.0
+annotated-doc==0.0.4
+zipp==3.23.0
+propcache==0.4.1
+mpmath==1.3.0
+nvidia-curand-cu12==10.3.9.90
+python-multipart==0.0.20
+nvidia-cufft-cu12==11.3.3.83
+pip==25.3
+aiofiles==24.1.0
+yarl==1.22.0
+nvidia-cudnn-cu12==9.10.2.21
+tomlkit==0.13.3
+pydantic==2.12.4
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..05bca784de2bdbff051bcdb701e8c7c53cb3a717
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.13.10",
+ "startedAt": "2025-12-10T10:58:33.251591Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/test/bin/python3.13",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12200011956224"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "gsh7dojjv7efi33ri5grdscpp81d6a5z"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-summary.json b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..7e29653e8870158d3fbebd62907d6fadcde28217
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/files/wandb-summary.json
@@ -0,0 +1 @@
+{"avg_masking_rate":0.6341594457626343,"_timestamp":1.7653651871397667e+09,"_step":500,"_wandb":{"runtime":963},"_runtime":963.417445979,"step_loss":10.161508560180664,"lr":5.9999999999999995e-05}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-core.log b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..535ffdcb05ce31f7579f901cf41a6b9d66c55e29
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-core.log
@@ -0,0 +1,14 @@
+{"time":"2025-12-10T10:58:33.330145802Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp566f72w7/port-1225720.txt","pid":1225720,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T10:58:33.330717487Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1225720}
+{"time":"2025-12-10T10:58:33.330730083Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1225720-1225978-1373833328/socket","Net":"unix"}}
+{"time":"2025-12-10T10:58:33.516447235Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T10:58:33.522585352Z","level":"INFO","msg":"handleInformInit: received","streamId":"im5q8jfr","id":"1(@)"}
+{"time":"2025-12-10T10:58:33.701451654Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"im5q8jfr","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.337228333Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.337300901Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-12-10T11:14:37.337290811Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.337379234Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1225720-1225978-1373833328/socket","Net":"unix"}}
+{"time":"2025-12-10T11:14:37.337410474Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.749424155Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.749441581Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-12-10T11:14:37.749450085Z","level":"INFO","msg":"server is closed"}
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-internal.log b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..6489bd693648981c5f33c8f3ca517cf12c0a3bf6
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-internal.log
@@ -0,0 +1,11 @@
+{"time":"2025-12-10T10:58:33.522737058Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T10:58:33.701175344Z","level":"INFO","msg":"stream: created new stream","id":"im5q8jfr"}
+{"time":"2025-12-10T10:58:33.701318417Z","level":"INFO","msg":"handler: started","stream_id":"im5q8jfr"}
+{"time":"2025-12-10T10:58:33.701444522Z","level":"INFO","msg":"stream: started","id":"im5q8jfr"}
+{"time":"2025-12-10T10:58:33.701462771Z","level":"INFO","msg":"sender: started","stream_id":"im5q8jfr"}
+{"time":"2025-12-10T10:58:33.701464614Z","level":"INFO","msg":"writer: started","stream_id":"im5q8jfr"}
+{"time":"2025-12-10T11:14:37.337303763Z","level":"INFO","msg":"stream: closing","id":"im5q8jfr"}
+{"time":"2025-12-10T11:14:37.579507859Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-12-10T11:14:37.746254099Z","level":"INFO","msg":"handler: closed","stream_id":"im5q8jfr"}
+{"time":"2025-12-10T11:14:37.74632182Z","level":"INFO","msg":"sender: closed","stream_id":"im5q8jfr"}
+{"time":"2025-12-10T11:14:37.746337064Z","level":"INFO","msg":"stream: closed","id":"im5q8jfr"}
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug.log b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8e1a09b3b4abe19a93105eef6d134f3ba191fbcf
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug.log
@@ -0,0 +1,24 @@
+2025-12-10 10:58:33,252 INFO MainThread:1225720 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 10:58:33,252 INFO MainThread:1225720 [wandb_setup.py:_flush():80] Configure stats pid to 1225720
+2025-12-10 10:58:33,252 INFO MainThread:1225720 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug.log
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_105833-im5q8jfr/logs/debug-internal.log
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_init.py:init():841] calling init triggers
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 10:58:33,253 INFO MainThread:1225720 [wandb_init.py:init():889] starting backend
+2025-12-10 10:58:33,516 INFO MainThread:1225720 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 10:58:33,521 INFO MainThread:1225720 [wandb_init.py:init():900] backend started and connected
+2025-12-10 10:58:33,523 INFO MainThread:1225720 [wandb_init.py:init():970] updated telemetry
+2025-12-10 10:58:33,527 INFO MainThread:1225720 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 10:58:33,919 INFO MainThread:1225720 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 10:58:34,008 INFO MainThread:1225720 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 10:58:34,008 INFO MainThread:1225720 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 10:58:34,008 INFO MainThread:1225720 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 10:58:34,008 INFO MainThread:1225720 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 10:58:34,010 INFO MainThread:1225720 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 10:58:34,011 INFO MainThread:1225720 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
+2025-12-10 11:14:37,337 INFO wandb-AsyncioManager-main:1225720 [service_client.py:_forward_responses():80] Reached EOF.
+2025-12-10 11:14:37,337 INFO wandb-AsyncioManager-main:1225720 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.
diff --git a/Meissonic/wandb/run-20251210_105833-im5q8jfr/run-im5q8jfr.wandb b/Meissonic/wandb/run-20251210_105833-im5q8jfr/run-im5q8jfr.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..42ca7803d926fd387492d033753dbad096cc8d68
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_105833-im5q8jfr/run-im5q8jfr.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b393857bb43e2b56cce1852678952a81bb1f035db15ae0d9adaaa790911193f5
+size 272716
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_56042e5c1e3ad351c124.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_56042e5c1e3ad351c124.png
new file mode 100644
index 0000000000000000000000000000000000000000..765049cc3b4cfc63f63db750cd28fc118528068c
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_56042e5c1e3ad351c124.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_716aacc5bc223fd0d079.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_716aacc5bc223fd0d079.png
new file mode 100644
index 0000000000000000000000000000000000000000..7d3ab8f7d36144d635bed78174c827c4afbaaba5
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1000_716aacc5bc223fd0d079.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_b05dd99c5903cf5f83ff.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_b05dd99c5903cf5f83ff.png
new file mode 100644
index 0000000000000000000000000000000000000000..87c6b04798ae294957f3fa0e5cae938c22d94fe6
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_b05dd99c5903cf5f83ff.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_e65dea2500a19c25f564.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_e65dea2500a19c25f564.png
new file mode 100644
index 0000000000000000000000000000000000000000..9ccd6d7be42a04df7bd83005a59f98b14d6c1659
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_1500_e65dea2500a19c25f564.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_175da7a7f9f39cf2ca22.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_175da7a7f9f39cf2ca22.png
new file mode 100644
index 0000000000000000000000000000000000000000..e7fe178ee86afd3497018541c3ac27baa8220c05
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_175da7a7f9f39cf2ca22.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_f225a4471ecef1dbfaeb.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_f225a4471ecef1dbfaeb.png
new file mode 100644
index 0000000000000000000000000000000000000000..760ada88e30cdf4dc4683a8e4fb7328a4d871a1d
Binary files /dev/null and b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_first_frame_500_f225a4471ecef1dbfaeb.png differ
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_654e84862d8c0a13f1b5.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_654e84862d8c0a13f1b5.png
new file mode 100644
index 0000000000000000000000000000000000000000..878e99d547a10280cfc35b864fda1c8b7542ffec
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_654e84862d8c0a13f1b5.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:654e84862d8c0a13f1b5c5321faf51f7e42bdea5337ed399b9d45addce85278a
+size 449974
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_cd153009051a7d605018.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_cd153009051a7d605018.png
new file mode 100644
index 0000000000000000000000000000000000000000..38f4e347989e21a17df32d0b41ad1253f05db150
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1000_cd153009051a7d605018.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd153009051a7d6050189d64dffa50ee778bbb8be7b2d706c53a9bfe104bc445
+size 408663
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_56d7d215080b273e9155.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_56d7d215080b273e9155.png
new file mode 100644
index 0000000000000000000000000000000000000000..4c7187945690a70279354741cc60575fe8e5d2ab
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_56d7d215080b273e9155.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56d7d215080b273e9155fabd655e8d067a165f6e2554fd135b5233869915f077
+size 452138
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_736c017ca88662cd1d11.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_736c017ca88662cd1d11.png
new file mode 100644
index 0000000000000000000000000000000000000000..86f2afb4ae6521a3ef13f10a289e3a29c9d7260f
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_1500_736c017ca88662cd1d11.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:736c017ca88662cd1d11e43ada8129c0fb46f23101f73c275e6bfae1f95d7b55
+size 452537
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_4f69c990d95a223b9d06.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_4f69c990d95a223b9d06.png
new file mode 100644
index 0000000000000000000000000000000000000000..8396f8c64b07e2f15cad630a54e1f9bac46f8758
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_4f69c990d95a223b9d06.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f69c990d95a223b9d06117c623b8a353a9415a910fddeaf43bddb5fd465c2b5
+size 330201
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_5fc1dbdaeeaef4847234.png b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_5fc1dbdaeeaef4847234.png
new file mode 100644
index 0000000000000000000000000000000000000000..6522c3ba8b5dbd7111ac918ba956841f177a9b48
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/media/images/generated_videos_grid_500_5fc1dbdaeeaef4847234.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc1dbdaeeaef4847234f332b9f24bad9d5540cd20f35cd09af9b6b8bd26e96a
+size 323804
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/output.log b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..be36da2ad8e60df2d1a2791ff4c093acb0f8f3fc
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/output.log
@@ -0,0 +1,252 @@
+12/10/2025 11:44:40 - INFO - __main__ - Using precomputed features - skipping text encoder and video tokenizer loading during training
+12/10/2025 11:44:40 - INFO - __main__ - Text encoder and video tokenizer will be loaded only during validation/inference
+12/10/2025 11:44:40 - INFO - __main__ - Loaded from metadata: codebook_size=64000, mask_token_id=64000
+12/10/2025 11:44:40 - INFO - __main__ - Minimal tokenizer created: mask_token_id=64000, codebook_size=64000
+12/10/2025 11:44:40 - INFO - __main__ - Getting compressed dimensions from precomputed features...
+12/10/2025 11:44:40 - INFO - __main__ - Got dimensions from metadata: F'=5, H'=16, W'=16
+12/10/2025 11:44:40 - INFO - __main__ - Got text_dim from metadata: 4096
+12/10/2025 11:44:40 - INFO - __main__ - Loading Wan config from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 11:44:40 - INFO - __main__ - Loaded Wan config: dim=1536, ffn_dim=8960, num_layers=30, num_heads=12
+12/10/2025 11:44:56 - INFO - __main__ - Loading Wan pretrained weights from: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 11:44:56 - INFO - __main__ - Loading weights from HuggingFace Hub: Wan-AI/Wan2.1-T2V-1.3B
+12/10/2025 11:44:58 - INFO - __main__ - ✓ Successfully loaded Wan pretrained weights into backbone (excluding text_embedding)
+12/10/2025 11:44:59 - INFO - __main__ - Parameter counts: backbone=1,418,996,800, other=2,112,033, total=1,421,108,833
+12/10/2025 11:44:59 - INFO - __main__ - Wan backbone lr = 0.000060 (base_lr * 0.2)
+12/10/2025 11:44:59 - INFO - __main__ - Other parts (token_embedding, logits_head) lr = 0.000300
+12/10/2025 11:44:59 - INFO - __main__ - Creating dataloaders and lr_scheduler
+12/10/2025 11:44:59 - INFO - __main__ - Using pre-extracted features from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features
+12/10/2025 11:44:59 - INFO - train.dataset_utils - Loaded metadata from /mnt/VideoGen/dataset/OpenVid1M/extracted_features/metadata.json
+12/10/2025 11:44:59 - INFO - train.dataset_utils - Total samples in metadata: 1024
+12/10/2025 11:44:59 - INFO - train.dataset_utils - PrecomputedFeatureDataset: 128 samples available
+12/10/2025 11:44:59 - INFO - train.dataset_utils - Index range: 0 to 127
+12/10/2025 11:44:59 - INFO - __main__ - Using precomputed features - DataLoader settings: prefetch_factor=1, pin_memory=True
+12/10/2025 11:44:59 - INFO - __main__ - Dataloader configuration:
+12/10/2025 11:44:59 - INFO - __main__ - - num_workers: 8 (0 = single-threaded, recommended: 4-8 for video)
+12/10/2025 11:44:59 - INFO - __main__ - - prefetch_factor: 2
+12/10/2025 11:44:59 - INFO - __main__ - - persistent_workers: True
+12/10/2025 11:44:59 - INFO - __main__ - - pin_memory: True
+12/10/2025 11:44:59 - INFO - __main__ - Preparing model, optimizer and dataloaders
+12/10/2025 11:45:04 - INFO - __main__ - Skipping text_encoder.to() - using precomputed features
+12/10/2025 11:45:04 - INFO - __main__ - Loading empty_embeds from: /mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy
+12/10/2025 11:45:04 - INFO - __main__ - Empty embeds info from metadata: shape=[1, 512, 4096]
+12/10/2025 11:45:04 - INFO - __main__ - Loaded empty_embeds: shape=torch.Size([1, 512, 4096]), dtype=torch.bfloat16
+12/10/2025 11:45:04 - INFO - __main__ - ***** Running training *****
+12/10/2025 11:45:04 - INFO - __main__ - Num training steps = 10000
+12/10/2025 11:45:04 - INFO - __main__ - Instantaneous batch size per device = 8
+12/10/2025 11:45:04 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 256
+12/10/2025 11:45:04 - INFO - __main__ - Gradient Accumulation steps = 4
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+[DEBUG] video_tokens: shape=torch.Size([8, 5, 16, 16]), dtype=torch.int32, device=cuda:0
+[DEBUG] encoder_hidden_states: shape=torch.Size([8, 512, 4096]), dtype=torch.bfloat16, device=cuda:0
+12/10/2025 11:45:26 - INFO - __main__ - Step: 10 Loss: 11.0753 LR: 0.000060
+12/10/2025 11:45:44 - INFO - __main__ - Step: 20 Loss: 11.0730 LR: 0.000060
+12/10/2025 11:46:02 - INFO - __main__ - Step: 30 Loss: 11.0695 LR: 0.000060
+12/10/2025 11:46:20 - INFO - __main__ - Step: 40 Loss: 11.0665 LR: 0.000060
+12/10/2025 11:46:37 - INFO - __main__ - Step: 50 Loss: 11.0638 LR: 0.000060
+12/10/2025 11:46:54 - INFO - __main__ - Step: 60 Loss: 11.0581 LR: 0.000060
+12/10/2025 11:47:11 - INFO - __main__ - Step: 70 Loss: 11.0504 LR: 0.000060
+12/10/2025 11:47:28 - INFO - __main__ - Step: 80 Loss: 11.0334 LR: 0.000060
+12/10/2025 11:47:46 - INFO - __main__ - Step: 90 Loss: 11.0099 LR: 0.000060
+12/10/2025 11:48:03 - INFO - __main__ - Step: 100 Loss: 10.9694 LR: 0.000060
+12/10/2025 11:48:19 - INFO - __main__ - Step: 110 Loss: 10.9227 LR: 0.000060
+12/10/2025 11:48:35 - INFO - __main__ - Step: 120 Loss: 10.8646 LR: 0.000060
+12/10/2025 11:48:51 - INFO - __main__ - Step: 130 Loss: 10.7967 LR: 0.000060
+12/10/2025 11:49:07 - INFO - __main__ - Step: 140 Loss: 10.7391 LR: 0.000060
+12/10/2025 11:49:23 - INFO - __main__ - Step: 150 Loss: 10.6720 LR: 0.000060
+12/10/2025 11:49:39 - INFO - __main__ - Step: 160 Loss: 10.6266 LR: 0.000060
+12/10/2025 11:49:55 - INFO - __main__ - Step: 170 Loss: 10.5701 LR: 0.000060
+12/10/2025 11:50:12 - INFO - __main__ - Step: 180 Loss: 10.5286 LR: 0.000060
+12/10/2025 11:50:28 - INFO - __main__ - Step: 190 Loss: 10.4709 LR: 0.000060
+12/10/2025 11:50:44 - INFO - __main__ - Step: 200 Loss: 10.4424 LR: 0.000060
+12/10/2025 11:51:00 - INFO - __main__ - Step: 210 Loss: 10.4137 LR: 0.000060
+12/10/2025 11:51:15 - INFO - __main__ - Step: 220 Loss: 10.3855 LR: 0.000060
+12/10/2025 11:51:30 - INFO - __main__ - Step: 230 Loss: 10.3617 LR: 0.000060
+12/10/2025 11:51:47 - INFO - __main__ - Step: 240 Loss: 10.3473 LR: 0.000060
+12/10/2025 11:52:03 - INFO - __main__ - Step: 250 Loss: 10.3278 LR: 0.000060
+12/10/2025 11:52:18 - INFO - __main__ - Step: 260 Loss: 10.3091 LR: 0.000060
+12/10/2025 11:52:34 - INFO - __main__ - Step: 270 Loss: 10.3043 LR: 0.000060
+12/10/2025 11:52:49 - INFO - __main__ - Step: 280 Loss: 10.3025 LR: 0.000060
+12/10/2025 11:53:05 - INFO - __main__ - Step: 290 Loss: 10.2772 LR: 0.000060
+12/10/2025 11:53:20 - INFO - __main__ - Step: 300 Loss: 10.2931 LR: 0.000060
+12/10/2025 11:53:35 - INFO - __main__ - Step: 310 Loss: 10.2803 LR: 0.000060
+12/10/2025 11:53:52 - INFO - __main__ - Step: 320 Loss: 10.2737 LR: 0.000060
+12/10/2025 11:54:08 - INFO - __main__ - Step: 330 Loss: 10.2750 LR: 0.000060
+12/10/2025 11:54:24 - INFO - __main__ - Step: 340 Loss: 10.2526 LR: 0.000060
+12/10/2025 11:54:41 - INFO - __main__ - Step: 350 Loss: 10.2730 LR: 0.000060
+12/10/2025 11:54:56 - INFO - __main__ - Step: 360 Loss: 10.2564 LR: 0.000060
+12/10/2025 11:55:11 - INFO - __main__ - Step: 370 Loss: 10.2203 LR: 0.000060
+12/10/2025 11:55:26 - INFO - __main__ - Step: 380 Loss: 10.2405 LR: 0.000060
+12/10/2025 11:55:41 - INFO - __main__ - Step: 390 Loss: 10.2115 LR: 0.000060
+12/10/2025 11:55:56 - INFO - __main__ - Step: 400 Loss: 10.2321 LR: 0.000060
+12/10/2025 11:56:12 - INFO - __main__ - Step: 410 Loss: 10.2205 LR: 0.000060
+12/10/2025 11:56:27 - INFO - __main__ - Step: 420 Loss: 10.2271 LR: 0.000060
+12/10/2025 11:56:42 - INFO - __main__ - Step: 430 Loss: 10.2097 LR: 0.000060
+12/10/2025 11:56:58 - INFO - __main__ - Step: 440 Loss: 10.2136 LR: 0.000060
+12/10/2025 11:57:14 - INFO - __main__ - Step: 450 Loss: 10.2033 LR: 0.000060
+12/10/2025 11:57:30 - INFO - __main__ - Step: 460 Loss: 10.1879 LR: 0.000060
+12/10/2025 11:57:46 - INFO - __main__ - Step: 470 Loss: 10.1792 LR: 0.000060
+12/10/2025 11:58:02 - INFO - __main__ - Step: 480 Loss: 10.1799 LR: 0.000060
+12/10/2025 11:58:18 - INFO - __main__ - Step: 490 Loss: 10.1722 LR: 0.000060
+12/10/2025 11:58:35 - INFO - __main__ - Step: 500 Loss: 10.1545 LR: 0.000060
+12/10/2025 11:58:35 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 11:59:26 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/optimizer.bin
+12/10/2025 11:59:26 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/scheduler.bin
+12/10/2025 11:59:26 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/sampler.bin
+12/10/2025 11:59:26 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500/random_states_0.pkl
+12/10/2025 11:59:26 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-500
+12/10/2025 11:59:26 - INFO - __main__ - Generating videos for validation...
+12/10/2025 11:59:26 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 94.25it/s]
+You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 3110.18it/s]
+12/10/2025 11:59:43 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 11:59:43 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.46it/s]
+12/10/2025 11:59:50 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 11:59:50 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+12/10/2025 12:00:08 - INFO - __main__ - Step: 510 Loss: 10.1603 LR: 0.000060
+12/10/2025 12:00:24 - INFO - __main__ - Step: 520 Loss: 10.1230 LR: 0.000060
+12/10/2025 12:00:40 - INFO - __main__ - Step: 530 Loss: 10.1212 LR: 0.000060
+12/10/2025 12:00:55 - INFO - __main__ - Step: 540 Loss: 10.0672 LR: 0.000060
+12/10/2025 12:01:11 - INFO - __main__ - Step: 550 Loss: 10.0958 LR: 0.000060
+12/10/2025 12:01:27 - INFO - __main__ - Step: 560 Loss: 10.0563 LR: 0.000060
+12/10/2025 12:01:43 - INFO - __main__ - Step: 570 Loss: 10.0529 LR: 0.000060
+12/10/2025 12:02:00 - INFO - __main__ - Step: 580 Loss: 10.0877 LR: 0.000060
+12/10/2025 12:02:16 - INFO - __main__ - Step: 590 Loss: 10.0004 LR: 0.000060
+12/10/2025 12:02:32 - INFO - __main__ - Step: 600 Loss: 9.8712 LR: 0.000060
+12/10/2025 12:02:48 - INFO - __main__ - Step: 610 Loss: 10.0444 LR: 0.000060
+12/10/2025 12:03:04 - INFO - __main__ - Step: 620 Loss: 9.8617 LR: 0.000060
+12/10/2025 12:03:20 - INFO - __main__ - Step: 630 Loss: 9.7717 LR: 0.000060
+12/10/2025 12:03:38 - INFO - __main__ - Step: 640 Loss: 9.8441 LR: 0.000060
+12/10/2025 12:03:56 - INFO - __main__ - Step: 650 Loss: 9.7777 LR: 0.000060
+12/10/2025 12:04:14 - INFO - __main__ - Step: 660 Loss: 10.0624 LR: 0.000060
+12/10/2025 12:04:32 - INFO - __main__ - Step: 670 Loss: 9.7147 LR: 0.000060
+12/10/2025 12:04:48 - INFO - __main__ - Step: 680 Loss: 9.8170 LR: 0.000060
+12/10/2025 12:05:05 - INFO - __main__ - Step: 690 Loss: 9.7774 LR: 0.000060
+12/10/2025 12:05:21 - INFO - __main__ - Step: 700 Loss: 9.4032 LR: 0.000060
+12/10/2025 12:05:37 - INFO - __main__ - Step: 710 Loss: 9.3151 LR: 0.000060
+12/10/2025 12:05:53 - INFO - __main__ - Step: 720 Loss: 9.0900 LR: 0.000060
+12/10/2025 12:06:08 - INFO - __main__ - Step: 730 Loss: 9.7551 LR: 0.000060
+12/10/2025 12:06:24 - INFO - __main__ - Step: 740 Loss: 9.2468 LR: 0.000060
+12/10/2025 12:06:39 - INFO - __main__ - Step: 750 Loss: 8.9362 LR: 0.000060
+12/10/2025 12:06:56 - INFO - __main__ - Step: 760 Loss: 9.3001 LR: 0.000060
+12/10/2025 12:07:14 - INFO - __main__ - Step: 770 Loss: 8.9890 LR: 0.000060
+12/10/2025 12:07:31 - INFO - __main__ - Step: 780 Loss: 9.1591 LR: 0.000060
+12/10/2025 12:07:49 - INFO - __main__ - Step: 790 Loss: 8.8347 LR: 0.000060
+12/10/2025 12:08:07 - INFO - __main__ - Step: 800 Loss: 8.5979 LR: 0.000060
+12/10/2025 12:08:25 - INFO - __main__ - Step: 810 Loss: 7.7741 LR: 0.000060
+12/10/2025 12:08:43 - INFO - __main__ - Step: 820 Loss: 8.4691 LR: 0.000060
+12/10/2025 12:09:01 - INFO - __main__ - Step: 830 Loss: 8.2816 LR: 0.000060
+12/10/2025 12:09:18 - INFO - __main__ - Step: 840 Loss: 8.7354 LR: 0.000060
+12/10/2025 12:09:36 - INFO - __main__ - Step: 850 Loss: 7.8491 LR: 0.000060
+12/10/2025 12:09:53 - INFO - __main__ - Step: 860 Loss: 9.0077 LR: 0.000060
+12/10/2025 12:10:11 - INFO - __main__ - Step: 870 Loss: 8.2152 LR: 0.000060
+12/10/2025 12:10:28 - INFO - __main__ - Step: 880 Loss: 8.5758 LR: 0.000060
+12/10/2025 12:10:44 - INFO - __main__ - Step: 890 Loss: 8.3967 LR: 0.000060
+12/10/2025 12:11:00 - INFO - __main__ - Step: 900 Loss: 7.8143 LR: 0.000060
+12/10/2025 12:11:16 - INFO - __main__ - Step: 910 Loss: 8.2686 LR: 0.000060
+12/10/2025 12:11:32 - INFO - __main__ - Step: 920 Loss: 7.6982 LR: 0.000060
+12/10/2025 12:11:47 - INFO - __main__ - Step: 930 Loss: 7.9535 LR: 0.000060
+12/10/2025 12:12:03 - INFO - __main__ - Step: 940 Loss: 7.0900 LR: 0.000060
+12/10/2025 12:12:19 - INFO - __main__ - Step: 950 Loss: 7.5838 LR: 0.000060
+12/10/2025 12:12:35 - INFO - __main__ - Step: 960 Loss: 7.6097 LR: 0.000060
+12/10/2025 12:12:50 - INFO - __main__ - Step: 970 Loss: 4.9570 LR: 0.000060
+12/10/2025 12:13:06 - INFO - __main__ - Step: 980 Loss: 7.8306 LR: 0.000060
+12/10/2025 12:13:24 - INFO - __main__ - Step: 990 Loss: 7.1266 LR: 0.000060
+12/10/2025 12:13:41 - INFO - __main__ - Step: 1000 Loss: 6.5655 LR: 0.000060
+12/10/2025 12:13:41 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000
+12/10/2025 12:14:30 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/optimizer.bin
+12/10/2025 12:14:30 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/scheduler.bin
+12/10/2025 12:14:30 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/sampler.bin
+12/10/2025 12:14:30 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000/random_states_0.pkl
+12/10/2025 12:14:30 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1000
+12/10/2025 12:14:30 - INFO - __main__ - Generating videos for validation...
+12/10/2025 12:14:30 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 97.14it/s]
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 6364.65it/s]
+12/10/2025 12:14:46 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 12:14:46 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.36it/s]
+12/10/2025 12:14:52 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 12:14:52 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+12/10/2025 12:15:08 - INFO - __main__ - Step: 1010 Loss: 6.2143 LR: 0.000060
+12/10/2025 12:15:25 - INFO - __main__ - Step: 1020 Loss: 6.1127 LR: 0.000060
+12/10/2025 12:15:43 - INFO - __main__ - Step: 1030 Loss: 4.2783 LR: 0.000060
+12/10/2025 12:16:00 - INFO - __main__ - Step: 1040 Loss: 6.1687 LR: 0.000060
+12/10/2025 12:16:18 - INFO - __main__ - Step: 1050 Loss: 7.1046 LR: 0.000060
+12/10/2025 12:16:35 - INFO - __main__ - Step: 1060 Loss: 6.0546 LR: 0.000060
+12/10/2025 12:16:50 - INFO - __main__ - Step: 1070 Loss: 6.2354 LR: 0.000060
+12/10/2025 12:17:06 - INFO - __main__ - Step: 1080 Loss: 4.4416 LR: 0.000060
+12/10/2025 12:17:22 - INFO - __main__ - Step: 1090 Loss: 5.3321 LR: 0.000060
+12/10/2025 12:17:39 - INFO - __main__ - Step: 1100 Loss: 5.4109 LR: 0.000060
+12/10/2025 12:17:56 - INFO - __main__ - Step: 1110 Loss: 5.9100 LR: 0.000060
+12/10/2025 12:18:11 - INFO - __main__ - Step: 1120 Loss: 5.4545 LR: 0.000060
+12/10/2025 12:18:28 - INFO - __main__ - Step: 1130 Loss: 4.8019 LR: 0.000060
+12/10/2025 12:18:43 - INFO - __main__ - Step: 1140 Loss: 4.0381 LR: 0.000060
+12/10/2025 12:18:59 - INFO - __main__ - Step: 1150 Loss: 5.6605 LR: 0.000060
+12/10/2025 12:19:15 - INFO - __main__ - Step: 1160 Loss: 4.4688 LR: 0.000060
+12/10/2025 12:19:30 - INFO - __main__ - Step: 1170 Loss: 4.5869 LR: 0.000060
+12/10/2025 12:19:46 - INFO - __main__ - Step: 1180 Loss: 4.6429 LR: 0.000060
+12/10/2025 12:20:01 - INFO - __main__ - Step: 1190 Loss: 6.6926 LR: 0.000060
+12/10/2025 12:20:16 - INFO - __main__ - Step: 1200 Loss: 2.8797 LR: 0.000060
+12/10/2025 12:20:32 - INFO - __main__ - Step: 1210 Loss: 6.2622 LR: 0.000060
+12/10/2025 12:20:47 - INFO - __main__ - Step: 1220 Loss: 3.0145 LR: 0.000060
+12/10/2025 12:21:03 - INFO - __main__ - Step: 1230 Loss: 3.7550 LR: 0.000060
+12/10/2025 12:21:18 - INFO - __main__ - Step: 1240 Loss: 3.6606 LR: 0.000060
+12/10/2025 12:21:34 - INFO - __main__ - Step: 1250 Loss: 2.9642 LR: 0.000060
+12/10/2025 12:21:50 - INFO - __main__ - Step: 1260 Loss: 3.9318 LR: 0.000060
+12/10/2025 12:22:06 - INFO - __main__ - Step: 1270 Loss: 2.6559 LR: 0.000060
+12/10/2025 12:22:23 - INFO - __main__ - Step: 1280 Loss: 5.4312 LR: 0.000060
+12/10/2025 12:22:39 - INFO - __main__ - Step: 1290 Loss: 3.5359 LR: 0.000060
+12/10/2025 12:22:56 - INFO - __main__ - Step: 1300 Loss: 4.5748 LR: 0.000060
+12/10/2025 12:23:13 - INFO - __main__ - Step: 1310 Loss: 4.7353 LR: 0.000060
+12/10/2025 12:23:29 - INFO - __main__ - Step: 1320 Loss: 2.0193 LR: 0.000060
+12/10/2025 12:23:45 - INFO - __main__ - Step: 1330 Loss: 7.2324 LR: 0.000060
+12/10/2025 12:24:01 - INFO - __main__ - Step: 1340 Loss: 3.9969 LR: 0.000060
+12/10/2025 12:24:17 - INFO - __main__ - Step: 1350 Loss: 4.6693 LR: 0.000060
+12/10/2025 12:24:33 - INFO - __main__ - Step: 1360 Loss: 3.2349 LR: 0.000060
+12/10/2025 12:24:49 - INFO - __main__ - Step: 1370 Loss: 5.9346 LR: 0.000060
+12/10/2025 12:25:05 - INFO - __main__ - Step: 1380 Loss: 3.4152 LR: 0.000060
+12/10/2025 12:25:22 - INFO - __main__ - Step: 1390 Loss: 3.6506 LR: 0.000060
+12/10/2025 12:25:38 - INFO - __main__ - Step: 1400 Loss: 4.7416 LR: 0.000060
+12/10/2025 12:25:53 - INFO - __main__ - Step: 1410 Loss: 1.4305 LR: 0.000060
+12/10/2025 12:26:09 - INFO - __main__ - Step: 1420 Loss: 6.6906 LR: 0.000060
+12/10/2025 12:26:26 - INFO - __main__ - Step: 1430 Loss: 5.7471 LR: 0.000060
+12/10/2025 12:26:42 - INFO - __main__ - Step: 1440 Loss: 3.6354 LR: 0.000060
+12/10/2025 12:26:58 - INFO - __main__ - Step: 1450 Loss: 3.8425 LR: 0.000060
+12/10/2025 12:27:13 - INFO - __main__ - Step: 1460 Loss: 2.7156 LR: 0.000060
+12/10/2025 12:27:29 - INFO - __main__ - Step: 1470 Loss: 2.9668 LR: 0.000060
+12/10/2025 12:27:44 - INFO - __main__ - Step: 1480 Loss: 4.7851 LR: 0.000060
+12/10/2025 12:27:59 - INFO - __main__ - Step: 1490 Loss: 4.0824 LR: 0.000060
+12/10/2025 12:28:15 - INFO - __main__ - Step: 1500 Loss: 7.5225 LR: 0.000060
+12/10/2025 12:28:15 - INFO - accelerate.accelerator - Saving current state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500
+12/10/2025 12:28:22 - INFO - accelerate.checkpointing - Optimizer state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500/optimizer.bin
+12/10/2025 12:28:22 - INFO - accelerate.checkpointing - Scheduler state saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500/scheduler.bin
+12/10/2025 12:28:22 - INFO - accelerate.checkpointing - Sampler state for dataloader 0 saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500/sampler.bin
+12/10/2025 12:28:22 - INFO - accelerate.checkpointing - Random states saved in output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500/random_states_0.pkl
+12/10/2025 12:28:22 - INFO - __main__ - Saved state to output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio/checkpoint-1500
+12/10/2025 12:28:22 - INFO - __main__ - Generating videos for validation...
+12/10/2025 12:28:22 - INFO - __main__ - Loading text encoder and video tokenizer for validation...
+Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 97.64it/s]
+Fetching 7 files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 3384.45it/s]
+12/10/2025 12:28:38 - INFO - __main__ - Text encoder and video tokenizer loaded for validation
+12/10/2025 12:28:38 - INFO - __main__ - Generating videos for validation...
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:05<00:00, 8.16it/s]
+12/10/2025 12:28:45 - INFO - __main__ - Validation videos saved to ./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio
+12/10/2025 12:28:45 - INFO - __main__ - Cleaned up validation models and freed GPU memory
+12/10/2025 12:29:01 - INFO - __main__ - Step: 1510 Loss: 3.9462 LR: 0.000060
+12/10/2025 12:29:16 - INFO - __main__ - Step: 1520 Loss: 3.0234 LR: 0.000060
+12/10/2025 12:29:32 - INFO - __main__ - Step: 1530 Loss: 3.3499 LR: 0.000060
+12/10/2025 12:29:48 - INFO - __main__ - Step: 1540 Loss: 6.9508 LR: 0.000060
+12/10/2025 12:30:05 - INFO - __main__ - Step: 1550 Loss: 3.1735 LR: 0.000060
+12/10/2025 12:30:22 - INFO - __main__ - Step: 1560 Loss: 3.1230 LR: 0.000060
+12/10/2025 12:30:37 - INFO - __main__ - Step: 1570 Loss: 2.6245 LR: 0.000060
+12/10/2025 12:30:53 - INFO - __main__ - Step: 1580 Loss: 4.4095 LR: 0.000060
+12/10/2025 12:31:08 - INFO - __main__ - Step: 1590 Loss: 3.9888 LR: 0.000060
+12/10/2025 12:31:23 - INFO - __main__ - Step: 1600 Loss: 2.9593 LR: 0.000060
+12/10/2025 12:31:38 - INFO - __main__ - Step: 1610 Loss: 4.4038 LR: 0.000060
+12/10/2025 12:31:54 - INFO - __main__ - Step: 1620 Loss: 4.3135 LR: 0.000060
+12/10/2025 12:32:11 - INFO - __main__ - Step: 1630 Loss: 3.0804 LR: 0.000060
+12/10/2025 12:32:27 - INFO - __main__ - Step: 1640 Loss: 1.9102 LR: 0.000060
+12/10/2025 12:32:44 - INFO - __main__ - Step: 1650 Loss: 2.6062 LR: 0.000060
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/requirements.txt b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bf356027904d253cf8d99c729116710476866e7d
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/requirements.txt
@@ -0,0 +1,124 @@
+typing-inspection==0.4.2
+dask==2025.11.0
+dill==0.4.0
+ffmpy==1.0.0
+xxhash==3.6.0
+partd==1.4.2
+brotli==1.2.0
+einops==0.8.1
+charset-normalizer==3.4.4
+tokenizers==0.22.1
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+pyarrow==22.0.0
+annotated-types==0.7.0
+GitPython==3.1.45
+rich==14.2.0
+nvidia-cufile-cu12==1.13.1.3
+nvidia-nvshmem-cu12==3.3.20
+orjson==3.11.5
+pydantic_core==2.41.5
+groovy==0.1.2
+psutil==7.1.3
+peft==0.18.0
+typing_extensions==4.15.0
+wcwidth==0.2.14
+MarkupSafe==3.0.3
+protobuf==6.33.2
+typer==0.20.0
+gradio==6.1.0
+sentry-sdk==2.47.0
+nvidia-cusparselt-cu12==0.7.1
+locket==1.0.0
+PyYAML==6.0.3
+nvidia-nvjitlink-cu12==12.8.93
+datasets==4.4.1
+pytorch-lightning==2.6.0
+frozenlist==1.8.0
+pydub==0.25.1
+huggingface-hub==0.36.0
+Pygments==2.19.2
+aiohttp==3.13.2
+torchvision==0.24.1
+fastapi==0.124.0
+cloudpickle==3.1.2
+wandb==0.23.1
+tqdm==4.67.1
+httpx==0.28.1
+open_clip_torch==3.2.0
+flash_attn==2.8.3
+mdurl==0.1.2
+pandas==2.3.3
+toolz==1.1.0
+ftfy==6.3.1
+platformdirs==4.5.1
+transformers==4.57.3
+requests==2.32.5
+pytz==2025.2
+Jinja2==3.1.6
+click==8.3.1
+attrs==25.4.0
+hf-xet==1.2.0
+shellingham==1.5.4
+nvidia-nccl-cu12==2.27.5
+nvidia-cuda-runtime-cu12==12.8.90
+bitsandbytes==0.48.2
+lightning-utilities==0.15.2
+wheel==0.45.1
+timm==1.0.22
+semantic-version==2.10.0
+triton==3.5.1
+nvidia-cublas-cu12==12.8.4.1
+httpcore==1.0.9
+fsspec==2025.10.0
+multidict==6.7.0
+regex==2025.11.3
+gradio_client==2.0.1
+importlib_metadata==8.7.0
+anyio==4.12.0
+nvidia-cusolver-cu12==11.7.3.90
+filelock==3.20.0
+torchmetrics==1.8.2
+numpy==2.3.5
+uvicorn==0.38.0
+packaging==25.0
+aiosignal==1.4.0
+nvidia-cuda-nvrtc-cu12==12.8.93
+tzdata==2025.2
+urllib3==2.6.1
+networkx==3.6.1
+setuptools==80.9.0
+sympy==1.14.0
+torch==2.9.1
+nvidia-cuda-cupti-cu12==12.8.90
+smmap==5.0.2
+safetensors==0.7.0
+gitdb==4.0.12
+safehttpx==0.1.7
+nvidia-cusparse-cu12==12.5.8.93
+multiprocess==0.70.18
+pillow==12.0.0
+h11==0.16.0
+certifi==2025.11.12
+idna==3.11
+nvidia-nvtx-cu12==12.8.90
+markdown-it-py==4.0.0
+six==1.17.0
+starlette==0.50.0
+audioop-lts==0.2.2
+accelerate==1.12.0
+diffusers==0.36.0
+annotated-doc==0.0.4
+zipp==3.23.0
+propcache==0.4.1
+mpmath==1.3.0
+sentencepiece==0.2.1
+nvidia-curand-cu12==10.3.9.90
+python-multipart==0.0.20
+nvidia-cufft-cu12==11.3.3.83
+pip==25.3
+aiofiles==24.1.0
+yarl==1.22.0
+nvidia-cudnn-cu12==9.10.2.21
+tomlkit==0.13.3
+pydantic==2.12.4
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/wandb-metadata.json b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..35053deba575cc499331b8e547a9c6bde5ed5bde
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/files/wandb-metadata.json
@@ -0,0 +1,156 @@
+{
+ "os": "Linux-6.8.0-1027-aws-x86_64-with-glibc2.35",
+ "python": "CPython 3.13.10",
+ "startedAt": "2025-12-10T11:44:39.581540Z",
+ "args": [
+ "--use_precomputed_features",
+ "--features_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/extracted_features",
+ "--text_encoder_architecture",
+ "umt5-xxl",
+ "--wan_pretrained_path",
+ "Wan-AI/Wan2.1-T2V-1.3B",
+ "--training_from_scratch",
+ "True",
+ "--pretrained_model_name_or_path",
+ "dummy",
+ "--wan_backbone_lr_ratio",
+ "0.2",
+ "--num_frames",
+ "17",
+ "--video_height",
+ "128",
+ "--video_width",
+ "128",
+ "--dataloader_num_workers",
+ "8",
+ "--video_tokenizer_model_id",
+ "Cosmos-0.1-Tokenizer-DV4x8x8",
+ "--instance_dataset",
+ "OpenVid1MDataset",
+ "--instance_data_dir",
+ "/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv",
+ "--train_batch_size",
+ "8",
+ "--gradient_accumulation_steps",
+ "4",
+ "--learning_rate",
+ "3e-4",
+ "--max_train_steps",
+ "10000",
+ "--checkpointing_steps",
+ "500",
+ "--validation_steps",
+ "500",
+ "--logging_steps",
+ "10",
+ "--validation_prompts",
+ "a cat playing",
+ "a girl walking",
+ "--output_dir",
+ "./output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio",
+ "--mixed_precision",
+ "bf16",
+ "--lr_scheduler",
+ "constant",
+ "--lr_warmup_steps",
+ "0",
+ "--use_8bit_adam",
+ "--gradient_checkpointing",
+ "--min_masking_rate",
+ "0.0",
+ "--cond_dropout_prob",
+ "0.1",
+ "--split_vae_encode",
+ "1",
+ "--allow_tf32",
+ "--seed",
+ "42",
+ "--report_to",
+ "wandb"
+ ],
+ "program": "/mnt/Meissonic/train/train_mei_video.py",
+ "codePath": "train/train_mei_video.py",
+ "codePathLocal": "train/train_mei_video.py",
+ "git": {
+ "remote": "https://github.com/viiika/Meissonic.git",
+ "commit": "6819d374ef1b86bdedad373aab1121a89687e5cf"
+ },
+ "email": "jinbin5bai@gmail.com",
+ "root": "/mnt/Meissonic",
+ "host": "ip-172-31-91-136",
+ "executable": "/home/ubuntu/miniconda3/envs/test/bin/python3.13",
+ "cpu_count": 48,
+ "cpu_count_logical": 96,
+ "gpu": "NVIDIA A100-SXM4-40GB",
+ "gpu_count": 8,
+ "disk": {
+ "/": {
+ "total": "16650112278528",
+ "used": "12200016330752"
+ }
+ },
+ "memory": {
+ "total": "1204521451520"
+ },
+ "gpu_nvidia": [
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-54a50f05-7a41-8b8e-59c5-e1774ec42215"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-71102f28-cd17-57e7-6181-120bf743d23d"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-57dfac44-bb50-f9b6-1534-27fbe79dfd87"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-f55652c0-bdaf-e7bb-a876-8fce14c3f879"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-303ab142-3206-9a14-c758-58ab97d7510e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-1adf5c34-24d0-c5e2-b33b-783100bbd6c3"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-5b4a0e50-96a5-74bd-f595-14de5614cc6e"
+ },
+ {
+ "name": "NVIDIA A100-SXM4-40GB",
+ "memoryTotal": "42949672960",
+ "cudaCores": 6912,
+ "architecture": "Ampere",
+ "uuid": "GPU-efb2d1fc-1eed-653d-ed51-5273085154ba"
+ }
+ ],
+ "cudaVersion": "12.8",
+ "writerId": "e7k4synno5s6446cv7tglm0fogbbud44"
+}
\ No newline at end of file
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-core.log b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-core.log
new file mode 100644
index 0000000000000000000000000000000000000000..1a9cdf779b238f7720ec7dac5b49a19502c81560
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-core.log
@@ -0,0 +1,7 @@
+{"time":"2025-12-10T11:44:39.658001682Z","level":"INFO","msg":"main: starting server","port-filename":"/opt/dlami/nvme/tmp_user/tmp_j5j81dh/port-1312594.txt","pid":1312594,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-10T11:44:39.658502262Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1312594}
+{"time":"2025-12-10T11:44:39.658508217Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/opt/dlami/nvme/tmp_user/wandb-1312594-1312849-1745419067/socket","Net":"unix"}}
+{"time":"2025-12-10T11:44:39.84508562Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-10T11:44:39.851289376Z","level":"INFO","msg":"handleInformInit: received","streamId":"mrtah7xe","id":"1(@)"}
+{"time":"2025-12-10T11:44:40.016807865Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"mrtah7xe","id":"1(@)"}
+{"time":"2025-12-10T12:33:00.214923867Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-internal.log b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..d8dbd8f1d87e2a1194f2fe70f37102cb3ea4a565
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-internal.log
@@ -0,0 +1,6 @@
+{"time":"2025-12-10T11:44:39.85137573Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-10T11:44:40.016600149Z","level":"INFO","msg":"stream: created new stream","id":"mrtah7xe"}
+{"time":"2025-12-10T11:44:40.016677758Z","level":"INFO","msg":"handler: started","stream_id":"mrtah7xe"}
+{"time":"2025-12-10T11:44:40.016799939Z","level":"INFO","msg":"stream: started","id":"mrtah7xe"}
+{"time":"2025-12-10T11:44:40.016877013Z","level":"INFO","msg":"writer: started","stream_id":"mrtah7xe"}
+{"time":"2025-12-10T11:44:40.016880098Z","level":"INFO","msg":"sender: started","stream_id":"mrtah7xe"}
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug.log b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..2c28c8a071776362325b31a769670028f231f277
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug.log
@@ -0,0 +1,22 @@
+2025-12-10 11:44:39,582 INFO MainThread:1312594 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-10 11:44:39,582 INFO MainThread:1312594 [wandb_setup.py:_flush():80] Configure stats pid to 1312594
+2025-12-10 11:44:39,582 INFO MainThread:1312594 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-12-10 11:44:39,582 INFO MainThread:1312594 [wandb_setup.py:_flush():80] Loading settings from /mnt/Meissonic/wandb/settings
+2025-12-10 11:44:39,582 INFO MainThread:1312594 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-10 11:44:39,583 INFO MainThread:1312594 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /mnt/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug.log
+2025-12-10 11:44:39,583 INFO MainThread:1312594 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /mnt/Meissonic/wandb/run-20251210_114439-mrtah7xe/logs/debug-internal.log
+2025-12-10 11:44:39,583 INFO MainThread:1312594 [wandb_init.py:init():841] calling init triggers
+2025-12-10 11:44:39,583 INFO MainThread:1312594 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-10 11:44:39,583 INFO MainThread:1312594 [wandb_init.py:init():889] starting backend
+2025-12-10 11:44:39,845 INFO MainThread:1312594 [wandb_init.py:init():892] sending inform_init request
+2025-12-10 11:44:39,849 INFO MainThread:1312594 [wandb_init.py:init():900] backend started and connected
+2025-12-10 11:44:39,851 INFO MainThread:1312594 [wandb_init.py:init():970] updated telemetry
+2025-12-10 11:44:39,855 INFO MainThread:1312594 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-10 11:44:40,285 INFO MainThread:1312594 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-10 11:44:40,377 INFO MainThread:1312594 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-10 11:44:40,377 INFO MainThread:1312594 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-10 11:44:40,377 INFO MainThread:1312594 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-10 11:44:40,377 INFO MainThread:1312594 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-10 11:44:40,380 INFO MainThread:1312594 [wandb_init.py:init():1081] run started, returning control to user process
+2025-12-10 11:44:40,380 INFO MainThread:1312594 [wandb_run.py:_config_callback():1396] config_cb None None {'text_encoder_architecture': 'umt5-xxl', 'instance_dataset': 'OpenVid1MDataset', 'training_from_scratch': True, 'pretrained_model_name_or_path': 'dummy', 'revision': None, 'variant': None, 'instance_data_dataset': None, 'instance_data_dir': '/mnt/VideoGen/dataset/OpenVid1M/video_reorg/OpenVid1M_reorganized.csv', 'instance_data_image': None, 'use_8bit_adam': True, 'dataloader_num_workers': 8, 'dataloader_prefetch_factor': 2, 'allow_tf32': True, 'use_ema': False, 'ema_decay': 0.9999, 'ema_update_after_step': 0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'output_dir': './output_128x128_17f_2*4bs_4*8*8vqvae_0_2_ratio', 'seed': 42, 'logging_dir': 'logs', 'max_train_steps': 10000, 'checkpointing_steps': 500, 'logging_steps': 10, 'checkpoints_total_limit': None, 'resume_from_checkpoint': None, 'train_batch_size': 8, 'gradient_accumulation_steps': 4, 'learning_rate': 0.0003, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'validation_steps': 500, 'mixed_precision': 'bf16', 'report_to': 'wandb', 'validation_prompts': ['a cat playing', 'a girl walking'], 'resolution': 512, 'split_vae_encode': 1, 'min_masking_rate': 0.0, 'cond_dropout_prob': 0.1, 'max_grad_norm': 50.0, 'use_lora': False, 'text_encoder_use_lora': False, 'lora_r': 16, 'lora_alpha': 32, 'lora_target_modules': ['to_q', 'to_k', 'to_v'], 'text_encoder_lora_r': 16, 'text_encoder_lora_alpha': 32, 'text_encoder_lora_target_modules': ['to_q', 'to_k', 'to_v'], 'train_text_encoder': False, 'image_key': None, 'prompt_key': None, 'gradient_checkpointing': True, 'prompt_prefix': None, 'num_frames': 17, 'video_height': 128, 'video_width': 128, 'video_tokenizer_model_id': 'Cosmos-0.1-Tokenizer-DV4x8x8', 'wan_pretrained_path': 'Wan-AI/Wan2.1-T2V-1.3B', 'freeze_wan_backbone': False, 'wan_backbone_lr_ratio': 0.2, 'use_precomputed_features': True, 'features_dir': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features', 'empty_embeds_path': '/mnt/VideoGen/dataset/OpenVid1M/extracted_features/empty_embeds.npy'}
diff --git a/Meissonic/wandb/run-20251210_114439-mrtah7xe/run-mrtah7xe.wandb b/Meissonic/wandb/run-20251210_114439-mrtah7xe/run-mrtah7xe.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..23b1a1e532997f949d7db7da4dc6651dd4315210
--- /dev/null
+++ b/Meissonic/wandb/run-20251210_114439-mrtah7xe/run-mrtah7xe.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df857026859ba42c655fcb92a570c4f9ef3fc41929d3a3a1eeaf86de7eb8256f
+size 851968
+
+
+
+
+
+
+
+