Final Samples

MMaDA/inference/demo/space_demo.yaml

@@ -0,0 +1,354 @@
+wandb:
+  entity: null
+#  run_id: askkz9i2
+  resume: 'auto'
+
+experiment:
+    project: "omada-instruction-tuning_0204"
+    name: "omada-instruction-tuning_0204"
+    output_dir: "/dataset/omada/ckpt/it-0204"
+    max_train_examples_t2i: 40000000
+    max_train_examples_mmu: 40000000
+    save_every: 500
+    eval_every: 99999999999999
+    generate_every: 1000000000
+    log_every: 1
+    log_grad_norm_every: 100
+    resume_from_checkpoint: "latest"
+
+model:
+    vq_model_image:
+        type: "magvitv2"
+        vq_model_name: "/dataset/omada/ckpt/showlab/magvitv2"
+    ### Omada ######################[#########################################
+    vq_model_audio:
+        type: "emova"
+        vq_model_name: "/dataset/omada/ckpt/Emova-ollm/emova_speech_tokenizer_hf"
+    omada:
+        tokenizer_path: "/dataset/omada/ckpt/it-0204/checkpoint-0207/unwrapped_model"
+        local_files_only: true
+        # pretrained_model_path: "Gen-Verse/MMaDA-8B-MixCoT"
+        pretrained_model_path: "/dataset/omada/ckpt/it-0204/checkpoint-0207/unwrapped_model"
+        w_clip_vit: False
+        new_vocab_size: 138752
+        llm_vocab_size: 126464
+        codebook_size: 8192
+        num_vq_tokens: 1024
+        num_new_special_tokens: 0 # v2s, s2s, i2i
+        tie_word_embeddings: False
+    #########################################################################
+
+    gradient_checkpointing: True
+
+dataset:
+    gen_type: "pass"
+    und_type: "pass"
+    combined_loader_mode: "max_size_cycle"
+    params:
+        train_t2i_shards_path_or_url: "/data_storage/shared/datasets/imagenet-1k/data/train"
+        train_mmu_shards_path_or_url: [ "/data_storage/shared/datasets/SA-1B/sa_{000000..000999}.tar",
+                                        "/data_storage/shared/datasets/cc12m/raw/raw/{0000..0999}.tar",
+                                      "/data_storage/shared/datasets/laion-aesthetics-12m/{00000..00999}.tar" 
+        ]
+        train_lm_shards_path_or_url: "/data_storage/shared/datasets/falcon-refinedweb/data/data/*.parquet"
+        add_caption_prompt: True
+        external_caption_path: "/data_storage/shared/datasets/SAM-LLaVA-Captions10M"
+        external_journeydb_caption_path: "/data_storage/shared/datasets/journeydb_anno/train_journeydb_anno.json"
+        external_laion12m_caption_path: "/data_storage/shared/datasets/laion-aesthetic-12m-captions"
+        external_cc12m_caption_path: "/data_storage/shared/datasets/cc12m/captions"
+        validation_prompts_file: "validation_prompts/imagenet_prompts.txt"
+        mmu_image_root: "/data_storage/ty/MMaDA/mmu_validation"
+        ### Omada ###############################################################
+        video_root: "/home/work/AIDAS/data/video/openvid1m/video/video"
+        video_speech_dataset:
+          sample_mode: "exclusive"
+          sample_method: "uniform_sequential"
+          v2t_sample_method: "uniform_sequential"
+          use_precomputed_tokens: true
+          precomputed_tokens_root: "/home/work/AIDAS/cache/openvid_speech_tokens"
+          index_cache_path: "/home/work/AIDAS/cache/video_speech_index.pt"
+          max_video_seconds: 10
+          llavavid_max_video_seconds: 10
+          llavavid_path: "/dataset/omada/datasets/video/LLaVA-Video-178K"
+          llavavid_local_files_only: true
+          llavavid_skip_configs:
+            - "llava_hound"
+            - "0_30_s_activitynetqa"
+            - "30_60_s_activitynetqa"
+            - "1_2_m_activitynetqa"
+            - "2_3_m_activitynetqa"
+            - "0_30_s_activitynet"
+            - "30_60_s_activitynet"
+            - "1_2_m_activitynet"
+            - "2_3_m_activitynet"
+          llavavid_skip_video_patterns:
+            - "activitynet"
+          use_llavavid: false
+          llavavid_max_samples: 500000
+          llavavid_sample_seed: 42
+          sharegptvideo_sft_path:
+            - "/dataset/omada/datasets/ShareGPTVideo/video_instruction/train/qa/chatgpt_qa_240k_sft_frames.jsonl"
+            # - "/dataset/omada/datasets/video/vlmeval_sft_train_20f.jsonl"
+            # - "/dataset/omada/datasets/video/vlmeval_sft_train_20f_no_videomme.jsonl"
+            # - "/dataset/omada/datasets/video/vlmeval_sft_train_20f_temp_act.jsonl"
+            - "/dataset/omada/datasets/video/vlmeval_sft_train_20f_mv_mme_corr.jsonl"
+          sharegptvideo_num_frames: 5
+          sharegptvideo_sample_method: "uniform_sequential"
+          sharegptvideo_strip_video_token: true
+          sharegptvideo_require_video: true
+        # video_dataset_name: "openvid1m"
+        hqedit_split: "train"
+        t2i_dataset: "prompt_image_jsonl+basic_edit_jsonl+dpg_jsonl"
+        # t2i_dataset: "basic_edit_jsonl+dpg_jsonl"
+        t2i_split: "train"
+        t2i_dataset_name: "jackyhate/text-to-image-2M"
+        flux_reason_dataset_name: "LucasFang/FLUX-Reason-6M"
+        flux_reason_score_threshold: 8.0
+        flux_reason_local_files_only: true
+        pickapic_dataset_name: "Min-Jaewon/pickapic-v2"
+        ultraedit_dataset_name: "BleachNick/UltraEdit_500k"
+        ultraedit_local_files_only: true
+        journeydb_jsonl_path: "/home/work/AIDAS/data/JourneyDB/data/train/train_anno_realease_repath.jsonl"
+        journeydb_image_root: "/home/work/AIDAS/data/JourneyDB/data/train"
+        journeydb_local_files_only: true
+        prompt_image_jsonl:
+          jsonl_path: "/dataset/omada/datasets/t2i/prompt_image_geneval_pass.jsonl"
+          prompt_keys: ["prompt", "query"]
+          image_keys: ["image_path", "image"]
+          skip_missing: true
+          cache_path: "dataset/omada/datasets/t2i/prompt_image_geneval_pass_0114.cache.jsonl"
+          max_samples: null
+          seed: 42
+        dpg_jsonl:
+          jsonl_path: "/dataset/omada/datasets/t2i/combined_dpg.jsonl"
+          prompt_keys: ["prompt", "query"]
+          image_keys: ["image_path", "image"]
+          skip_missing: true
+          cache_path: "/dataset/omada/datasets/t2i/combined_dpg.cache.jsonl"
+          max_samples: null
+          seed: 42
+        i2i_prompt_image_jsonl:
+          jsonl_path:
+            - "/dataset/omada/datasets/i2i/basic_edit_all_pair_pass.jsonl"
+            - "/dataset/omada/datasets/ImgEdit/Singleturn/ImgEdit_pairs_from_parquet_300k.jsonl"
+          prompt_keys: ["prompt"]
+          image_keys: ["image_path"]
+          skip_missing: true
+          cache_path: "/dataset/omada/datasets/i2i/basic_edit_all_pair_pass.cache_0114.jsonl"
+          max_samples: null
+          seed: 42
+        t2i_local_files_only: true
+        openimage_i2i:
+          sft_jsonl: "/home/work/AIDAS/data/openimage_source_images/sft_with_local_source_image_path.jsonl"
+          pref_jsonl: "/home/work/AIDAS/data/openimage_source_images/pref_with_local_source_image_path.jsonl"
+          multi_turn_jsonl: "/home/work/AIDAS/data/openimage_source_images/multi-turn_with_local_source_image_path.jsonl"
+          image_root: "/home/work/AIDAS/data/nano_edited_images"
+          prefer_summarized_text: true
+          pref_positive_only: true
+          skip_missing: true
+          max_samples_per_source: null
+          max_total_samples: null
+          seed: 42
+        hf_instruction_lm:
+          split: "all"
+          max_samples_per_source: 1000000
+          max_total_samples: 20000000
+          seed: 42
+        gsm8k_aug:
+          split: "all"
+          seed: 42
+          train_files:
+            - "/dataset/omada/datasets/lm/GSM8K/train_aug/google_gemma-3-27b-it/train.csv"
+            - "/dataset/omada/datasets/lm/GSM8K/train_aug/Qwen_Qwen3-30B-A3B-Instruct-2507/train.csv"
+            - "/dataset/omada/datasets/lm/GSM8K/train_aug/Qwen_Qwen3-32B/train.csv"
+            - "/dataset/omada/datasets/lm/MATH/train_aug/google_gemma-3-27b-it/train.csv"
+            - "/dataset/omada/datasets/lm/MATH/train_aug/Qwen_Qwen3-30B-A3B-Instruct-2507/train.csv"
+            - "/dataset/omada/datasets/lm/MATH/train_aug/Qwen_Qwen3-32B/train.csv"
+          test_files:
+            - "/dataset/omada/datasets/lm/GSM8K/test_aug/google_gemma-3-27b-it/test.csv"
+            - "/dataset/omada/datasets/lm/GSM8K/test_aug/Qwen_Qwen3-30B-A3B-Instruct-2507/test.csv"
+            - "/dataset/omada/datasets/lm/GSM8K/test_aug/Qwen_Qwen3-32B/test.csv"
+            - "/dataset/omada/datasets/lm/MATH/test_aug/google_gemma-3-27b-it/test.csv"
+            - "/dataset/omada/datasets/lm/MATH/test_aug/Qwen_Qwen3-30B-A3B-Instruct-2507/test.csv"
+            - "/dataset/omada/datasets/lm/MATH/test_aug/Qwen_Qwen3-32B/test.csv"
+          include_reasoning: true
+          include_answer: false
+          max_total_samples: null
+        # mmlu_aux:
+          # dataset_dir: "/dataset/omada/datasets/lm/MMLU"
+          # seed: 42
+          # max_total_samples: null
+          # add_ntm: true
+          # split: "val+test"
+        # gpqa_train:
+        #   dataset_dir: "/dataset/omada/datasets/lm/GPQA/train"
+        #   seed: 42
+        #   max_total_samples: null
+        #   answer_mode: "label_text"
+        # arc_c_train:
+        #   dataset_dir: "/dataset/omada/datasets/lm/ARC/ARC-Challenge"
+        #   seed: 42
+        #   max_total_samples: null
+        #   split: "all"
+        reasoning_sft_csv:
+          csv_path: "/dataset/omada/datasets/lm/filtered/1024_trimmed_aug_datasets.csv"
+          seed: 42
+          max_total_samples: null
+        speech2speech:
+          - name: "instructs2s_200k_en"
+            wav_pairs_file: "/dataset/omada/datasets/speech/InstructS2S-200K/en/wav/pairs.txt"
+            use_precomputed_tokens: true
+            precomputed_tokens_root: "/dataset/omada/datasets/speech_tokens/instructs2s_200k_en"
+          - name: "instructs2s_eval"
+            wav_pairs_file: "/dataset/omada/datasets/speech/instructs2s_eval_whisper_with_assistant.pairs.txt"
+            use_precomputed_tokens: true
+            precomputed_tokens_root: "/dataset/omada/datasets/speech_tokens/instructs2s_eval"
+        mmu_interleaved:
+          # - jsonl_path: "/dataset/omada/datasets/mmbench_test_pseudo_cambrian_shared.jsonl"
+          - jsonl_path: "/dataset/omada/datasets/mmbench_test_pseudo_cambrian_shared_wrongdup.jsonl"
+            # - jsonl_path: ""
+            image_root: "/"
+            resolution: 480
+          # - dataset_name: "lmms-lab/POPE"
+          #   split: "test"
+          #   resolution: 480
+          #   cache_dir: "/dataset/omada/datasets"
+          #   local_files_only: true
+          - dataset_name: "lmms-lab/MME"
+            split: "test"
+            resolution: 480
+            cache_dir: "/dataset/omada/datasets"
+            local_files_only: true
+            answer_noise_prob: 0.50
+            answer_noise_seed: 42
+            answer_noise_strategy: "swap"
+          # - dataset_name: "lmms-lab/MMBench_EN"
+          #   split: "dev"
+          #   resolution: 480
+          #   cache_dir: "/dataset/omada/datasets"
+          #   local_files_only: true
+          # - dataset_name: "lmms-lab/MMMU"
+          #   split: "all_except_test"
+          #   resolution: 480
+          #   cache_dir: "/dataset/omada/datasets"
+          #   local_files_only: true
+          # - dataset_name: "GQA_TestDev_Balanced"
+          #   gqa_jsonl_path: "/dataset/omada/datasets/gqa/GQA_TestDev_Balanced.jsonl"
+          #   resolution: 480
+          - jsonl_path: "/dataset/omada/datasets/Cambrian-10M/jsons/Cambrian7M_withsystemprompt_300k_balanced.jsonl"
+            image_root: "/dataset/omada/datasets/Cambrian-10M"
+            resolution: 480
+        # subset for gigaspeech: xs, xl
+        # subset for librispeech: train-clean-360, train-clean-100
+        # subset for commonvoice: validated, invalidated
+        audio_data:
+          - name: "jsonl"
+            jsonl_path: "/dataset/omada/datasets/speech/seedtts_test_combined_en.jsonl"
+            text_key: "text"
+            audio_key: "speech"
+            use_precomputed_tokens: true
+            require_precomputed_tokens: true
+            precomputed_tokens_root: "/dataset/omada/datasets/speech_tokens/seedtts_test_combined_en"
+          - name: "jsonl"
+            jsonl_path: "/dataset/omada/datasets/speech/instructs2s_s2t_t2s_combined.jsonl"
+            text_key: "text"
+            audio_key: "speech"
+            use_precomputed_tokens: true
+            require_precomputed_tokens: false
+            precomputed_tokens_root: "/dataset/omada/datasets/speech_tokens/instructs2s_200k_en"
+          - name: "librispeech"
+            subset: "clean"
+            split: "all"
+            use_precomputed_tokens: true
+            require_precomputed_tokens: true
+            precomputed_tokens_root: "/dataset/omada/cache/librispeech_tokens"
+          # - name: "commonvoice"
+          #   subset: "validated"
+        #########################################################################
+        require_cached_audio_tokens: true
+        shuffle_buffer_size: 1000
+        num_workers: 2
+        resolution: 336
+        t2i_resolution: 512
+        # resolution: 16
+        pin_memory: True
+        persistent_workers: True
+        dataloader_timeout: 0
+        
+
+    speech_token_cache:
+        enable: true
+        root: "cache/speech_tokens"
+        max_items_in_memory: 4096
+
+    preprocessing:
+        max_seq_length: 128 # backward compatibility
+        max_seq_length_text: 1024   # for pure text/lm outputs (input trunc 256, output pad 768)
+        max_seq_length_lm_input: 1024   # tokenizer truncation for LM inputs
+        max_seq_length_mmu: 128    # for mmu/video text (output pad)
+        max_seq_length_mmu_input: 128  # for mmu/video text input truncation
+        max_seq_length_s2t: 128    # for speech-to-text prompts/targets
+        max_seq_length_t2i: 128    # for text-to-image prompts
+        max_seq_length_t2s: 128    # for text-to-speech prompts
+        max_aud_length: 512 # for audio tokens
+        max_aud_length_short: 256 # for short audio tokens
+        resolution: 224 # for video tokens
+        # max_seq_length: 16 # for text tokens
+        # max_aud_length: 16 # for audio tokens
+        # resolution: 16 # for video tokens
+        center_crop: False
+        random_flip: False
+
+optimizer:
+    name: adamw
+    params: # default adamw params
+        learning_rate: 0.00002
+        # learning_rate: 0.00004859840219369731
+        scale_lr: False # scale learning rate by total batch size
+        beta1: 0.9
+        beta2: 0.999
+        weight_decay: 0.01
+        epsilon: 1e-8
+
+lr_scheduler:
+    scheduler: "cosine"
+    params:
+        learning_rate: ${optimizer.params.learning_rate}
+        warmup_steps: 1000
+        # warmup_steps: 0
+        min_lr_scale: 0.1
+
+training:
+    gradient_accumulation_steps: 2
+    noise_type: "mask"
+    batch_size_t2i: 1
+    batch_size_lm: 1
+    batch_size_mmu: 1
+    batch_size_v2t: 1
+    batch_size_v2s: 0
+    batch_size_s2t: 1
+    batch_size_t2s: 1
+    batch_size_s2s: 0
+
+    mixed_precision: "bf16"
+    enable_tf32: True
+    seed: 10086
+    max_train_steps: 1000000 
+    max_train_epochs: NONE
+    overfit_one_batch: False
+    cond_dropout_prob: 0.1
+    min_masking_rate: 0.0
+    label_smoothing: 0.0
+    max_grad_norm: 1
+    guidance_scale: 3.5
+    generation_timesteps: 20
+
+    t2i_coeff: 0.2
+    i2i_coeff: 0.2
+    lm_coeff: 0.2
+    mmu_coeff: 0.3
+    v2t_coeff: 0.5
+    v2s_coeff: 0.0
+    t2s_coeff: 0.4
+    s2t_coeff: 0.4
+    s2s_coeff: 0.0

MMaDA/inference/gradio_multimodal_demo_inst.py

@@ -1777,6 +1777,7 @@ class OmadaDemo:
                 noise_schedule=self.mask_schedule,
                 noise_type=self.noise_type,
                 seq_len=seq_len,
+                resolution=seq_len,
                 mask_token_id=self.mask_token_id,
                 codebook_size=self.codebook_size,
                 uni_prompting=self.uni_prompting,
@@ -1854,6 +1855,7 @@ class OmadaDemo:
             noise_schedule=self.mask_schedule,
             noise_type=self.noise_type,
             seq_len=seq_len,
+            resolution=seq_len,
             mask_token_id=self.mask_token_id,
             codebook_size=self.codebook_size,
             uni_prompting=self.uni_prompting,

app.py

@@ -879,13 +879,14 @@ def get_app() -> OmadaDemo:
                 # Concurrent init race (warmup vs request): safe to ignore.
                 pass
 
+        # Prefer a repo-local Space config first, then fall back to demo configs.
+        space_demo_cfg = PROJECT_ROOT / "MMaDA" / "inference" / "demo" / "space_demo.yaml"
         default_cfg = PROJECT_ROOT / "MMaDA" / "inference" / "demo" / "demo.yaml"
         legacy_cfg = PROJECT_ROOT / "MMaDA" / "configs" / "mmada_demo.yaml"
-        eval_cfg = Path("/dataset/omada/OMaDA/MMaDA/configs/omada_instruction_tuning2.yaml")
         train_config = os.getenv("TRAIN_CONFIG_PATH")
         if not train_config:
-            if eval_cfg.exists():
-                train_config = str(eval_cfg)
+            if space_demo_cfg.exists():
+                train_config = str(space_demo_cfg)
             else:
                 train_config = str(default_cfg if default_cfg.exists() else legacy_cfg)
 
@@ -2882,8 +2883,8 @@ with gr.Blocks(**_blocks_kwargs) as demo:
                 {"mode": "MMU (Image → Text)", "text": _get_example_value(MMU_EXAMPLES, 1, 1, _get_example_value(MMU_EXAMPLES, 0, 1, DEFAULT_MMU_PROMPT)), "image": _get_example_value(MMU_EXAMPLES, 1, 0, _get_example_value(MMU_EXAMPLES, 0, 0, None)), "audio": None, "video": None},
             ],
             "MMU (Video → Text)": [
-                {"mode": "MMU (Video → Text)", "text": "", "image": None, "audio": None, "video": _get_example_value(V2T_EXAMPLES, 0, 0, None)},
-                {"mode": "MMU (Video → Text)", "text": "", "image": None, "audio": None, "video": _get_example_value(V2T_EXAMPLES, 1, 0, _get_example_value(V2T_EXAMPLES, 0, 0, None))},
+                {"mode": "MMU (Video → Text)", "text": "", "image": None, "audio": None, "video": _get_example_value(V2T_EXAMPLES, -2, 0, _get_example_value(V2T_EXAMPLES, 0, 0, None))},
+                {"mode": "MMU (Video → Text)", "text": "", "image": None, "audio": None, "video": _get_example_value(V2T_EXAMPLES, -1, 0, _get_example_value(V2T_EXAMPLES, 1, 0, _get_example_value(V2T_EXAMPLES, 0, 0, None)))},
             ],
             "Image Generation": [
                 {"mode": "Image Generation", "text": _get_example_value(T2I_EXAMPLES, 0, 0, "A cinematic mountain landscape at sunrise."), "image": None, "audio": None, "video": None},