Delete sft/1M3

sft/1M3/Full_remoe/checkpoint-13786/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

sft/1M3/Full_remoe/checkpoint-13786/config.json

@@ -1,203 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "bal_comp_loss_coef": 0.01,
-  "balance_loss_coef": 1.5e-323,
-  "bos_token_id": 1,
-  "clip_smoe": true,
-  "diversity_loss_coef": 0.01,
-  "dropout": false,
-  "e_loss_coef": 0.001,
-  "embd_pdrop": 0.0,
-  "entropy_advance_loss": false,
-  "eos_token_id": 32000,
-  "freeze_backbone": false,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "hybrid": false,
-  "image_aspect_ratio": "pad",
-  "init_weight": true,
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "is_cosine": false,
-  "is_norm_weight": false,
-  "local_rank": 0,
-  "loss1": "balanceloss",
-  "loss2": "zloss",
-  "luna": false,
-  "max_compete_in_iter": 3,
-  "max_position_embeddings": 131072,
-  "mlp_smoe": true,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "moe",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft",
-  "model_type": "llava_phi",
-  "moe_name": "remoe",
-  "moe_relu_l1_reg_coeff_multiplier": 1.2,
-  "mp_pixel_shuffle_factor": 1,
-  "norm_softmax": false,
-  "normalization": true,
-  "num_attention_heads": 32,
-  "num_experts": 4,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 2,
-  "number_of_previous_tokens": 2,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "pretrain_mm_mlp_adapter": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/mm_projector.bin",
-  "rate_compete": 0.2,
-  "rate_flip": 0.05,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": {
-    "long_factor": [
-      1.0800000429153442,
-      1.1100000143051147,
-      1.1399999856948853,
-      1.340000033378601,
-      1.5899999141693115,
-      1.600000023841858,
-      1.6200000047683716,
-      2.620000123977661,
-      3.2300000190734863,
-      3.2300000190734863,
-      4.789999961853027,
-      7.400000095367432,
-      7.700000286102295,
-      9.09000015258789,
-      12.199999809265137,
-      17.670000076293945,
-      24.46000099182129,
-      28.57000160217285,
-      30.420001983642578,
-      30.840002059936523,
-      32.590003967285156,
-      32.93000411987305,
-      42.320003509521484,
-      44.96000289916992,
-      50.340003967285156,
-      50.45000457763672,
-      57.55000305175781,
-      57.93000411987305,
-      58.21000289916992,
-      60.1400032043457,
-      62.61000442504883,
-      62.62000274658203,
-      62.71000289916992,
-      63.1400032043457,
-      63.1400032043457,
-      63.77000427246094,
-      63.93000411987305,
-      63.96000289916992,
-      63.970001220703125,
-      64.02999877929688,
-      64.06999969482422,
-      64.08000183105469,
-      64.12000274658203,
-      64.41000366210938,
-      64.4800033569336,
-      64.51000213623047,
-      64.52999877929688,
-      64.83999633789062
-    ],
-    "short_factor": [
-      1.0,
-      1.0199999809265137,
-      1.0299999713897705,
-      1.0299999713897705,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0699999332427979,
-      1.0999999046325684,
-      1.1099998950958252,
-      1.1599998474121094,
-      1.1599998474121094,
-      1.1699998378753662,
-      1.2899998426437378,
-      1.339999794960022,
-      1.679999828338623,
-      1.7899998426437378,
-      1.8199998140335083,
-      1.8499997854232788,
-      1.8799997568130493,
-      1.9099997282028198,
-      1.9399996995925903,
-      1.9899996519088745,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0799996852874756,
-      2.0899996757507324,
-      2.189999580383301,
-      2.2199995517730713,
-      2.5899994373321533,
-      2.729999542236328,
-      2.749999523162842,
-      2.8399994373321533
-    ],
-    "type": "longrope"
-  },
-  "rope_theta": 10000.0,
-  "router_loss_coef": 0.01,
-  "router_theta": 0.1,
-  "router_z_loss_coef": 0.001,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 262144,
-  "sparse_upcycling": true,
-  "std_gate": 0.02,
-  "strategy_train": "base",
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 2048,
-  "tokenizer_padding_side": "right",
-  "topk_max": 2,
-  "topk_min": 1,
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.0",
-  "tune_mm_mlp_adapter": false,
-  "unit_test": true,
-  "use_cache": false,
-  "use_mm_proj": true,
-  "use_old": false,
-  "version": "phi35",
-  "vision_tower": "google/siglip-so400m-patch14-224",
-  "vision_tower_dir": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/clip.bin",
-  "vocab_size": 32064,
-  "warm_up": 0.05
-}

sft/1M3/Full_remoe/checkpoint-13786/generation_config.json

@@ -1,12 +0,0 @@
-{
-  "_from_model_config": true,
-  "bos_token_id": 1,
-  "do_sample": true,
-  "eos_token_id": [
-    32007,
-    32001,
-    32000
-  ],
-  "pad_token_id": 32000,
-  "transformers_version": "4.43.0"
-}

sft/1M3/Full_remoe/checkpoint-13786/latest

@@ -1 +0,0 @@
-global_step13786

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1412_llava...image_llava_model_args_595126/rank0_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 0 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1412_llava...image_llava_model_args_595126/results.json

@@ -1,146 +0,0 @@
-{
-    "results": {
-        "hallusion_bench_image": {
-            "aAcc,none": 44.164,
-            "aAcc_stderr,none": "N/A",
-            "fAcc,none": 14.4509,
-            "fAcc_stderr,none": "N/A",
-            "qAcc,none": 10.5495,
-            "qAcc_stderr,none": "N/A",
-            "alias": "hallusion_bench_image"
-        },
-        "mathvista_testmini": {
-            "gpt_eval_score,none": 32.3,
-            "gpt_eval_score_stderr,none": "N/A",
-            "alias": "mathvista_testmini"
-        }
-    },
-    "configs": {
-        "hallusion_bench_image": {
-            "task": "hallusion_bench_image",
-            "dataset_path": "lmms-lab/HallusionBench",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "image",
-            "doc_to_visual": "<function hb_doc_to_visual at 0x7fe2d0dbcee0>",
-            "doc_to_text": "<function hb_doc_to_text at 0x7fe2d0c13670>",
-            "doc_to_target": "gt_answer_details",
-            "process_results": "<function hb_process_results at 0x7fe2d0c13e50>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "aAcc",
-                    "aggregation": "<function hb_aggregation_result_aAcc at 0x7fe2d0a94820>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "qAcc",
-                    "aggregation": "<function hb_aggregation_result_qAcc at 0x7fe2d0a94e50>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "fAcc",
-                    "aggregation": "<function hb_aggregation_result_fAcc at 0x7fe2d0a60670>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 128,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": ""
-                }
-            }
-        },
-        "mathvista_testmini": {
-            "task": "mathvista_testmini",
-            "dataset_path": "AI4Math/MathVista",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "testmini",
-            "doc_to_visual": "<function mathvista_doc_to_visual at 0x7fe2a4fc89d0>",
-            "doc_to_text": "<function mathvista_doc_to_text at 0x7fe2a4fd30d0>",
-            "doc_to_target": "answer",
-            "process_results": "<function mathvista_process_results at 0x7fe2a4bcb790>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "gpt_eval_score",
-                    "aggregation": "<function mathvista_aggregate_results at 0x7fe2a4bd3e50>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "until": [
-                    "ASSISTANT:"
-                ],
-                "max_new_tokens": 1024,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "image_aspect_ratio": "original"
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "shot_type": "format-prompt",
-                    "shot": 0,
-                    "use_caption": false,
-                    "use_ocr": false
-                },
-                "phi3v": {
-                    "shot_type": "solution"
-                }
-            },
-            "model_specific_generation_kwargs": {
-                "llava": {
-                    "image_aspect_ratio": "original"
-                }
-            }
-        }
-    },
-    "versions": {
-        "hallusion_bench_image": "Yaml",
-        "mathvista_testmini": "Yaml"
-    },
-    "n-shot": {
-        "hallusion_bench_image": 0,
-        "mathvista_testmini": 0
-    },
-    "model_configs": {
-        "model": "llava",
-        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/Full_remoe/checkpoint-13786,conv_template=phi35",
-        "batch_size": "1",
-        "device": null,
-        "limit": null,
-        "bootstrap_iters": 100000,
-        "gen_kwargs": ""
-    },
-    "git_hash": "289c7fe5"
-}

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/infovqa_val.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f26b32df54ad8f9f5c8ad9413761458b2923524459e0498689101f28c8e7d4bd
-size 576448204

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/mmbench_en_dev.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1ef6a3572059b9b3123aeff247cb17eaa934c5f33b7c4fdb862f5d56d9564c26
-size 14568248

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/mmerealworld_lite.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e8320b5d0524c75293d97a563749adb88c6e9812000a4c17282dc9cc68c17cb4
-size 1994104281

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/rank0_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 0 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/rank1_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 1 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/rank2_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 2 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/results.json

@@ -1,683 +0,0 @@
-{
-    "results": {
-        "ai2d": {
-            "exact_match,flexible-extract": 0.614961139896373,
-            "exact_match_stderr,flexible-extract": 0.00875805772380654,
-            "alias": "ai2d"
-        },
-        "infovqa_val": {
-            "anls,none": 0.25276686897536593,
-            "anls_stderr,none": 0.008213137970952944,
-            "alias": "infovqa_val"
-        },
-        "mmbench_en_dev": {
-            "gpt_eval_score,none": 54.98281786941581,
-            "gpt_eval_score_stderr,none": "N/A",
-            "submission,none": null,
-            "submission_stderr,none": "N/A",
-            "alias": "mmbench_en_dev"
-        },
-        "mme": {
-            "mme_cognition_score,none": 276.7857142857143,
-            "mme_cognition_score_stderr,none": "N/A",
-            "mme_percetion_score,none": 1103.8987595038016,
-            "mme_percetion_score_stderr,none": "N/A",
-            "alias": "mme"
-        },
-        "mmerealworld_lite": {
-            "mme_realworld_score,none": 0.2636789994788953,
-            "mme_realworld_score_stderr,none": "N/A",
-            "alias": "mmerealworld_lite"
-        },
-        "mmmu_val": {
-            "mmmu_acc,none": 0.40111,
-            "mmmu_acc_stderr,none": "N/A",
-            "alias": "mmmu_val"
-        },
-        "mmstar": {
-            "coarse perception,none": 0.4745496248584787,
-            "coarse perception_stderr,none": "N/A",
-            "fine-grained perception,none": 0.26887128469663474,
-            "fine-grained perception_stderr,none": "N/A",
-            "instance reasoning,none": 0.3861223080142979,
-            "instance reasoning_stderr,none": "N/A",
-            "logical reasoning,none": 0.35995214906106,
-            "logical reasoning_stderr,none": "N/A",
-            "math,none": 0.39322150940033856,
-            "math_stderr,none": "N/A",
-            "science & technology,none": 0.25396890595797994,
-            "science & technology_stderr,none": "N/A",
-            "alias": "mmstar"
-        },
-        "scienceqa_img": {
-            "exact_match,none": 0.7441745166088249,
-            "exact_match_stderr,none": 0.00971771224462157,
-            "alias": "scienceqa_img"
-        },
-        "seedbench_2_plus": {
-            "seedbench_2_plus_Chart,none": 0.44814814814814813,
-            "seedbench_2_plus_Chart_stderr,none": "N/A",
-            "seedbench_2_plus_all,none": 0.45322793148880103,
-            "seedbench_2_plus_all_stderr,none": "N/A",
-            "seedbench_2_plus_Web,none": 0.4954545454545455,
-            "seedbench_2_plus_Web_stderr,none": "N/A",
-            "seedbench_2_plus_Map,none": 0.42379182156133827,
-            "seedbench_2_plus_Map_stderr,none": "N/A",
-            "alias": "seedbench_2_plus"
-        },
-        "textvqa_val": {
-            "exact_match,none": 0.2150200002193451,
-            "exact_match_stderr,none": 0.005632905626047891,
-            "submission,none": null,
-            "submission_stderr,none": "N/A",
-            "alias": "textvqa_val"
-        }
-    },
-    "configs": {
-        "ai2d": {
-            "task": "ai2d",
-            "dataset_path": "lmms-lab/ai2d",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "test",
-            "doc_to_visual": "<function ai2d_doc_to_visual at 0x7f4c749b3af0>",
-            "doc_to_text": "<function ai2d_doc_to_text at 0x7f4c749b3dc0>",
-            "doc_to_target": "<function ai2d_doc_to_target at 0x7f4c749b93a0>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "exact_match",
-                    "aggregation": "mean",
-                    "higher_is_better": true,
-                    "ignore_case": true,
-                    "ignore_punctuation": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 512,
-                "temperature": 0.0,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "filter_list": [
-                {
-                    "name": "flexible-extract",
-                    "filter": [
-                        {
-                            "function": "<class 'utils.MultiChoiceRegexFilter'>",
-                            "group_select": 0,
-                            "ignore_case": true,
-                            "ignore_punctuation": true,
-                            "regex_pattern": "([A-Z])\\."
-                        }
-                    ]
-                }
-            ],
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "prompt_format": "mcq",
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer with the option's letter from the given choices directly."
-                },
-                "gpt4v": {
-                    "prompt_format": "mcq",
-                    "pre_prompt": "",
-                    "post_prompt": "\nAbove choices are given in {option}. {content} format.\nPlease answer with the option letter from the given choices directly."
-                },
-                "qwen_vl": {
-                    "prompt_format": "qa",
-                    "pre_prompt": "",
-                    "post_prompt": " Answer:"
-                },
-                "xcomposer2_4khd": {
-                    "prompt_format": "mcq_xcomposer",
-                    "pre_prompt": "[UNUSED_TOKEN_146]user\nQuestion: ",
-                    "post_prompt": "[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\nThe answer is"
-                }
-            },
-            "model_specific_target_kwargs": {
-                "default": "mcq",
-                "qwen_vl": "qa"
-            }
-        },
-        "infovqa_val": {
-            "task": "infovqa_val",
-            "dataset_path": "lmms-lab/DocVQA",
-            "dataset_name": "InfographicVQA",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "validation",
-            "doc_to_visual": "<function infovqa_doc_to_visual at 0x7f4ca74199d0>",
-            "doc_to_text": "<function infovqa_doc_to_text at 0x7f4ca7419ca0>",
-            "doc_to_target": "answers",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "anls",
-                    "aggregation": "mean",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 32,
-                "temperature": 0.0,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer the question using a single word or phrase."
-                }
-            }
-        },
-        "mmbench_en_dev": {
-            "task": "mmbench_en_dev",
-            "dataset_path": "lmms-lab/MMBench",
-            "dataset_name": "en",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "dev",
-            "doc_to_visual": "<function mmbench_doc_to_visual at 0x7f4cd45bd430>",
-            "doc_to_text": "<function mmbench_doc_to_text at 0x7f4cd45bd940>",
-            "doc_to_target": "answer",
-            "process_results": "<function mmbench_process_results at 0x7f4cd45bde50>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "gpt_eval_score",
-                    "aggregation": "<function mmbench_aggregate_dev_results_eval at 0x7f4cd4631ca0>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "submission",
-                    "aggregation": "<function mmbench_aggregate_dev_results_submission at 0x7f4cd45bd1f0>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "until": [
-                    "ASSISTANT:"
-                ],
-                "max_new_tokens": 1024,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "image_aspect_ratio": "original"
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer with the option's letter from the given choices directly."
-                }
-            },
-            "model_specific_generation_kwargs": {
-                "llava": {
-                    "image_aspect_ratio": "original"
-                }
-            }
-        },
-        "mme": {
-            "task": "mme",
-            "dataset_path": "lmms-lab/MME",
-            "dataset_kwargs": {
-                "token": false
-            },
-            "test_split": "test",
-            "doc_to_visual": "<function mme_doc_to_visual at 0x7f4cd56dc8b0>",
-            "doc_to_text": "<function mme_doc_to_text at 0x7f4cd463d160>",
-            "doc_to_target": "answer",
-            "process_results": "<function mme_process_results at 0x7f4cd463d700>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "mme_percetion_score",
-                    "aggregation": "<function mme_aggregate_results at 0x7f4cd463dc10>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "mme_cognition_score",
-                    "aggregation": "<function mme_aggregate_results at 0x7f4cd46450d0>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 16,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer the question using a single word or phrase."
-                },
-                "gpt4v": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer the question with Yes or No."
-                },
-                "qwen_vl": {
-                    "pre_prompt": "",
-                    "post_prompt": " Answer:"
-                },
-                "otterhd": {
-                    "pre_prompt": "",
-                    "post_prompt": " Answer:"
-                },
-                "xcomposer2_4khd": {
-                    "pre_prompt": "[UNUSED_TOKEN_146]user\n",
-                    "post_prompt": " Answer this question briefly[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"
-                }
-            }
-        },
-        "mmerealworld_lite": {
-            "task": "mmerealworld_lite",
-            "dataset_path": "yifanzhang114/MME-RealWorld-lite-lmms-eval",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "train",
-            "doc_to_visual": "<function mme_realworld_doc_to_visual at 0x7f4c7515aca0>",
-            "doc_to_text": "<function mme_realworld_doc_to_text at 0x7f4c7515e790>",
-            "doc_to_target": "answer",
-            "process_results": "<function mme_realworld_process_results at 0x7f4c75167430>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "mme_realworld_score",
-                    "aggregation": "<function mme_realworld_aggregate_results at 0x7f4c75167f70>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 16,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nSelect the best answer to the above multiple-choice question based on the image. Respond with only the letter (A, B, C, D, or E) of the correct option."
-                },
-                "gpt4v": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nSelect the best answer to the above multiple-choice question based on the image. Respond with only the letter (A, B, C, D, or E) of the correct option."
-                },
-                "xcomposer2_4khd": {
-                    "pre_prompt": "[UNUSED_TOKEN_146]user\n",
-                    "post_prompt": " Answer this question with A, B, C, or D.[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n"
-                }
-            }
-        },
-        "mmmu_val": {
-            "task": "mmmu_val",
-            "dataset_path": "lmms-lab/MMMU",
-            "test_split": "validation",
-            "doc_to_visual": "<function mmmu_doc_to_visual at 0x7f4c8746f700>",
-            "doc_to_text": "<function mmmu_doc_to_text at 0x7f4c8743d430>",
-            "doc_to_target": "answer",
-            "process_results": "<function mmmu_process_results at 0x7f4c872ad310>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "mmmu_acc",
-                    "aggregation": "<function mmmu_aggregate_results at 0x7f4c8706c280>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 128,
-                "until": [
-                    "\n\n"
-                ],
-                "image_aspect_ratio": "original"
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_generation_kwargs": {
-                "llava": {
-                    "image_aspect_ratio": "original"
-                }
-            }
-        },
-        "mmstar": {
-            "task": "mmstar",
-            "dataset_path": "Lin-Chen/MMStar",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "val",
-            "doc_to_visual": "<function mmstar_doc_to_visual at 0x7f4c7490f160>",
-            "doc_to_text": "<function mmstar_doc_to_text at 0x7f4c7490f5e0>",
-            "doc_to_target": "answer",
-            "process_results": "<function mmstar_process_results at 0x7f4c7490faf0>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "coarse perception",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c7490ff70>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "fine-grained perception",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c749193a0>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "instance reasoning",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c74919790>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "logical reasoning",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c74919b80>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "science & technology",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c74919f70>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "math",
-                    "aggregation": "<function mmstar_aggregate_results at 0x7f4c749213a0>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "until": [
-                    "\n\n"
-                ],
-                "do_sample": false
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer with the option's letter from the given choices directly"
-                }
-            }
-        },
-        "scienceqa_img": {
-            "task": "scienceqa_img",
-            "dataset_path": "lmms-lab/ScienceQA",
-            "dataset_name": "ScienceQA-IMG",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "test",
-            "doc_to_visual": "<function sqa_doc_to_visual at 0x7f4c9f37bb80>",
-            "doc_to_text": "<function sqa_doc_to_text at 0x7f4c9f37bdc0>",
-            "doc_to_target": "<function sqa_doc_to_target at 0x7f4c9f2fc1f0>",
-            "process_results": "<function sqa_process_results at 0x7f4c9f2fc550>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "exact_match",
-                    "aggregation": "mean",
-                    "higher_is_better": true,
-                    "ignore_case": true,
-                    "ignore_punctuation": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 16,
-                "temperature": 0.0,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ],
-                "image_aspect_ratio": "original"
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "format": "default",
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer with the option's letter from the given choices directly."
-                },
-                "qwen_vl": {
-                    "format": "qwen_vl"
-                },
-                "idefics2": {
-                    "format": "default",
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer:"
-                }
-            },
-            "model_specific_generation_kwargs": {
-                "llava": {
-                    "image_aspect_ratio": "original"
-                }
-            }
-        },
-        "seedbench_2_plus": {
-            "task": "seedbench_2_plus",
-            "dataset_path": "doolayer/SEED-Bench-2-Plus",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "test",
-            "doc_to_visual": "<function seed_doc_to_visual at 0x7f4c9dcfcca0>",
-            "doc_to_text": "<function seed_doc_to_text at 0x7f4c9dc5a310>",
-            "doc_to_target": "answer",
-            "process_results": "<function seed_process_result at 0x7f4c9dc5a820>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "seedbench_2_plus_Chart",
-                    "aggregation": "<function seed_aggregation_result at 0x7f4c9dc5ad30>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "seedbench_2_plus_Map",
-                    "aggregation": "<function seed_aggregation_result at 0x7f4c9dc7b1f0>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "seedbench_2_plus_Web",
-                    "aggregation": "<function seed_aggregation_result at 0x7f4c9dc7b670>",
-                    "higher_is_better": true
-                },
-                {
-                    "metric": "seedbench_2_plus_all",
-                    "aggregation": "<function seed_aggregation_result at 0x7f4c9dc7baf0>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "until": [
-                    "ASSISTANT:"
-                ],
-                "max_new_tokens": 16,
-                "image_aspect_ratio": "original"
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ],
-            "model_specific_prompt_kwargs": {
-                "llava": {
-                    "img_token": "<image>",
-                    "post_prompt": "Answer with the option's letter from the given choices directly."
-                },
-                "gpt4V": {
-                    "img_token": "<image>",
-                    "post_prompt": "Answer with the option's letter from the given choices directly."
-                },
-                "default": {
-                    "img_token": "<image>",
-                    "post_prompt": "Answer with the option's letter from the given choices directly."
-                }
-            }
-        },
-        "textvqa_val": {
-            "task": "textvqa_val",
-            "dataset_path": "lmms-lab/textvqa",
-            "test_split": "validation",
-            "doc_to_visual": "<function textvqa_doc_to_visual at 0x7f4ca0a8e790>",
-            "doc_to_text": "<function textvqa_doc_to_text at 0x7f4ca0a8ec10>",
-            "doc_to_target": "answer",
-            "process_results": "<function textvqa_process_results at 0x7f4ca0a8eee0>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "exact_match",
-                    "aggregation": "mean",
-                    "higher_is_better": true,
-                    "ignore_case": true,
-                    "ignore_punctuation": true
-                },
-                {
-                    "metric": "submission",
-                    "aggregation": "<function textvqa_aggregate_submissions at 0x7f4ca0a8e550>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "until": [
-                    "ASSISTANT:"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "model_specific_prompt_kwargs": {
-                "default": {
-                    "pre_prompt": "",
-                    "post_prompt": "\nAnswer the question using a single word or phrase.",
-                    "ocr": false
-                },
-                "qwen_vl": {
-                    "pre_prompt": "",
-                    "post_prompt": " Answer:"
-                }
-            }
-        }
-    },
-    "versions": {
-        "ai2d": "Yaml",
-        "infovqa_val": "Yaml",
-        "mmbench_en_dev": "Yaml",
-        "mme": "Yaml",
-        "mmerealworld_lite": "Yaml",
-        "mmmu_val": "Yaml",
-        "mmstar": "Yaml",
-        "scienceqa_img": "Yaml",
-        "seedbench_2_plus": "Yaml",
-        "textvqa_val": "Yaml"
-    },
-    "n-shot": {
-        "ai2d": 0,
-        "infovqa_val": 0,
-        "mmbench_en_dev": 0,
-        "mme": 0,
-        "mmerealworld_lite": 0,
-        "mmmu_val": 0,
-        "mmstar": 0,
-        "scienceqa_img": 0,
-        "seedbench_2_plus": 0,
-        "textvqa_val": 0
-    },
-    "model_configs": {
-        "model": "llava",
-        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/Full_remoe/checkpoint-13786,conv_template=phi35",
-        "batch_size": "1",
-        "device": null,
-        "limit": null,
-        "bootstrap_iters": 100000,
-        "gen_kwargs": ""
-    },
-    "git_hash": "289c7fe5"
-}

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/submissions/mmbench_en_dev_results.json

@@ -1 +0,0 @@
-{"overall_acc": 0.5498281786941581, "category_acc": {"action_recognition": 0.7962962962962963, "attribute_comparison": 0.36363636363636365, "attribute_recognition": 0.6621621621621622, "celebrity_recognition": 0.48484848484848486, "function_reasoning": 0.7721518987341772, "future_prediction": 0.375, "identity_reasoning": 0.9333333333333333, "image_emotion": 0.54, "image_quality": 0.09433962264150944, "image_scene": 0.875, "image_style": 0.6981132075471698, "image_topic": 0.7222222222222222, "nature_relation": 0.3958333333333333, "object_localization": 0.2345679012345679, "ocr": 0.7948717948717948, "physical_property_reasoning": 0.49333333333333335, "physical_relation": 0.5416666666666666, "social_relation": 0.5813953488372093, "spatial_relationship": 0.17777777777777778, "structuralized_imagetext_understanding": 0.358974358974359}, "l2_category_acc": {"attribute_reasoning": 0.7035175879396985, "coarse_perception": 0.6283783783783784, "finegrained_perception (cross-instance)": 0.46853146853146854, "finegrained_perception (instance-level)": 0.5017064846416383, "logic_reasoning": 0.3644067796610169, "relation_reasoning": 0.4956521739130435}}

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/submissions/mmbench_en_dev_results.xlsx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2d0e0fe22a7c47e99366258abf96d09e305355def5dc23c3e7e2f2f78e51fdda
-size 843188

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1428_llava...l_mme_llava_model_args_595126/textvqa_val.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6c92ae4eb14b5fb41c01b5c572fb62bd0815727bfc0aababd01756efebff0fbc
-size 13260575

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1459_llava...bench_llava_model_args_595126/rank0_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 0 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1459_llava...bench_llava_model_args_595126/rank1_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 1 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1459_llava...bench_llava_model_args_595126/rank2_metric_eval_done.txt

@@ -1 +0,0 @@
-rank 2 eval done

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1459_llava...bench_llava_model_args_595126/results.json

@@ -1,67 +0,0 @@
-{
-    "results": {
-        "ocrbench": {
-            "ocrbench_accuracy,none": 0.191,
-            "ocrbench_accuracy_stderr,none": "N/A",
-            "alias": "ocrbench"
-        }
-    },
-    "configs": {
-        "ocrbench": {
-            "task": "ocrbench",
-            "dataset_path": "echo840/OCRBench",
-            "dataset_kwargs": {
-                "token": true
-            },
-            "test_split": "test",
-            "doc_to_visual": "<function ocrbench_doc_to_visual at 0x7fb593092b80>",
-            "doc_to_text": "<function ocrbench_doc_to_text at 0x7fb592b56160>",
-            "doc_to_target": "answer",
-            "process_results": "<function ocrbench_process_results at 0x7fb592b564c0>",
-            "description": "",
-            "target_delimiter": " ",
-            "fewshot_delimiter": "\n\n",
-            "metric_list": [
-                {
-                    "metric": "ocrbench_accuracy",
-                    "aggregation": "<function ocrbench_aggregate_accuracy at 0x7fb592b56820>",
-                    "higher_is_better": true
-                }
-            ],
-            "output_type": "generate_until",
-            "generation_kwargs": {
-                "max_new_tokens": 128,
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "num_beams": 1,
-                "do_sample": false,
-                "until": [
-                    "\n\n"
-                ]
-            },
-            "repeats": 1,
-            "should_decontaminate": false,
-            "metadata": [
-                {
-                    "version": 0.0
-                }
-            ]
-        }
-    },
-    "versions": {
-        "ocrbench": "Yaml"
-    },
-    "n-shot": {
-        "ocrbench": 0
-    },
-    "model_configs": {
-        "model": "llava",
-        "model_args": "pretrained=/cm/archive/namnv78/checkpoints/Xphi35-siglip224/SMOE/1M3/Full_remoe/checkpoint-13786,conv_template=phi35",
-        "batch_size": "1",
-        "device": null,
-        "limit": null,
-        "bootstrap_iters": 100000,
-        "gen_kwargs": ""
-    },
-    "git_hash": "289c7fe5"
-}

sft/1M3/Full_remoe/checkpoint-13786/logs/0627_1459_llava...bench_llava_model_args_595126/results/ocrbench_results.txt

@@ -1,18 +0,0 @@
-######################### OCRBench #############################
-Text Recognition(Total 300): 109
----------------- Details of Recognition Score ------------------
-Regular Text Recognition(Total 50): 23
-Irregular Text Recognition(Total 50): 22
-Artistic Text Recognition(Total 50): 36
-Handwriting Recognition(Total 50): 15
-Digit String Recognition(Total 50): 6
-Non-Semantic Text Recognition(Total 50): 7
-----------------------------------------------------------------
-Scene Text-centric VQA(Total 200): 62
-----------------------------------------------------------------
-Doc-oriented VQA(Total 200): 17
-----------------------------------------------------------------
-Key Information Extraction(Total 200): 2
-Handwritten Mathematical Expression Recognition(Total 100): 1
---------------------- Final Score ------------------------------
-Final Score(Total 1000): 191

sft/1M3/Full_remoe/checkpoint-13786/model-00001-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b61b40c5a7d6b823014de1f3aac0bd78c5e5870ce6db656b12990131d8830e92
-size 4972489328

sft/1M3/Full_remoe/checkpoint-13786/model-00002-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:915ec471ac15b80f811f6966233edb819c8be0375f68dcfd0cbe9347ada2590a
-size 4985529648

sft/1M3/Full_remoe/checkpoint-13786/model-00003-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:26658d3efca94e9d3bd68dccf60b1eca21d4751d582b4d067c20986d699486aa
-size 248943552

sft/1M3/Full_remoe/checkpoint-13786/rng_state_0.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d98bcd53a09c8bc1238dda64e113dd6869500cd3502c9a4d572b3f9ffea5dc29
-size 14960

sft/1M3/Full_remoe/checkpoint-13786/rng_state_1.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f793f5c89da7b5544f5038c073ba86532c704fdd2918ea5c27bfab848c20ad52
-size 14960

sft/1M3/Full_remoe/checkpoint-13786/rng_state_2.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cee33fb501fe6367a1ba811f1189b771eecbe30ed36d18486d5aeb9e083e4fdf
-size 14960

sft/1M3/Full_remoe/checkpoint-13786/rng_state_3.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d1aa2a4d8920b433e30b4dffe2a189b30027ce66925ec5b6b0ae0f1ccd678ec7
-size 14960

sft/1M3/Full_remoe/checkpoint-13786/special_tokens_map.json

@@ -1,24 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "<unk>",
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

sft/1M3/Full_remoe/checkpoint-13786/tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723

sft/1M3/Full_remoe/checkpoint-13786/tokenizer_config.json

@@ -1,132 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "add_prefix_space": true,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": false
-    },
-    "32000": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<|placeholder1|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32003": {
-      "content": "<|placeholder2|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32004": {
-      "content": "<|placeholder3|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32005": {
-      "content": "<|placeholder4|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32006": {
-      "content": "<|system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32007": {
-      "content": "<|end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32008": {
-      "content": "<|placeholder5|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32009": {
-      "content": "<|placeholder6|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32010": {
-      "content": "<|user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "legacy": false,
-  "model_max_length": 2048,
-  "pad_token": "<unk>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false
-}

sft/1M3/Full_remoe/checkpoint-13786/trainer_state.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef3ee1b637156b86e778b44896c89f7a3ec5372e63369a290b21ac0e38205939
-size 11276881

sft/1M3/Full_remoe/checkpoint-13786/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3babbfcec145033842a4b1261cac4f82423da7e133b1a6117c45af84c17ebc6d
-size 7992

sft/1M3/Full_remoe/checkpoint-13786/zero_to_fp32.py

@@ -1,604 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: Apache-2.0
-
-# DeepSpeed Team
-
-# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
-# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
-# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
-# application.
-#
-# example: python zero_to_fp32.py . pytorch_model.bin
-
-import argparse
-import torch
-import glob
-import math
-import os
-import re
-from collections import OrderedDict
-from dataclasses import dataclass
-
-# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
-# DeepSpeed data structures it has to be available in the current python environment.
-from deepspeed.utils import logger
-from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
-                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
-                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
-
-
-@dataclass
-class zero_model_state:
-    buffers: dict()
-    param_shapes: dict()
-    shared_params: list
-    ds_version: int
-    frozen_param_shapes: dict()
-    frozen_param_fragments: dict()
-
-
-debug = 0
-
-# load to cpu
-device = torch.device('cpu')
-
-
-def atoi(text):
-    return int(text) if text.isdigit() else text
-
-
-def natural_keys(text):
-    '''
-    alist.sort(key=natural_keys) sorts in human order
-    http://nedbatchelder.com/blog/200712/human_sorting.html
-    (See Toothy's implementation in the comments)
-    '''
-    return [atoi(c) for c in re.split(r'(\d+)', text)]
-
-
-def get_model_state_file(checkpoint_dir, zero_stage):
-    if not os.path.isdir(checkpoint_dir):
-        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
-
-    # there should be only one file
-    if zero_stage <= 2:
-        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
-    elif zero_stage == 3:
-        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
-
-    if not os.path.exists(file):
-        raise FileNotFoundError(f"can't find model states file at '{file}'")
-
-    return file
-
-
-def get_checkpoint_files(checkpoint_dir, glob_pattern):
-    # XXX: need to test that this simple glob rule works for multi-node setup too
-    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
-
-    if len(ckpt_files) == 0:
-        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
-
-    return ckpt_files
-
-
-def get_optim_files(checkpoint_dir):
-    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
-
-
-def get_model_state_files(checkpoint_dir):
-    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
-
-
-def parse_model_states(files):
-    zero_model_states = []
-    for file in files:
-        state_dict = torch.load(file, map_location=device)
-
-        if BUFFER_NAMES not in state_dict:
-            raise ValueError(f"{file} is not a model state checkpoint")
-        buffer_names = state_dict[BUFFER_NAMES]
-        if debug:
-            print("Found buffers:", buffer_names)
-
-        # recover just the buffers while restoring them to fp32 if they were saved in fp16
-        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
-        param_shapes = state_dict[PARAM_SHAPES]
-
-        # collect parameters that are included in param_shapes
-        param_names = []
-        for s in param_shapes:
-            for name in s.keys():
-                param_names.append(name)
-
-        # update with frozen parameters
-        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
-        if frozen_param_shapes is not None:
-            if debug:
-                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
-            param_names += list(frozen_param_shapes.keys())
-
-        # handle shared params
-        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
-
-        ds_version = state_dict.get(DS_VERSION, None)
-
-        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
-
-        z_model_state = zero_model_state(buffers=buffers,
-                                         param_shapes=param_shapes,
-                                         shared_params=shared_params,
-                                         ds_version=ds_version,
-                                         frozen_param_shapes=frozen_param_shapes,
-                                         frozen_param_fragments=frozen_param_fragments)
-        zero_model_states.append(z_model_state)
-
-    return zero_model_states
-
-
-def parse_optim_states(files, ds_checkpoint_dir):
-
-    total_files = len(files)
-    state_dicts = []
-    for f in files:
-        state_dict = torch.load(f, map_location=device)
-        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
-        # and also handle the case where it was already removed by another helper script
-        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
-        state_dicts.append(state_dict)
-
-    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
-        raise ValueError(f"{files[0]} is not a zero checkpoint")
-    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
-    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
-
-    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
-    # parameters can be different from data parallelism for non-expert parameters. So we can just
-    # use the max of the partition_count to get the dp world_size.
-
-    if type(world_size) is list:
-        world_size = max(world_size)
-
-    if world_size != total_files:
-        raise ValueError(
-            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
-            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
-        )
-
-    # the groups are named differently in each stage
-    if zero_stage <= 2:
-        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
-    elif zero_stage == 3:
-        fp32_groups_key = FP32_FLAT_GROUPS
-    else:
-        raise ValueError(f"unknown zero stage {zero_stage}")
-
-    if zero_stage <= 2:
-        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
-    elif zero_stage == 3:
-        # if there is more than one param group, there will be multiple flattened tensors - one
-        # flattened tensor per group - for simplicity merge them into a single tensor
-        #
-        # XXX: could make the script more memory efficient for when there are multiple groups - it
-        # will require matching the sub-lists of param_shapes for each param group flattened tensor
-
-        fp32_flat_groups = [
-            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
-        ]
-
-    return zero_stage, world_size, fp32_flat_groups
-
-
-def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
-    """
-    Returns fp32 state_dict reconstructed from ds checkpoint
-
-    Args:
-        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
-
-    """
-    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
-
-    optim_files = get_optim_files(ds_checkpoint_dir)
-    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
-    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
-
-    model_files = get_model_state_files(ds_checkpoint_dir)
-
-    zero_model_states = parse_model_states(model_files)
-    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
-
-    if zero_stage <= 2:
-        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                                          exclude_frozen_parameters)
-    elif zero_stage == 3:
-        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                                          exclude_frozen_parameters)
-
-
-def _zero2_merge_frozen_params(state_dict, zero_model_states):
-    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
-        return
-
-    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
-    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
-
-    if debug:
-        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
-        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
-
-        wanted_params = len(frozen_param_shapes)
-        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
-        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
-        print(f'Frozen params: Have {avail_numel} numels to process.')
-        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
-
-    total_params = 0
-    total_numel = 0
-    for name, shape in frozen_param_shapes.items():
-        total_params += 1
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-
-        state_dict[name] = frozen_param_fragments[name]
-
-        if debug:
-            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
-
-    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _has_callable(obj, fn):
-    attr = getattr(obj, fn, None)
-    return callable(attr)
-
-
-def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
-    param_shapes = zero_model_states[0].param_shapes
-
-    # Reconstruction protocol:
-    #
-    # XXX: document this
-
-    if debug:
-        for i in range(world_size):
-            for j in range(len(fp32_flat_groups[0])):
-                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
-
-    # XXX: memory usage doubles here (zero2)
-    num_param_groups = len(fp32_flat_groups[0])
-    merged_single_partition_of_fp32_groups = []
-    for i in range(num_param_groups):
-        merged_partitions = [sd[i] for sd in fp32_flat_groups]
-        full_single_fp32_vector = torch.cat(merged_partitions, 0)
-        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
-    avail_numel = sum(
-        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
-
-    if debug:
-        wanted_params = sum([len(shapes) for shapes in param_shapes])
-        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
-        # not asserting if there is a mismatch due to possible padding
-        print(f"Have {avail_numel} numels to process.")
-        print(f"Need {wanted_numel} numels in {wanted_params} params.")
-
-    # params
-    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
-    # out-of-core computing solution
-    total_numel = 0
-    total_params = 0
-    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
-        offset = 0
-        avail_numel = full_single_fp32_vector.numel()
-        for name, shape in shapes.items():
-
-            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
-            total_numel += unpartitioned_numel
-            total_params += 1
-
-            if debug:
-                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
-            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
-            offset += unpartitioned_numel
-
-        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
-        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
-        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
-        # live optimizer object, so we are checking that the numbers are within the right range
-        align_to = 2 * world_size
-
-        def zero2_align(x):
-            return align_to * math.ceil(x / align_to)
-
-        if debug:
-            print(f"original offset={offset}, avail_numel={avail_numel}")
-
-        offset = zero2_align(offset)
-        avail_numel = zero2_align(avail_numel)
-
-        if debug:
-            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
-
-        # Sanity check
-        if offset != avail_numel:
-            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
-
-    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                               exclude_frozen_parameters):
-    state_dict = OrderedDict()
-
-    # buffers
-    buffers = zero_model_states[0].buffers
-    state_dict.update(buffers)
-    if debug:
-        print(f"added {len(buffers)} buffers")
-
-    if not exclude_frozen_parameters:
-        _zero2_merge_frozen_params(state_dict, zero_model_states)
-
-    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
-
-    # recover shared parameters
-    for pair in zero_model_states[0].shared_params:
-        if pair[1] in state_dict:
-            state_dict[pair[0]] = state_dict[pair[1]]
-
-    return state_dict
-
-
-def zero3_partitioned_param_info(unpartitioned_numel, world_size):
-    remainder = unpartitioned_numel % world_size
-    padding_numel = (world_size - remainder) if remainder else 0
-    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
-    return partitioned_numel, padding_numel
-
-
-def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
-    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
-        return
-
-    if debug:
-        for i in range(world_size):
-            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
-            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
-
-        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
-        wanted_params = len(frozen_param_shapes)
-        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
-        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
-        print(f'Frozen params: Have {avail_numel} numels to process.')
-        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
-
-    total_params = 0
-    total_numel = 0
-    for name, shape in zero_model_states[0].frozen_param_shapes.items():
-        total_params += 1
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-
-        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
-        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
-
-        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
-
-        if debug:
-            print(
-                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
-            )
-
-    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
-    param_shapes = zero_model_states[0].param_shapes
-    avail_numel = fp32_flat_groups[0].numel() * world_size
-    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
-    # param, re-consolidating each param, while dealing with padding if any
-
-    # merge list of dicts, preserving order
-    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
-
-    if debug:
-        for i in range(world_size):
-            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
-
-        wanted_params = len(param_shapes)
-        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
-        # not asserting if there is a mismatch due to possible padding
-        avail_numel = fp32_flat_groups[0].numel() * world_size
-        print(f"Trainable params: Have {avail_numel} numels to process.")
-        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
-
-    # params
-    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
-    # out-of-core computing solution
-    offset = 0
-    total_numel = 0
-    total_params = 0
-    for name, shape in param_shapes.items():
-
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-        total_params += 1
-
-        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
-
-        if debug:
-            print(
-                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
-            )
-
-        # XXX: memory usage doubles here
-        state_dict[name] = torch.cat(
-            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
-            0).narrow(0, 0, unpartitioned_numel).view(shape)
-        offset += partitioned_numel
-
-    offset *= world_size
-
-    # Sanity check
-    if offset != avail_numel:
-        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
-
-    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                               exclude_frozen_parameters):
-    state_dict = OrderedDict()
-
-    # buffers
-    buffers = zero_model_states[0].buffers
-    state_dict.update(buffers)
-    if debug:
-        print(f"added {len(buffers)} buffers")
-
-    if not exclude_frozen_parameters:
-        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
-
-    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
-
-    # recover shared parameters
-    for pair in zero_model_states[0].shared_params:
-        if pair[1] in state_dict:
-            state_dict[pair[0]] = state_dict[pair[1]]
-
-    return state_dict
-
-
-def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
-    """
-    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
-    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
-    via a model hub.
-
-    Args:
-        - ``checkpoint_dir``: path to the desired checkpoint folder
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
-        - ``exclude_frozen_parameters``: exclude frozen parameters
-
-    Returns:
-        - pytorch ``state_dict``
-
-    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
-    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
-    the checkpoint.
-
-    A typical usage might be ::
-
-        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
-        # do the training and checkpoint saving
-        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
-        model = model.cpu() # move to cpu
-        model.load_state_dict(state_dict)
-        # submit to model hub or save the model to share with others
-
-    In this example the ``model`` will no longer be usable in the deepspeed context of the same
-    application. i.e. you will need to re-initialize the deepspeed engine, since
-    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
-
-    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
-
-    """
-    if tag is None:
-        latest_path = os.path.join(checkpoint_dir, 'latest')
-        if os.path.isfile(latest_path):
-            with open(latest_path, 'r') as fd:
-                tag = fd.read().strip()
-        else:
-            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
-
-    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
-
-    if not os.path.isdir(ds_checkpoint_dir):
-        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
-
-    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
-
-
-def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
-    """
-    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
-    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
-
-    Args:
-        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
-        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
-        - ``exclude_frozen_parameters``: exclude frozen parameters
-    """
-
-    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
-    print(f"Saving fp32 state dict to {output_file}")
-    torch.save(state_dict, output_file)
-
-
-def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
-    """
-    1. Put the provided model to cpu
-    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
-    3. Load it into the provided model
-
-    Args:
-        - ``model``: the model object to update
-        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
-
-    Returns:
-        - ``model`: modified model
-
-    Make sure you have plenty of CPU memory available before you call this function. If you don't
-    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
-    conveniently placed for you in the checkpoint folder.
-
-    A typical usage might be ::
-
-        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
-        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
-        # submit to model hub or save the model to share with others
-
-    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
-    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
-    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
-
-    """
-    logger.info(f"Extracting fp32 weights")
-    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
-
-    logger.info(f"Overwriting model with fp32 weights")
-    model = model.cpu()
-    model.load_state_dict(state_dict, strict=False)
-
-    return model
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("checkpoint_dir",
-                        type=str,
-                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
-    parser.add_argument(
-        "output_file",
-        type=str,
-        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
-    parser.add_argument("-t",
-                        "--tag",
-                        type=str,
-                        default=None,
-                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
-    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
-    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
-    args = parser.parse_args()
-
-    debug = args.debug
-
-    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
-                                               args.output_file,
-                                               tag=args.tag,
-                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

sft/1M3/Full_remoe/checkpoint-20679/added_tokens.json

@@ -1,13 +0,0 @@
-{
-  "<|assistant|>": 32001,
-  "<|endoftext|>": 32000,
-  "<|end|>": 32007,
-  "<|placeholder1|>": 32002,
-  "<|placeholder2|>": 32003,
-  "<|placeholder3|>": 32004,
-  "<|placeholder4|>": 32005,
-  "<|placeholder5|>": 32008,
-  "<|placeholder6|>": 32009,
-  "<|system|>": 32006,
-  "<|user|>": 32010
-}

sft/1M3/Full_remoe/checkpoint-20679/config.json

@@ -1,203 +0,0 @@
-{
-  "_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft",
-  "architectures": [
-    "LlavaPhiForCausalLM"
-  ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi3.Phi3Config",
-    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
-  },
-  "bal_comp_loss_coef": 0.01,
-  "balance_loss_coef": 1.5e-323,
-  "bos_token_id": 1,
-  "clip_smoe": true,
-  "diversity_loss_coef": 0.01,
-  "dropout": false,
-  "e_loss_coef": 0.001,
-  "embd_pdrop": 0.0,
-  "entropy_advance_loss": false,
-  "eos_token_id": 32000,
-  "freeze_backbone": false,
-  "freeze_mm_mlp_adapter": false,
-  "hidden_act": "silu",
-  "hidden_size": 3072,
-  "hybrid": false,
-  "image_aspect_ratio": "pad",
-  "init_weight": true,
-  "initializer_range": 0.02,
-  "intermediate_size": 8192,
-  "is_cosine": false,
-  "is_norm_weight": false,
-  "local_rank": 0,
-  "loss1": "balanceloss",
-  "loss2": "zloss",
-  "luna": false,
-  "max_compete_in_iter": 3,
-  "max_position_embeddings": 131072,
-  "mlp_smoe": true,
-  "mm_hidden_size": 1152,
-  "mm_patch_merge_type": "flat",
-  "mm_projector_lr": null,
-  "mm_projector_type": "moe",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-224",
-  "model_name_or_path": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft",
-  "model_type": "llava_phi",
-  "moe_name": "remoe",
-  "moe_relu_l1_reg_coeff_multiplier": 1.2,
-  "mp_pixel_shuffle_factor": 1,
-  "norm_softmax": false,
-  "normalization": true,
-  "num_attention_heads": 32,
-  "num_experts": 4,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 32,
-  "num_layers": 3,
-  "num_selected": 2,
-  "number_of_previous_tokens": 2,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "pretrain_mm_mlp_adapter": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/mm_projector.bin",
-  "rate_compete": 0.2,
-  "rate_flip": 0.05,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": {
-    "long_factor": [
-      1.0800000429153442,
-      1.1100000143051147,
-      1.1399999856948853,
-      1.340000033378601,
-      1.5899999141693115,
-      1.600000023841858,
-      1.6200000047683716,
-      2.620000123977661,
-      3.2300000190734863,
-      3.2300000190734863,
-      4.789999961853027,
-      7.400000095367432,
-      7.700000286102295,
-      9.09000015258789,
-      12.199999809265137,
-      17.670000076293945,
-      24.46000099182129,
-      28.57000160217285,
-      30.420001983642578,
-      30.840002059936523,
-      32.590003967285156,
-      32.93000411987305,
-      42.320003509521484,
-      44.96000289916992,
-      50.340003967285156,
-      50.45000457763672,
-      57.55000305175781,
-      57.93000411987305,
-      58.21000289916992,
-      60.1400032043457,
-      62.61000442504883,
-      62.62000274658203,
-      62.71000289916992,
-      63.1400032043457,
-      63.1400032043457,
-      63.77000427246094,
-      63.93000411987305,
-      63.96000289916992,
-      63.970001220703125,
-      64.02999877929688,
-      64.06999969482422,
-      64.08000183105469,
-      64.12000274658203,
-      64.41000366210938,
-      64.4800033569336,
-      64.51000213623047,
-      64.52999877929688,
-      64.83999633789062
-    ],
-    "short_factor": [
-      1.0,
-      1.0199999809265137,
-      1.0299999713897705,
-      1.0299999713897705,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0699999332427979,
-      1.0999999046325684,
-      1.1099998950958252,
-      1.1599998474121094,
-      1.1599998474121094,
-      1.1699998378753662,
-      1.2899998426437378,
-      1.339999794960022,
-      1.679999828338623,
-      1.7899998426437378,
-      1.8199998140335083,
-      1.8499997854232788,
-      1.8799997568130493,
-      1.9099997282028198,
-      1.9399996995925903,
-      1.9899996519088745,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0199997425079346,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0299997329711914,
-      2.0799996852874756,
-      2.0899996757507324,
-      2.189999580383301,
-      2.2199995517730713,
-      2.5899994373321533,
-      2.729999542236328,
-      2.749999523162842,
-      2.8399994373321533
-    ],
-    "type": "longrope"
-  },
-  "rope_theta": 10000.0,
-  "router_loss_coef": 0.01,
-  "router_theta": 0.1,
-  "router_z_loss_coef": 0.001,
-  "scales": [
-    1,
-    3
-  ],
-  "sliding_window": 262144,
-  "sparse_upcycling": true,
-  "std_gate": 0.02,
-  "strategy_train": "base",
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 2048,
-  "tokenizer_padding_side": "right",
-  "topk_max": 2,
-  "topk_min": 1,
-  "torch_dtype": "bfloat16",
-  "training": true,
-  "transformers_version": "4.43.0",
-  "tune_mm_mlp_adapter": false,
-  "unit_test": true,
-  "use_cache": false,
-  "use_mm_proj": true,
-  "use_old": false,
-  "version": "phi35",
-  "vision_tower": "google/siglip-so400m-patch14-224",
-  "vision_tower_dir": "/cm/archive/namnv78/checkpoints/phi35-siglip224/pft/clip.bin",
-  "vocab_size": 32064,
-  "warm_up": 0.05
-}