Add files using upload-large-folder tool

.gitattributes

@@ -61,3 +61,4 @@ llava-ov-lora/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 internvl3-8b-instruct-lora_epoch10_5e-6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 qwen2.5vl-7b-qvq_thinking_full_v2/v0-20250823-125422/checkpoint-280/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 ood/qwen2.5vl-7b-thinking_full_v3_ood_wd001_e10-checkpoint-228/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/tokenizer.json filter=lfs diff=lfs merge=lfs -text

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6120fda4d24a89d0c6727f21dbb5c7d912949ff73fec9215b417d888088ea61
+size 4991123960

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741ec72cbb1ca29f64e84aa34c896bddf28ba23f1aa7823ec58044987fe77ea8
+size 4958443072

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e708763ce6944322a4c0c839cb165c40e99d2aa5f7307fc090c97d221673813
+size 4796984024

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5f2ddcefe8097dd39857e3e07157c35edde43d603301f4223590892315a11fa
+size 1142280864

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c830b56eaefdd9b372fae4488e1ba93148ecf58aa2ed05c8103ab7afe964ee
+size 15365

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6f3949e2b25ad05905da16bc2d58aeadc9a9f1db0b0e97a04ceb63467acff1d
+size 15429

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c3812c5e7f13c9d32f97d83cd625428061b1c9486e869caffd453d83ac07ceb
+size 15429

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eda756b33768639c1e1932ae00c5d43269de5705c09162bc09acdf71d3533c0
+size 15429

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384610a93cdb0dbf9f7ae8a31314cca71b4e2080e5883139a239afbb81a12680
+size 1465

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9ba4b4a6625b5047a1356f6081b641c3e4e6a4a198facbd4bef217747d1685
+size 11423548

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed47cc7ce6963bb954a625c84c64cbecc03cd4252442ae976f419c066288820
+size 9105

ood/ivl-8b-instruct-full_sft_ood/v0-20251004-170240/checkpoint-228/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/args.json

@@ -0,0 +1,385 @@
+{
+  "output_dir": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 2,
+  "per_device_eval_batch_size": 2,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 2e-06,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 5.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.1,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": 10,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": null,
+  "dataloader_num_workers": 4,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": true,
+  "metric_for_best_model": "eval_loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "swanlab"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": null,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "metric": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "use_flash_ckpt": false,
+  "model": "/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct",
+  "model_type": "internvl3",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "internvl2_5",
+  "system": null,
+  "max_length": 32768,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "/mnt/data/users/liamding/data/3AM_Plus/final/training/qvq-thinking_answer/ambi_normal_train_thinking_772.json",
+    "/mnt/data/users/liamding/data/3AM_Plus/final/training/qvq-thinking_answer/ambi_normal_aug_575.json"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.1,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "lazy_tokenize": true,
+  "cached_dataset": [],
+  "custom_register_path": [],
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [
+    "vision_model",
+    "mlp1"
+  ],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "swanlab_lark_webhook_url": null,
+  "swanlab_lark_secret": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "early_stop_interval": 200,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "InternVL3-8B-Instruct",
+  "model_info": "ModelInfo(model_type='internvl3', model_dir='/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='internvl3', model_groups=[ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-Pretrained', hf_model_id='OpenGVLab/InternVL3-1B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-Pretrained', hf_model_id='OpenGVLab/InternVL3-2B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-Pretrained', hf_model_id='OpenGVLab/InternVL3-8B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-Pretrained', hf_model_id='OpenGVLab/InternVL3-9B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-Pretrained', hf_model_id='OpenGVLab/InternVL3-14B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-Pretrained', hf_model_id='OpenGVLab/InternVL3-38B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-Pretrained', hf_model_id='OpenGVLab/InternVL3-78B-Pretrained', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-Instruct', hf_model_id='OpenGVLab/InternVL3-1B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-Instruct', hf_model_id='OpenGVLab/InternVL3-2B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-Instruct', hf_model_id='OpenGVLab/InternVL3-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-Instruct', hf_model_id='OpenGVLab/InternVL3-9B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-Instruct', hf_model_id='OpenGVLab/InternVL3-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-Instruct', hf_model_id='OpenGVLab/InternVL3-38B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-Instruct', hf_model_id='OpenGVLab/InternVL3-78B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B', hf_model_id='OpenGVLab/InternVL3-1B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B', hf_model_id='OpenGVLab/InternVL3-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B', hf_model_id='OpenGVLab/InternVL3-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B', hf_model_id='OpenGVLab/InternVL3-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B', hf_model_id='OpenGVLab/InternVL3-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B', hf_model_id='OpenGVLab/InternVL3-38B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B', hf_model_id='OpenGVLab/InternVL3-78B', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-AWQ', hf_model_id='OpenGVLab/InternVL3-1B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-AWQ', hf_model_id='OpenGVLab/InternVL3-2B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-AWQ', hf_model_id='OpenGVLab/InternVL3-8B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-AWQ', hf_model_id='OpenGVLab/InternVL3-9B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-AWQ', hf_model_id='OpenGVLab/InternVL3-14B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-AWQ', hf_model_id='OpenGVLab/InternVL3-38B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-AWQ', hf_model_id='OpenGVLab/InternVL3-78B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='internvl2_5', get_function=<function get_model_tokenizer_internvl at 0x7f4e7f5f2950>, model_arch=MultiModelKeys(arch_name='internvl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['language_model'], aligner=['mlp1'], vision_tower=['vision_model'], generator=[]), architectures=['InternVLChatModel'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37.2', 'timm'], tags=['vision', 'video'])",
+  "model_dir": "/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=2e-06, weight_decay=0.01, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.1, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=10, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=None, dataloader_num_workers=4, dataloader_prefetch_factor=10, past_index=-1, run_name='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=True, metric_for_best_model='eval_loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['swanlab'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/added_tokens.json

@@ -0,0 +1,33 @@
+{
+  "</box>": 151673,
+  "</img>": 151666,
+  "</quad>": 151669,
+  "</ref>": 151671,
+  "</tool_call>": 151658,
+  "<IMG_CONTEXT>": 151667,
+  "<box>": 151672,
+  "<img>": 151665,
+  "<quad>": 151668,
+  "<ref>": 151670,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/args.json

@@ -0,0 +1,385 @@
+{
+  "output_dir": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 2,
+  "per_device_eval_batch_size": 2,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 2e-06,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 5.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.1,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": 10,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": null,
+  "dataloader_num_workers": 4,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": true,
+  "metric_for_best_model": "eval_loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "swanlab"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": null,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "metric": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "use_flash_ckpt": false,
+  "model": "/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct",
+  "model_type": "internvl3",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "internvl2_5",
+  "system": null,
+  "max_length": 32768,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "/mnt/data/users/liamding/data/3AM_Plus/final/training/qvq-thinking_answer/ambi_normal_train_thinking_772.json",
+    "/mnt/data/users/liamding/data/3AM_Plus/final/training/qvq-thinking_answer/ambi_normal_aug_575.json"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.1,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "lazy_tokenize": true,
+  "cached_dataset": [],
+  "custom_register_path": [],
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [
+    "vision_model",
+    "mlp1"
+  ],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639",
+  "swanlab_lark_webhook_url": null,
+  "swanlab_lark_secret": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "early_stop_interval": 200,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "InternVL3-8B-Instruct",
+  "model_info": "ModelInfo(model_type='internvl3', model_dir='/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct', torch_dtype=torch.bfloat16, max_model_len=32768, quant_method=None, quant_bits=None, rope_scaling={'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='internvl3', model_groups=[ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-Pretrained', hf_model_id='OpenGVLab/InternVL3-1B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-Pretrained', hf_model_id='OpenGVLab/InternVL3-2B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-Pretrained', hf_model_id='OpenGVLab/InternVL3-8B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-Pretrained', hf_model_id='OpenGVLab/InternVL3-9B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-Pretrained', hf_model_id='OpenGVLab/InternVL3-14B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-Pretrained', hf_model_id='OpenGVLab/InternVL3-38B-Pretrained', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-Pretrained', hf_model_id='OpenGVLab/InternVL3-78B-Pretrained', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-Instruct', hf_model_id='OpenGVLab/InternVL3-1B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-Instruct', hf_model_id='OpenGVLab/InternVL3-2B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-Instruct', hf_model_id='OpenGVLab/InternVL3-8B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-Instruct', hf_model_id='OpenGVLab/InternVL3-9B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-Instruct', hf_model_id='OpenGVLab/InternVL3-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-Instruct', hf_model_id='OpenGVLab/InternVL3-38B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-Instruct', hf_model_id='OpenGVLab/InternVL3-78B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B', hf_model_id='OpenGVLab/InternVL3-1B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B', hf_model_id='OpenGVLab/InternVL3-2B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B', hf_model_id='OpenGVLab/InternVL3-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B', hf_model_id='OpenGVLab/InternVL3-9B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B', hf_model_id='OpenGVLab/InternVL3-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B', hf_model_id='OpenGVLab/InternVL3-38B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B', hf_model_id='OpenGVLab/InternVL3-78B', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='OpenGVLab/InternVL3-1B-AWQ', hf_model_id='OpenGVLab/InternVL3-1B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-2B-AWQ', hf_model_id='OpenGVLab/InternVL3-2B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-8B-AWQ', hf_model_id='OpenGVLab/InternVL3-8B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-9B-AWQ', hf_model_id='OpenGVLab/InternVL3-9B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-14B-AWQ', hf_model_id='OpenGVLab/InternVL3-14B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-38B-AWQ', hf_model_id='OpenGVLab/InternVL3-38B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='OpenGVLab/InternVL3-78B-AWQ', hf_model_id='OpenGVLab/InternVL3-78B-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='internvl2_5', get_function=<function get_model_tokenizer_internvl at 0x7f4e7f5f2950>, model_arch=MultiModelKeys(arch_name='internvl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['language_model'], aligner=['mlp1'], vision_tower=['vision_model'], generator=[]), architectures=['InternVLChatModel'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37.2', 'timm'], tags=['vision', 'video'])",
+  "model_dir": "/mnt/data/users/liamding/data/models/InternVL3-8B-Instruct",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=2e-06, weight_decay=0.01, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.1, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=10, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=None, dataloader_num_workers=4, dataloader_prefetch_factor=10, past_index=-1, run_name='/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=True, metric_for_best_model='eval_loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['swanlab'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/config.json

@@ -0,0 +1,144 @@
+{
+  "architectures": [
+    "InternVLChatModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
+    "AutoModel": "modeling_internvl_chat.InternVLChatModel",
+    "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
+  },
+  "downsample_ratio": 0.5,
+  "dynamic_image_size": true,
+  "force_image_size": 448,
+  "hidden_size": 3584,
+  "image_fold": null,
+  "keys_to_ignore_at_inference": [
+    "past_key_values"
+  ],
+  "llm_config": {
+    "_name_or_path": "./pretrained/Qwen2.5-32B-Instruct",
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151643,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 32768,
+    "max_window_layers": 70,
+    "model_type": "qwen2",
+    "moe_config": null,
+    "num_attention_heads": 28,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "factor": 2.0,
+      "rope_type": "dynamic",
+      "type": "dynamic"
+    },
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "use_bfloat16": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151674
+  },
+  "max_dynamic_patch": 12,
+  "min_dynamic_patch": 1,
+  "model_type": "internvl_chat",
+  "output_attentions": false,
+  "pad2square": false,
+  "pad_token_id": 151643,
+  "ps_version": "v2",
+  "select_layer": -1,
+  "template": "internvl2_5",
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": null,
+  "use_backbone_lora": 0,
+  "use_llm_lora": 0,
+  "use_thumbnail": true,
+  "vision_config": {
+    "_name_or_path": "OpenGVLab/InternViT-6B-448px-V1-5",
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_intern_vit.InternVisionConfig",
+      "AutoModel": "modeling_intern_vit.InternVisionModel"
+    },
+    "capacity_factor": 1.2,
+    "drop_path_rate": 0.0,
+    "dropout": 0.0,
+    "eval_capacity_factor": 1.4,
+    "hidden_act": "gelu",
+    "hidden_size": 1024,
+    "image_size": 448,
+    "initializer_factor": 0.1,
+    "initializer_range": 1e-10,
+    "intermediate_size": 4096,
+    "laux_allreduce": "all_nodes",
+    "layer_norm_eps": 1e-06,
+    "model_type": "intern_vit_6b",
+    "moe_coeff_ratio": 0.5,
+    "moe_intermediate_size": 768,
+    "moe_output_scale": 4.0,
+    "noisy_gate_policy": "RSample_before",
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_experts": 8,
+    "num_hidden_layers": 24,
+    "num_routed_experts": 4,
+    "num_shared_experts": 4,
+    "pad_token_id": 151643,
+    "patch_size": 14,
+    "qk_normalization": false,
+    "qkv_bias": true,
+    "shared_expert_intermediate_size": 3072,
+    "torch_dtype": "bfloat16",
+    "use_bfloat16": true,
+    "use_flash_attn": true,
+    "use_moe": false,
+    "use_residual": true,
+    "use_rts": false,
+    "use_weighted_residual": false
+  }
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/configuration_intern_vit.py

@@ -0,0 +1,120 @@
+# --------------------------------------------------------
+# InternVL
+# Copyright (c) 2024 OpenGVLab
+# Licensed under The MIT License [see LICENSE for details]
+# --------------------------------------------------------
+
+import os
+from typing import Union
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class InternVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to
+    instantiate a vision encoder according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of color channels in the input images (e.g., 3 for RGB).
+        patch_size (`int`, *optional*, defaults to 14):
+            The size (resolution) of each patch.
+        image_size (`int`, *optional*, defaults to 224):
+            The size (resolution) of each image.
+        qkv_bias (`bool`, *optional*, defaults to `False`):
+            Whether to add a bias to the queries and values in the self-attention layers.
+        hidden_size (`int`, *optional*, defaults to 3200):
+            Dimensionality of the encoder layers and the pooler layer.
+        num_attention_heads (`int`, *optional*, defaults to 25):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        intermediate_size (`int`, *optional*, defaults to 12800):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        qk_normalization (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the queries and keys in the self-attention layers.
+        num_hidden_layers (`int`, *optional*, defaults to 48):
+            Number of hidden layers in the Transformer encoder.
+        use_flash_attn (`bool`, *optional*, defaults to `True`):
+            Whether to use flash attention mechanism.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
+            The epsilon used by the layer normalization layers.
+        dropout (`float`, *optional*, defaults to 0.0):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        drop_path_rate (`float`, *optional*, defaults to 0.0):
+            Dropout rate for stochastic depth.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        initializer_factor (`float`, *optional*, defaults to 0.1):
+            A factor for layer scale.
+    """
+
+    model_type = 'intern_vit_6b'
+
+    def __init__(
+            self,
+            num_channels=3,
+            patch_size=14,
+            image_size=224,
+            qkv_bias=False,
+            hidden_size=3200,
+            num_attention_heads=25,
+            intermediate_size=12800,
+            qk_normalization=True,
+            num_hidden_layers=48,
+            use_flash_attn=True,
+            hidden_act='gelu',
+            norm_type='rms_norm',
+            layer_norm_eps=1e-6,
+            dropout=0.0,
+            drop_path_rate=0.0,
+            attention_dropout=0.0,
+            initializer_range=0.02,
+            initializer_factor=0.1,
+            **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.dropout = dropout
+        self.drop_path_rate = drop_path_rate
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.initializer_range = initializer_range
+        self.initializer_factor = initializer_factor
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.norm_type = norm_type
+        self.qkv_bias = qkv_bias
+        self.qk_normalization = qk_normalization
+        self.use_flash_attn = use_flash_attn
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig':
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        if 'vision_config' in config_dict:
+            config_dict = config_dict['vision_config']
+
+        if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
+            )
+
+        return cls.from_dict(config_dict, **kwargs)

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/configuration_internvl_chat.py

@@ -0,0 +1,97 @@
+# --------------------------------------------------------
+# InternVL
+# Copyright (c) 2024 OpenGVLab
+# Licensed under The MIT License [see LICENSE for details]
+# --------------------------------------------------------
+
+import copy
+
+from transformers import AutoConfig, LlamaConfig, Qwen2Config
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+from .configuration_intern_vit import InternVisionConfig
+
+logger = logging.get_logger(__name__)
+
+
+class InternVLChatConfig(PretrainedConfig):
+    model_type = 'internvl_chat'
+    is_composition = True
+
+    def __init__(
+            self,
+            vision_config=None,
+            llm_config=None,
+            use_backbone_lora=0,
+            use_llm_lora=0,
+            select_layer=-1,
+            force_image_size=None,
+            downsample_ratio=0.5,
+            template=None,
+            dynamic_image_size=False,
+            use_thumbnail=False,
+            ps_version='v1',
+            min_dynamic_patch=1,
+            max_dynamic_patch=6,
+            **kwargs):
+        super().__init__(**kwargs)
+
+        if vision_config is None:
+            vision_config = {'architectures': ['InternVisionModel']}
+            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')
+
+        if llm_config is None:
+            llm_config = {'architectures': ['Qwen2ForCausalLM']}
+            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')
+
+        self.vision_config = InternVisionConfig(**vision_config)
+        if llm_config.get('architectures')[0] == 'LlamaForCausalLM':
+            self.llm_config = LlamaConfig(**llm_config)
+        elif llm_config.get('architectures')[0] == 'Qwen2ForCausalLM':
+            self.llm_config = Qwen2Config(**llm_config)
+        else:
+            raise ValueError('Unsupported architecture: {}'.format(llm_config.get('architectures')[0]))
+        self.use_backbone_lora = use_backbone_lora
+        self.use_llm_lora = use_llm_lora
+        self.select_layer = select_layer
+        self.force_image_size = force_image_size
+        self.downsample_ratio = downsample_ratio
+        self.template = template
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.ps_version = ps_version  # pixel shuffle version
+        self.min_dynamic_patch = min_dynamic_patch
+        self.max_dynamic_patch = max_dynamic_patch
+        # By default, we use tie_word_embeddings=False for models of all sizes.
+        self.tie_word_embeddings = self.llm_config.tie_word_embeddings
+
+        logger.info(f'vision_select_layer: {self.select_layer}')
+        logger.info(f'ps_version: {self.ps_version}')
+        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
+        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = copy.deepcopy(self.__dict__)
+        output['vision_config'] = self.vision_config.to_dict()
+        output['llm_config'] = self.llm_config.to_dict()
+        output['model_type'] = self.__class__.model_type
+        output['use_backbone_lora'] = self.use_backbone_lora
+        output['use_llm_lora'] = self.use_llm_lora
+        output['select_layer'] = self.select_layer
+        output['force_image_size'] = self.force_image_size
+        output['downsample_ratio'] = self.downsample_ratio
+        output['template'] = self.template
+        output['dynamic_image_size'] = self.dynamic_image_size
+        output['use_thumbnail'] = self.use_thumbnail
+        output['ps_version'] = self.ps_version
+        output['min_dynamic_patch'] = self.min_dynamic_patch
+        output['max_dynamic_patch'] = self.max_dynamic_patch
+
+        return output

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/conversation.py

@@ -0,0 +1,391 @@
+"""
+Conversation prompt templates.
+
+We kindly request that you import fastchat instead of copying this file if you wish to use it.
+If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
+
+Modified from https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
+"""
+
+import dataclasses
+from enum import IntEnum, auto
+from typing import Dict, List, Tuple, Union
+
+
+class SeparatorStyle(IntEnum):
+    """Separator styles."""
+
+    ADD_COLON_SINGLE = auto()
+    ADD_COLON_TWO = auto()
+    ADD_COLON_SPACE_SINGLE = auto()
+    NO_COLON_SINGLE = auto()
+    NO_COLON_TWO = auto()
+    ADD_NEW_LINE_SINGLE = auto()
+    LLAMA2 = auto()
+    CHATGLM = auto()
+    CHATML = auto()
+    CHATINTERN = auto()
+    DOLLY = auto()
+    RWKV = auto()
+    PHOENIX = auto()
+    ROBIN = auto()
+    FALCON_CHAT = auto()
+    CHATGLM3 = auto()
+    INTERNVL_ZH = auto()
+    MPT = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that manages prompt templates and keeps all conversation history."""
+
+    # The name of this template
+    name: str
+    # The template of the system prompt
+    system_template: str = '{system_message}'
+    # The system message
+    system_message: str = ''
+    # The names of two roles
+    roles: Tuple[str] = ('USER', 'ASSISTANT')
+    # All messages. Each item is (role, message).
+    messages: List[List[str]] = ()
+    # The number of few shot examples
+    offset: int = 0
+    # The separator style and configurations
+    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
+    sep: str = '\n'
+    sep2: str = None
+    # Stop criteria (the default one is EOS token)
+    stop_str: Union[str, List[str]] = None
+    # Stops generation if meeting any token in this list
+    stop_token_ids: List[int] = None
+
+    def get_prompt(self) -> str:
+        """Get the prompt for generation."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ': ' + message + seps[i % 2]
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ': '  # must be end with a space
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
+            ret = '' if system_prompt == '' else system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + message + self.sep
+                else:
+                    ret += role + '\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
+            ret = system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + message + seps[i % 2]
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.RWKV:
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += (
+                        role
+                        + ': '
+                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
+                    )
+                    ret += '\n\n'
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.LLAMA2:
+            seps = [self.sep, self.sep2]
+            if self.system_message:
+                ret = system_prompt
+            else:
+                ret = '[INST] '
+            for i, (role, message) in enumerate(self.messages):
+                tag = self.roles[i % 2]
+                if message:
+                    if i == 0:
+                        ret += message + ' '
+                    else:
+                        ret += tag + ' ' + message + seps[i % 2]
+                else:
+                    ret += tag
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATGLM:
+            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
+            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
+            round_add_n = 1 if self.name == 'chatglm2' else 0
+            if system_prompt:
+                ret = system_prompt + self.sep
+            else:
+                ret = ''
+
+            for i, (role, message) in enumerate(self.messages):
+                if i % 2 == 0:
+                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
+
+                if message:
+                    ret += f'{role}：{message}{self.sep}'
+                else:
+                    ret += f'{role}：'
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATML:
+            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + message + self.sep + '\n'
+                else:
+                    ret += role + '\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATGLM3:
+            ret = ''
+            if self.system_message:
+                ret += system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + ' ' + message
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATINTERN:
+            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                # if i % 2 == 0:
+                #     ret += "<s>"
+                if message:
+                    ret += role + ':' + message + seps[i % 2] + '\n'
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.DOLLY:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ':\n' + message + seps[i % 2]
+                    if i % 2 == 1:
+                        ret += '\n\n'
+                else:
+                    ret += role + ':\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.PHOENIX:
+            ret = system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + '<s>' + message + '</s>'
+                else:
+                    ret += role + ': ' + '<s>'
+            return ret
+        elif self.sep_style == SeparatorStyle.ROBIN:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ':\n' + message + self.sep
+                else:
+                    ret += role + ':\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
+            ret = ''
+            if self.system_message:
+                ret += system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ':'
+
+            return ret
+        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
+            seps = [self.sep, self.sep2]
+            ret = self.system_message + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ': ' + message + seps[i % 2]
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        else:
+            raise ValueError(f'Invalid style: {self.sep_style}')
+
+    def set_system_message(self, system_message: str):
+        """Set the system message."""
+        self.system_message = system_message
+
+    def append_message(self, role: str, message: str):
+        """Append a new message."""
+        self.messages.append([role, message])
+
+    def update_last_message(self, message: str):
+        """Update the last output.
+
+        The last message is typically set to be None when constructing the prompt,
+        so we need to update it in-place after getting the response from a model.
+        """
+        self.messages[-1][1] = message
+
+    def to_gradio_chatbot(self):
+        """Convert the conversation to gradio chatbot format."""
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def to_openai_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        ret = [{'role': 'system', 'content': self.system_message}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({'role': 'user', 'content': msg})
+            else:
+                if msg is not None:
+                    ret.append({'role': 'assistant', 'content': msg})
+        return ret
+
+    def copy(self):
+        return Conversation(
+            name=self.name,
+            system_template=self.system_template,
+            system_message=self.system_message,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            stop_str=self.stop_str,
+            stop_token_ids=self.stop_token_ids,
+        )
+
+    def dict(self):
+        return {
+            'template_name': self.name,
+            'system_message': self.system_message,
+            'roles': self.roles,
+            'messages': self.messages,
+            'offset': self.offset,
+        }
+
+
+# A global registry for all conversation templates
+conv_templates: Dict[str, Conversation] = {}
+
+
+def register_conv_template(template: Conversation, override: bool = False):
+    """Register a new conversation template."""
+    if not override:
+        assert (
+            template.name not in conv_templates
+        ), f'{template.name} has been registered.'
+
+    conv_templates[template.name] = template
+
+
+def get_conv_template(name: str) -> Conversation:
+    """Get a conversation template."""
+    return conv_templates[name].copy()
+
+
+# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
+# is that during training, the preprocessing function for the Hermes-2 template doesn't add
+# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
+# Therefore, they are completely equivalent during inference.
+register_conv_template(
+    Conversation(
+        name='Hermes-2',
+        system_template='<|im_start|>system\n{system_message}',
+        # note: The new system prompt was not used here to avoid changes in benchmark performance.
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
+        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
+        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|im_end|>',
+        stop_str='<|endoftext|>',
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='internlm2-chat',
+        system_template='<|im_start|>system\n{system_message}',
+        # note: The new system prompt was not used here to avoid changes in benchmark performance.
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
+        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
+        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|im_end|>',
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='phi3-chat',
+        system_template='<|system|>\n{system_message}',
+        # note: The new system prompt was not used here to avoid changes in benchmark performance.
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
+        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
+        roles=('<|user|>\n', '<|assistant|>\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|end|>',
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='internvl2_5',
+        system_template='<|im_start|>system\n{system_message}',
+        system_message='你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
+        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|im_end|>\n',
+    )
+)

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 151645,
+  "transformers_version": "4.55.4"
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/latest

@@ -0,0 +1 @@
+global_step228

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fe38424d305b0e19c17661cbe165f0411248b1dd0c1bf3ff7e6aa9251287747
+size 4991123960

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da8427b89bf7fa298faaf21f388d4479c15ec668ee8900059c1d72e7ab3b9bc
+size 1142280864

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/model.safetensors.index.json

@@ -0,0 +1,693 @@
+{
+  "metadata": {
+    "total_parameters": 719360,
+    "total_size": 15888747520
+  },
+  "weight_map": {
+    "language_model.lm_head.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.norm.weight": "model-00003-of-00004.safetensors",
+    "mlp1.0.bias": "model-00004-of-00004.safetensors",
+    "mlp1.0.weight": "model-00004-of-00004.safetensors",
+    "mlp1.1.bias": "model-00004-of-00004.safetensors",
+    "mlp1.1.weight": "model-00004-of-00004.safetensors",
+    "mlp1.3.bias": "model-00004-of-00004.safetensors",
+    "mlp1.3.weight": "model-00004-of-00004.safetensors",
+    "vision_model.embeddings.class_embedding": "model-00001-of-00004.safetensors",
+    "vision_model.embeddings.patch_embedding.bias": "model-00001-of-00004.safetensors",
+    "vision_model.embeddings.patch_embedding.weight": "model-00001-of-00004.safetensors",
+    "vision_model.embeddings.position_embedding": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.0.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.1.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.10.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.11.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.12.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.13.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.14.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.15.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.16.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.17.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.18.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.19.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.2.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.20.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.21.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.22.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.23.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.3.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.4.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.5.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.6.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.7.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.8.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.ls1": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.ls2": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.encoder.layers.9.norm2.weight": "model-00001-of-00004.safetensors"
+  }
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/modeling_intern_vit.py

@@ -0,0 +1,431 @@
+# --------------------------------------------------------
+# InternVL
+# Copyright (c) 2024 OpenGVLab
+# Licensed under The MIT License [see LICENSE for details]
+# --------------------------------------------------------
+
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from einops import rearrange
+from timm.layers import DropPath
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import (BaseModelOutput,
+                                           BaseModelOutputWithPooling)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+from .configuration_intern_vit import InternVisionConfig
+
+try:
+    from flash_attn.bert_padding import pad_input, unpad_input
+    from flash_attn.flash_attn_interface import \
+        flash_attn_varlen_qkvpacked_func
+    has_flash_attn = True
+except:
+    print('FlashAttention2 is not installed.')
+    has_flash_attn = False
+
+logger = logging.get_logger(__name__)
+
+
+class FlashAttention(nn.Module):
+    """Implement the scaled dot product attention with softmax.
+    Arguments
+    ---------
+        softmax_scale: The temperature to use for the softmax attention.
+                      (default: 1/sqrt(d_keys) where d_keys is computed at
+                      runtime)
+        attention_dropout: The dropout rate to apply to the attention
+                           (default: 0.0)
+    """
+
+    def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
+        super().__init__()
+        self.softmax_scale = softmax_scale
+        self.dropout_p = attention_dropout
+
+    def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
+                max_s=None, need_weights=False):
+        """Implements the multihead softmax attention.
+        Arguments
+        ---------
+            qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
+                if unpadded: (nnz, 3, h, d)
+            key_padding_mask: a bool tensor of shape (B, S)
+        """
+        assert not need_weights
+        assert qkv.dtype in [torch.float16, torch.bfloat16]
+        assert qkv.is_cuda
+
+        if cu_seqlens is None:
+            batch_size = qkv.shape[0]
+            seqlen = qkv.shape[1]
+            if key_padding_mask is None:
+                qkv = rearrange(qkv, 'b s ... -> (b s) ...')
+                max_s = seqlen
+                cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
+                                          device=qkv.device)
+                output = flash_attn_varlen_qkvpacked_func(
+                    qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                    softmax_scale=self.softmax_scale, causal=causal
+                )
+                output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
+            else:
+                nheads = qkv.shape[-2]
+                x = rearrange(qkv, 'b s three h d -> b s (three h d)')
+                x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
+                x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
+                output_unpad = flash_attn_varlen_qkvpacked_func(
+                    x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                    softmax_scale=self.softmax_scale, causal=causal
+                )
+                output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
+                                             indices, batch_size, seqlen),
+                                   'b s (h d) -> b s h d', h=nheads)
+        else:
+            assert max_s is not None
+            output = flash_attn_varlen_qkvpacked_func(
+                qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
+                softmax_scale=self.softmax_scale, causal=causal
+            )
+
+        return output, None
+
+
+class InternRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+try:
+    from apex.normalization import FusedRMSNorm
+
+    InternRMSNorm = FusedRMSNorm  # noqa
+
+    logger.info('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm')
+except ImportError:
+    # using the normal InternRMSNorm
+    pass
+except Exception:
+    logger.warning('discovered apex but it failed to load, falling back to InternRMSNorm')
+    pass
+
+
+NORM2FN = {
+    'rms_norm': InternRMSNorm,
+    'layer_norm': nn.LayerNorm,
+}
+
+
+class InternVisionEmbeddings(nn.Module):
+    def __init__(self, config: InternVisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+
+        self.class_embedding = nn.Parameter(
+            torch.randn(1, 1, self.embed_dim),
+        )
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size
+        )
+
+        self.num_patches = (self.image_size // self.patch_size) ** 2
+        self.num_positions = self.num_patches + 1
+
+        self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim))
+
+    def _get_pos_embed(self, pos_embed, H, W):
+        target_dtype = pos_embed.dtype
+        pos_embed = pos_embed.float().reshape(
+            1, self.image_size // self.patch_size, self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
+        pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False). \
+            reshape(1, -1, H * W).permute(0, 2, 1).to(target_dtype)
+        return pos_embed
+
+    def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
+        target_dtype = self.patch_embedding.weight.dtype
+        patch_embeds = self.patch_embedding(pixel_values)  # shape = [*, channel, width, height]
+        batch_size, _, height, width = patch_embeds.shape
+        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
+        class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype)
+        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
+        position_embedding = torch.cat([
+            self.position_embedding[:, :1, :],
+            self._get_pos_embed(self.position_embedding[:, 1:, :], height, width)
+        ], dim=1)
+        embeddings = embeddings + position_embedding.to(target_dtype)
+        return embeddings
+
+
+class InternAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: InternVisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.use_flash_attn = config.use_flash_attn and has_flash_attn
+        if config.use_flash_attn and not has_flash_attn:
+            print('Warning: Flash Attention is not available, use_flash_attn is set to False.')
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:'
+                f' {self.num_heads}).'
+            )
+
+        self.scale = self.head_dim ** -0.5
+        self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=config.qkv_bias)
+        self.attn_drop = nn.Dropout(config.attention_dropout)
+        self.proj_drop = nn.Dropout(config.dropout)
+
+        self.qk_normalization = config.qk_normalization
+
+        if self.qk_normalization:
+            self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
+            self.k_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+        if self.use_flash_attn:
+            self.inner_attn = FlashAttention(attention_dropout=config.attention_dropout)
+        self.proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def _naive_attn(self, x):
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
+
+        if self.qk_normalization:
+            B_, H_, N_, D_ = q.shape
+            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
+
+        attn = ((q * self.scale) @ k.transpose(-2, -1))
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
+        qkv = self.qkv(x)
+        qkv = rearrange(qkv, 'b s (three h d) -> b s three h d', three=3, h=self.num_heads)
+
+        if self.qk_normalization:
+            q, k, v = qkv.unbind(2)
+            q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
+            k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
+            qkv = torch.stack([q, k, v], dim=2)
+
+        context, _ = self.inner_attn(
+            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=False
+        )
+        outs = self.proj(rearrange(context, 'b s h d -> b s (h d)'))
+        outs = self.proj_drop(outs)
+        return outs
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        x = self._naive_attn(hidden_states) if not self.use_flash_attn else self._flash_attn(hidden_states)
+        return x
+
+
+class InternMLP(nn.Module):
+    def __init__(self, config: InternVisionConfig):
+        super().__init__()
+        self.config = config
+        self.act = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class InternVisionEncoderLayer(nn.Module):
+    def __init__(self, config: InternVisionConfig, drop_path_rate: float):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.norm_type = config.norm_type
+
+        self.attn = InternAttention(config)
+        self.mlp = InternMLP(config)
+        self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
+        self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
+
+        self.ls1 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
+        self.ls2 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
+        self.drop_path1 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
+        self.drop_path2 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+    ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor], Optional[Tuple[torch.FloatTensor]]]:
+        """
+        Args:
+            hidden_states (`Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]`): input to the layer of shape `(batch, seq_len, embed_dim)`
+        """
+        hidden_states = hidden_states + self.drop_path1(self.attn(self.norm1(hidden_states).to(hidden_states.dtype)) * self.ls1)
+
+        hidden_states = hidden_states + self.drop_path2(self.mlp(self.norm2(hidden_states).to(hidden_states.dtype)) * self.ls2)
+
+        return hidden_states
+
+
+class InternVisionEncoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`InternEncoderLayer`].
+
+    Args:
+        config (`InternConfig`):
+            The corresponding vision configuration for the `InternEncoder`.
+    """
+
+    def __init__(self, config: InternVisionConfig):
+        super().__init__()
+        self.config = config
+        # stochastic depth decay rule
+        dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)]
+        self.layers = nn.ModuleList([
+            InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = True
+
+    def forward(
+            self,
+            inputs_embeds,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutput]:
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Embedded representation of the inputs. Should be float, not int tokens.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_states = () if output_hidden_states else None
+        hidden_states = inputs_embeds
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = torch.utils.checkpoint.checkpoint(
+                    encoder_layer,
+                    hidden_states)
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                )
+            hidden_states = layer_outputs
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states
+        )
+
+
+class InternVisionModel(PreTrainedModel):
+    main_input_name = 'pixel_values'
+    _supports_flash_attn_2 = True
+    supports_gradient_checkpointing = True
+    config_class = InternVisionConfig
+    _no_split_modules = ['InternVisionEncoderLayer']
+
+    def __init__(self, config: InternVisionConfig):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = InternVisionEmbeddings(config)
+        self.encoder = InternVisionEncoder(config)
+
+    def resize_pos_embeddings(self, old_size, new_size, patch_size):
+        pos_emb = self.embeddings.position_embedding
+        _, num_positions, embed_dim = pos_emb.shape
+        cls_emb = pos_emb[:, :1, :]
+        pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size, old_size // patch_size, -1).permute(0, 3, 1, 2)
+        pos_emb = F.interpolate(pos_emb.float(), size=new_size // patch_size, mode='bicubic', align_corners=False)
+        pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim, -1).permute(0, 2, 1)
+        pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
+        self.embeddings.position_embedding = nn.Parameter(pos_emb)
+        self.embeddings.image_size = new_size
+        logger.info('Resized position embeddings from {} to {}'.format(old_size, new_size))
+
+    def get_input_embeddings(self):
+        return self.embeddings
+
+    def forward(
+            self,
+            pixel_values: Optional[torch.FloatTensor] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+            pixel_embeds: Optional[torch.FloatTensor] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPooling]:
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if pixel_values is None and pixel_embeds is None:
+            raise ValueError('You have to specify pixel_values or pixel_embeds')
+
+        if pixel_embeds is not None:
+            hidden_states = pixel_embeds
+        else:
+            if len(pixel_values.shape) == 4:
+                hidden_states = self.embeddings(pixel_values)
+            else:
+                raise ValueError(f'wrong pixel_values size: {pixel_values.shape}')
+        encoder_outputs = self.encoder(
+            inputs_embeds=hidden_states,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        last_hidden_state = encoder_outputs.last_hidden_state
+        pooled_output = last_hidden_state[:, 0, :]
+
+        if not return_dict:
+            return (last_hidden_state, pooled_output) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=pooled_output,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/modeling_internvl_chat.py

@@ -0,0 +1,359 @@
+# --------------------------------------------------------
+# InternVL
+# Copyright (c) 2024 OpenGVLab
+# Licensed under The MIT License [see LICENSE for details]
+# --------------------------------------------------------
+
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import torch.utils.checkpoint
+import transformers
+from torch import nn
+from torch.nn import CrossEntropyLoss
+from transformers import (AutoModel, GenerationConfig, LlamaForCausalLM,
+                          Qwen2ForCausalLM)
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import ModelOutput, logging
+
+from .configuration_internvl_chat import InternVLChatConfig
+from .conversation import get_conv_template
+from .modeling_intern_vit import InternVisionModel, has_flash_attn
+
+logger = logging.get_logger(__name__)
+
+
+def version_cmp(v1, v2, op='eq'):
+    import operator
+
+    from packaging import version
+    op_func = getattr(operator, op)
+    return op_func(version.parse(v1), version.parse(v2))
+
+
+class InternVLChatModel(PreTrainedModel):
+    config_class = InternVLChatConfig
+    main_input_name = 'pixel_values'
+    base_model_prefix = 'language_model'
+    _supports_flash_attn_2 = True
+    supports_gradient_checkpointing = True
+    _no_split_modules = ['InternVisionModel', 'LlamaDecoderLayer', 'Qwen2DecoderLayer']
+
+    def __init__(self, config: InternVLChatConfig, vision_model=None, language_model=None, use_flash_attn=True):
+        super().__init__(config)
+
+        assert version_cmp(transformers.__version__, '4.37.0', 'ge')
+        image_size = config.force_image_size or config.vision_config.image_size
+        patch_size = config.vision_config.patch_size
+        self.patch_size = patch_size
+        self.select_layer = config.select_layer
+        self.template = config.template
+        self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
+        self.downsample_ratio = config.downsample_ratio
+        self.ps_version = config.ps_version
+        use_flash_attn = use_flash_attn if has_flash_attn else False
+        config.vision_config.use_flash_attn = True if use_flash_attn else False
+        config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
+
+        logger.info(f'num_image_token: {self.num_image_token}')
+        logger.info(f'ps_version: {self.ps_version}')
+        if vision_model is not None:
+            self.vision_model = vision_model
+        else:
+            self.vision_model = InternVisionModel(config.vision_config)
+        if language_model is not None:
+            self.language_model = language_model
+        else:
+            if config.llm_config.architectures[0] == 'LlamaForCausalLM':
+                self.language_model = LlamaForCausalLM(config.llm_config)
+            elif config.llm_config.architectures[0] == 'Qwen2ForCausalLM':
+                self.language_model = Qwen2ForCausalLM(config.llm_config)
+            else:
+                raise NotImplementedError(f'{config.llm_config.architectures[0]} is not implemented.')
+
+        vit_hidden_size = config.vision_config.hidden_size
+        llm_hidden_size = config.llm_config.hidden_size
+
+        self.mlp1 = nn.Sequential(
+            nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
+            nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size),
+            nn.GELU(),
+            nn.Linear(llm_hidden_size, llm_hidden_size)
+        )
+
+        self.img_context_token_id = None
+        self.conv_template = get_conv_template(self.template)
+        self.system_message = self.conv_template.system_message
+
+    def forward(
+            self,
+            pixel_values: torch.FloatTensor,
+            input_ids: torch.LongTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            image_flags: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[List[torch.FloatTensor]] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        image_flags = image_flags.squeeze(-1)
+        input_embeds = self.language_model.get_input_embeddings()(input_ids).clone()
+
+        vit_embeds = self.extract_feature(pixel_values)
+        vit_embeds = vit_embeds[image_flags == 1]
+        vit_batch_size = pixel_values.shape[0]
+
+        B, N, C = input_embeds.shape
+        input_embeds = input_embeds.reshape(B * N, C)
+
+        if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
+            print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}')
+
+        input_ids = input_ids.reshape(B * N)
+        selected = (input_ids == self.img_context_token_id)
+        try:
+            input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C)
+        except Exception as e:
+            vit_embeds = vit_embeds.reshape(-1, C)
+            print(f'warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, '
+                  f'vit_embeds.shape={vit_embeds.shape}')
+            n_token = min(selected.sum(), vit_embeds.size(0))
+            input_embeds[selected][:n_token] = input_embeds[selected][:n_token] * 0.0 + vit_embeds[:n_token]
+
+        input_embeds = input_embeds.reshape(B, N, C)
+
+        outputs = self.language_model(
+            inputs_embeds=input_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        logits = outputs.logits
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def pixel_shuffle(self, x, scale_factor=0.5):
+        n, w, h, c = x.size()
+        # N, W, H, C --> N, W, H * scale, C // scale
+        x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
+        # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
+        x = x.permute(0, 2, 1, 3).contiguous()
+        # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
+        x = x.view(n, int(h * scale_factor), int(w * scale_factor),
+                   int(c / (scale_factor * scale_factor)))
+        if self.ps_version == 'v1':
+            warnings.warn("In ps_version 'v1', the height and width have not been swapped back, "
+                          'which results in a transposed image.')
+        else:
+            x = x.permute(0, 2, 1, 3).contiguous()
+        return x
+
+    def extract_feature(self, pixel_values):
+        if self.select_layer == -1:
+            vit_embeds = self.vision_model(
+                pixel_values=pixel_values,
+                output_hidden_states=False,
+                return_dict=True).last_hidden_state
+        else:
+            vit_embeds = self.vision_model(
+                pixel_values=pixel_values,
+                output_hidden_states=True,
+                return_dict=True).hidden_states[self.select_layer]
+        vit_embeds = vit_embeds[:, 1:, :]
+
+        h = w = int(vit_embeds.shape[1] ** 0.5)
+        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
+        vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
+        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
+        vit_embeds = self.mlp1(vit_embeds)
+        return vit_embeds
+
+    def batch_chat(self, tokenizer, pixel_values, questions, generation_config, num_patches_list=None,
+                   history=None, return_history=False, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>',
+                   IMG_CONTEXT_TOKEN='<IMG_CONTEXT>', verbose=False, image_counts=None):
+        if history is not None or return_history:
+            print('Now multi-turn chat is not supported in batch_chat.')
+            raise NotImplementedError
+
+        if image_counts is not None:
+            num_patches_list = image_counts
+            print('Warning: `image_counts` is deprecated. Please use `num_patches_list` instead.')
+
+        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
+        self.img_context_token_id = img_context_token_id
+
+        if verbose and pixel_values is not None:
+            image_bs = pixel_values.shape[0]
+            print(f'dynamic ViT batch size: {image_bs}')
+
+        queries = []
+        for idx, num_patches in enumerate(num_patches_list):
+            question = questions[idx]
+            if pixel_values is not None and '<image>' not in question:
+                question = '<image>\n' + question
+            template = get_conv_template(self.template)
+            template.system_message = self.system_message
+            template.append_message(template.roles[0], question)
+            template.append_message(template.roles[1], None)
+            query = template.get_prompt()
+
+            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
+            query = query.replace('<image>', image_tokens, 1)
+            queries.append(query)
+
+        tokenizer.padding_side = 'left'
+        model_inputs = tokenizer(queries, return_tensors='pt', padding=True)
+        input_ids = model_inputs['input_ids'].to(self.device)
+        attention_mask = model_inputs['attention_mask'].to(self.device)
+        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip())
+        generation_config['eos_token_id'] = eos_token_id
+        generation_output = self.generate(
+            pixel_values=pixel_values,
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            **generation_config
+        )
+        responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
+        responses = [response.split(template.sep.strip())[0].strip() for response in responses]
+        return responses
+
+    def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
+             num_patches_list=None, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>', IMG_CONTEXT_TOKEN='<IMG_CONTEXT>',
+             verbose=False):
+
+        if history is None and pixel_values is not None and '<image>' not in question:
+            question = '<image>\n' + question
+
+        if num_patches_list is None:
+            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
+        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
+
+        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
+        self.img_context_token_id = img_context_token_id
+
+        template = get_conv_template(self.template)
+        template.system_message = self.system_message
+        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip())
+
+        history = [] if history is None else history
+        for (old_question, old_answer) in history:
+            template.append_message(template.roles[0], old_question)
+            template.append_message(template.roles[1], old_answer)
+        template.append_message(template.roles[0], question)
+        template.append_message(template.roles[1], None)
+        query = template.get_prompt()
+
+        if verbose and pixel_values is not None:
+            image_bs = pixel_values.shape[0]
+            print(f'dynamic ViT batch size: {image_bs}')
+
+        for num_patches in num_patches_list:
+            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
+            query = query.replace('<image>', image_tokens, 1)
+
+        model_inputs = tokenizer(query, return_tensors='pt')
+        input_ids = model_inputs['input_ids'].to(self.device)
+        attention_mask = model_inputs['attention_mask'].to(self.device)
+        generation_config['eos_token_id'] = eos_token_id
+        generation_output = self.generate(
+            pixel_values=pixel_values,
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            **generation_config
+        )
+        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
+        response = response.split(template.sep.strip())[0].strip()
+        history.append((question, response))
+        if return_history:
+            return response, history
+        else:
+            query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
+            query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
+            if verbose:
+                print(query_to_print, response)
+            return response
+
+    @torch.no_grad()
+    def generate(
+            self,
+            pixel_values: Optional[torch.FloatTensor] = None,
+            input_ids: Optional[torch.FloatTensor] = None,
+            attention_mask: Optional[torch.LongTensor] = None,
+            visual_features: Optional[torch.FloatTensor] = None,
+            generation_config: Optional[GenerationConfig] = None,
+            output_hidden_states: Optional[bool] = None,
+            **generate_kwargs,
+    ) -> torch.LongTensor:
+
+        assert self.img_context_token_id is not None
+        if pixel_values is not None:
+            if visual_features is not None:
+                vit_embeds = visual_features
+            else:
+                vit_embeds = self.extract_feature(pixel_values)
+            input_embeds = self.language_model.get_input_embeddings()(input_ids)
+            B, N, C = input_embeds.shape
+            input_embeds = input_embeds.reshape(B * N, C)
+
+            input_ids = input_ids.reshape(B * N)
+            selected = (input_ids == self.img_context_token_id)
+            assert selected.sum() != 0
+            input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
+
+            input_embeds = input_embeds.reshape(B, N, C)
+        else:
+            input_embeds = self.language_model.get_input_embeddings()(input_ids)
+
+        outputs = self.language_model.generate(
+            inputs_embeds=input_embeds,
+            attention_mask=attention_mask,
+            generation_config=generation_config,
+            output_hidden_states=output_hidden_states,
+            use_cache=True,
+            **generate_kwargs,
+        )
+
+        return outputs
+
+    @property
+    def lm_head(self):
+        return self.language_model.get_output_embeddings()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "crop_size": 448,
+  "do_center_crop": true,
+  "do_normalize": true,
+  "do_resize": true,
+  "feature_extractor_type": "CLIPFeatureExtractor",
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 3,
+  "size": 448
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c830b56eaefdd9b372fae4488e1ba93148ecf58aa2ed05c8103ab7afe964ee
+size 15365

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6f3949e2b25ad05905da16bc2d58aeadc9a9f1db0b0e97a04ceb63467acff1d
+size 15429

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c3812c5e7f13c9d32f97d83cd625428061b1c9486e869caffd453d83ac07ceb
+size 15429

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eda756b33768639c1e1932ae00c5d43269de5705c09162bc09acdf71d3533c0
+size 15429

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4501fff0ee5438d0b9846a6dc44b07779ec2c539bccf765f336f11aa4a06ce6
+size 1465

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/tokenizer_config.json

@@ -0,0 +1,280 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/trainer_state.json

@@ -0,0 +1,429 @@
+{
+  "best_global_step": 152,
+  "best_metric": 0.58733511,
+  "best_model_checkpoint": "/mnt/data/users/liamding/data/MMMT/lora/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-152",
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 228,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013157894736842105,
+      "grad_norm": 7.443980742894238,
+      "learning_rate": 5.2631578947368416e-08,
+      "loss": 1.2054955959320068,
+      "step": 1,
+      "token_acc": 0.6650124069478908
+    },
+    {
+      "epoch": 0.06578947368421052,
+      "grad_norm": 6.994858478446768,
+      "learning_rate": 2.631578947368421e-07,
+      "loss": 1.1846562623977661,
+      "step": 5,
+      "token_acc": 0.6773799979084603
+    },
+    {
+      "epoch": 0.13157894736842105,
+      "grad_norm": 6.882946793485346,
+      "learning_rate": 5.263157894736842e-07,
+      "loss": 1.1630187034606934,
+      "step": 10,
+      "token_acc": 0.6727064875678765
+    },
+    {
+      "epoch": 0.19736842105263158,
+      "grad_norm": 6.56262502850702,
+      "learning_rate": 7.894736842105263e-07,
+      "loss": 1.1065165519714355,
+      "step": 15,
+      "token_acc": 0.6860647322634679
+    },
+    {
+      "epoch": 0.2631578947368421,
+      "grad_norm": 4.349898766527815,
+      "learning_rate": 1.0526315789473683e-06,
+      "loss": 1.0518027305603028,
+      "step": 20,
+      "token_acc": 0.6943658580629627
+    },
+    {
+      "epoch": 0.32894736842105265,
+      "grad_norm": 3.911631629267191,
+      "learning_rate": 1.3157894736842106e-06,
+      "loss": 0.9281719207763672,
+      "step": 25,
+      "token_acc": 0.7247400800580286
+    },
+    {
+      "epoch": 0.39473684210526316,
+      "grad_norm": 4.172022146440079,
+      "learning_rate": 1.5789473684210526e-06,
+      "loss": 0.8204439163208008,
+      "step": 30,
+      "token_acc": 0.7546454605699139
+    },
+    {
+      "epoch": 0.4605263157894737,
+      "grad_norm": 3.089806220840059,
+      "learning_rate": 1.8421052631578946e-06,
+      "loss": 0.7725029945373535,
+      "step": 35,
+      "token_acc": 0.7605701730882589
+    },
+    {
+      "epoch": 0.5263157894736842,
+      "grad_norm": 3.573307688479329,
+      "learning_rate": 1.9998312416333223e-06,
+      "loss": 0.736858606338501,
+      "step": 40,
+      "token_acc": 0.7723412646246878
+    },
+    {
+      "epoch": 0.5921052631578947,
+      "grad_norm": 3.005279630810127,
+      "learning_rate": 1.9979333640833945e-06,
+      "loss": 0.7091458797454834,
+      "step": 45,
+      "token_acc": 0.7757977404962559
+    },
+    {
+      "epoch": 0.6578947368421053,
+      "grad_norm": 2.6547906270389987,
+      "learning_rate": 1.9939306773179494e-06,
+      "loss": 0.6816033363342285,
+      "step": 50,
+      "token_acc": 0.7845786750315021
+    },
+    {
+      "epoch": 0.7236842105263158,
+      "grad_norm": 2.6397159135636863,
+      "learning_rate": 1.9878316236762193e-06,
+      "loss": 0.6458231925964355,
+      "step": 55,
+      "token_acc": 0.7937998091455394
+    },
+    {
+      "epoch": 0.7894736842105263,
+      "grad_norm": 2.854823622950123,
+      "learning_rate": 1.9796490670875738e-06,
+      "loss": 0.6452323913574218,
+      "step": 60,
+      "token_acc": 0.7941120452752156
+    },
+    {
+      "epoch": 0.8552631578947368,
+      "grad_norm": 2.893500925424196,
+      "learning_rate": 1.9694002659393305e-06,
+      "loss": 0.6606711387634278,
+      "step": 65,
+      "token_acc": 0.7889678924563938
+    },
+    {
+      "epoch": 0.9210526315789473,
+      "grad_norm": 2.783788162077395,
+      "learning_rate": 1.957106836675914e-06,
+      "loss": 0.6494266986846924,
+      "step": 70,
+      "token_acc": 0.7904519555235935
+    },
+    {
+      "epoch": 0.9868421052631579,
+      "grad_norm": 2.7988099493453547,
+      "learning_rate": 1.942794708206143e-06,
+      "loss": 0.6436448574066163,
+      "step": 75,
+      "token_acc": 0.7935710698141637
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.6168130040168762,
+      "eval_runtime": 14.5426,
+      "eval_samples_per_second": 9.214,
+      "eval_steps_per_second": 1.169,
+      "eval_token_acc": 0.8025923865012602,
+      "step": 76
+    },
+    {
+      "epoch": 1.0526315789473684,
+      "grad_norm": 2.843676295839037,
+      "learning_rate": 1.9264940672148015e-06,
+      "loss": 0.5734551429748536,
+      "step": 80,
+      "token_acc": 0.8115053003533569
+    },
+    {
+      "epoch": 1.118421052631579,
+      "grad_norm": 2.7171783073156064,
+      "learning_rate": 1.9082392944938463e-06,
+      "loss": 0.5453813076019287,
+      "step": 85,
+      "token_acc": 0.817947946030269
+    },
+    {
+      "epoch": 1.1842105263157894,
+      "grad_norm": 3.0474411761804863,
+      "learning_rate": 1.8880688924275375e-06,
+      "loss": 0.5494410514831543,
+      "step": 90,
+      "token_acc": 0.8209292435277231
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 2.6264194208633915,
+      "learning_rate": 1.8660254037844386e-06,
+      "loss": 0.538231086730957,
+      "step": 95,
+      "token_acc": 0.8209094297428252
+    },
+    {
+      "epoch": 1.3157894736842106,
+      "grad_norm": 2.729094045019327,
+      "learning_rate": 1.8421553219875656e-06,
+      "loss": 0.553615951538086,
+      "step": 100,
+      "token_acc": 0.8173360210885409
+    },
+    {
+      "epoch": 1.381578947368421,
+      "grad_norm": 2.7672404883247057,
+      "learning_rate": 1.8165089930519428e-06,
+      "loss": 0.5331531047821045,
+      "step": 105,
+      "token_acc": 0.8239764441951767
+    },
+    {
+      "epoch": 1.4473684210526316,
+      "grad_norm": 2.7388761924928016,
+      "learning_rate": 1.7891405093963937e-06,
+      "loss": 0.5318719387054444,
+      "step": 110,
+      "token_acc": 0.8226405746841714
+    },
+    {
+      "epoch": 1.513157894736842,
+      "grad_norm": 2.5418644828797468,
+      "learning_rate": 1.7601075957535362e-06,
+      "loss": 0.5143476486206054,
+      "step": 115,
+      "token_acc": 0.8280721235194586
+    },
+    {
+      "epoch": 1.5789473684210527,
+      "grad_norm": 2.7544656818664928,
+      "learning_rate": 1.7294714874186208e-06,
+      "loss": 0.5286956787109375,
+      "step": 120,
+      "token_acc": 0.8235277770058083
+    },
+    {
+      "epoch": 1.6447368421052633,
+      "grad_norm": 2.818526777429581,
+      "learning_rate": 1.6972968010939952e-06,
+      "loss": 0.538817024230957,
+      "step": 125,
+      "token_acc": 0.8173395294801046
+    },
+    {
+      "epoch": 1.7105263157894737,
+      "grad_norm": 2.570309846472994,
+      "learning_rate": 1.6636513986016212e-06,
+      "loss": 0.5275390148162842,
+      "step": 130,
+      "token_acc": 0.8256479119108365
+    },
+    {
+      "epoch": 1.776315789473684,
+      "grad_norm": 2.642287833986887,
+      "learning_rate": 1.628606243751082e-06,
+      "loss": 0.5242605209350586,
+      "step": 135,
+      "token_acc": 0.8259167355941549
+    },
+    {
+      "epoch": 1.8421052631578947,
+      "grad_norm": 2.8193490230655005,
+      "learning_rate": 1.5922352526649801e-06,
+      "loss": 0.5326420783996582,
+      "step": 140,
+      "token_acc": 0.8207143059771353
+    },
+    {
+      "epoch": 1.9078947368421053,
+      "grad_norm": 2.829235622860951,
+      "learning_rate": 1.5546151378774087e-06,
+      "loss": 0.5403931617736817,
+      "step": 145,
+      "token_acc": 0.821919714937921
+    },
+    {
+      "epoch": 1.973684210526316,
+      "grad_norm": 2.5244815201810953,
+      "learning_rate": 1.515825246534324e-06,
+      "loss": 0.5291405200958252,
+      "step": 150,
+      "token_acc": 0.8255817110586191
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.5873351097106934,
+      "eval_runtime": 15.0089,
+      "eval_samples_per_second": 8.928,
+      "eval_steps_per_second": 1.133,
+      "eval_token_acc": 0.8101535137966024,
+      "step": 152
+    },
+    {
+      "epoch": 2.039473684210526,
+      "grad_norm": 2.634004731790707,
+      "learning_rate": 1.4759473930370736e-06,
+      "loss": 0.480256462097168,
+      "step": 155,
+      "token_acc": 0.8391755048235364
+    },
+    {
+      "epoch": 2.1052631578947367,
+      "grad_norm": 2.458163907263063,
+      "learning_rate": 1.4350656864820732e-06,
+      "loss": 0.4241488456726074,
+      "step": 160,
+      "token_acc": 0.8546429349009362
+    },
+    {
+      "epoch": 2.1710526315789473,
+      "grad_norm": 3.0072031878594743,
+      "learning_rate": 1.393266353260583e-06,
+      "loss": 0.4239327430725098,
+      "step": 165,
+      "token_acc": 0.8566516285211268
+    },
+    {
+      "epoch": 2.236842105263158,
+      "grad_norm": 2.765452730417083,
+      "learning_rate": 1.3506375551927544e-06,
+      "loss": 0.4281306266784668,
+      "step": 170,
+      "token_acc": 0.8572701074141996
+    },
+    {
+      "epoch": 2.3026315789473686,
+      "grad_norm": 2.851903657831883,
+      "learning_rate": 1.3072692035795304e-06,
+      "loss": 0.4317302703857422,
+      "step": 175,
+      "token_acc": 0.8539255373738526
+    },
+    {
+      "epoch": 2.3684210526315788,
+      "grad_norm": 2.9069316927957245,
+      "learning_rate": 1.263252769564599e-06,
+      "loss": 0.4294229507446289,
+      "step": 180,
+      "token_acc": 0.853015476639856
+    },
+    {
+      "epoch": 2.4342105263157894,
+      "grad_norm": 2.766993468219373,
+      "learning_rate": 1.2186810912063758e-06,
+      "loss": 0.4209444046020508,
+      "step": 185,
+      "token_acc": 0.8580880008502497
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 2.910279148073952,
+      "learning_rate": 1.1736481776669305e-06,
+      "loss": 0.4108576774597168,
+      "step": 190,
+      "token_acc": 0.8612809434526235
+    },
+    {
+      "epoch": 2.5657894736842106,
+      "grad_norm": 2.70203438317374,
+      "learning_rate": 1.1282490109308631e-06,
+      "loss": 0.42939186096191406,
+      "step": 195,
+      "token_acc": 0.8561881120597943
+    },
+    {
+      "epoch": 2.6315789473684212,
+      "grad_norm": 2.6721363594078316,
+      "learning_rate": 1.0825793454723324e-06,
+      "loss": 0.4140902042388916,
+      "step": 200,
+      "token_acc": 0.8595277649173453
+    },
+    {
+      "epoch": 2.6973684210526314,
+      "grad_norm": 2.685613098664299,
+      "learning_rate": 1.0367355062927725e-06,
+      "loss": 0.4143805503845215,
+      "step": 205,
+      "token_acc": 0.8593918191603875
+    },
+    {
+      "epoch": 2.763157894736842,
+      "grad_norm": 2.678658171344208,
+      "learning_rate": 9.908141857552737e-07,
+      "loss": 0.3974578380584717,
+      "step": 210,
+      "token_acc": 0.8657520695579737
+    },
+    {
+      "epoch": 2.8289473684210527,
+      "grad_norm": 2.8574376892263107,
+      "learning_rate": 9.449122396441343e-07,
+      "loss": 0.40328640937805177,
+      "step": 215,
+      "token_acc": 0.8625554895806825
+    },
+    {
+      "epoch": 2.8947368421052633,
+      "grad_norm": 2.6535340329998482,
+      "learning_rate": 8.991264828797318e-07,
+      "loss": 0.40736236572265627,
+      "step": 220,
+      "token_acc": 0.8601765750705256
+    },
+    {
+      "epoch": 2.9605263157894735,
+      "grad_norm": 2.7012120688258126,
+      "learning_rate": 8.535534853195784e-07,
+      "loss": 0.4065969467163086,
+      "step": 225,
+      "token_acc": 0.8599431898079141
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.5999683141708374,
+      "eval_runtime": 14.9348,
+      "eval_samples_per_second": 8.972,
+      "eval_steps_per_second": 1.138,
+      "eval_token_acc": 0.8112173087623973,
+      "step": 228
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 380,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6813882256601907e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/checkpoint-228/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

ood/ivl-8b-instruct-thinking_full_qvq_ood_e5/v0-20250928-190639/logging.jsonl

@@ -0,0 +1,82 @@
+{"loss": 1.2054956, "grad_norm": 7.44398074, "learning_rate": 5e-08, "token_acc": 0.66501241, "epoch": 0.01315789, "global_step/max_steps": "1/380", "percentage": "0.26%", "elapsed_time": "13s", "remaining_time": "1h 23m 7s", "memory(GiB)": 48.39, "train_speed(iter/s)": 0.075998}
+{"loss": 1.18465626, "grad_norm": 6.99485848, "learning_rate": 2.6e-07, "token_acc": 0.67738, "epoch": 0.06578947, "global_step/max_steps": "5/380", "percentage": "1.32%", "elapsed_time": "43s", "remaining_time": "54m 31s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.114616}
+{"loss": 1.1630187, "grad_norm": 6.88294679, "learning_rate": 5.3e-07, "token_acc": 0.67270649, "epoch": 0.13157895, "global_step/max_steps": "10/380", "percentage": "2.63%", "elapsed_time": "1m 15s", "remaining_time": "46m 24s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.132869}
+{"loss": 1.10651655, "grad_norm": 6.56262503, "learning_rate": 7.9e-07, "token_acc": 0.68606473, "epoch": 0.19736842, "global_step/max_steps": "15/380", "percentage": "3.95%", "elapsed_time": "1m 49s", "remaining_time": "44m 31s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.13664}
+{"loss": 1.05180273, "grad_norm": 4.34989877, "learning_rate": 1.05e-06, "token_acc": 0.69436586, "epoch": 0.26315789, "global_step/max_steps": "20/380", "percentage": "5.26%", "elapsed_time": "2m 22s", "remaining_time": "42m 40s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.140586}
+{"loss": 0.92817192, "grad_norm": 3.91163163, "learning_rate": 1.32e-06, "token_acc": 0.72474008, "epoch": 0.32894737, "global_step/max_steps": "25/380", "percentage": "6.58%", "elapsed_time": "2m 54s", "remaining_time": "41m 16s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.143333}
+{"loss": 0.82044392, "grad_norm": 4.17202215, "learning_rate": 1.58e-06, "token_acc": 0.75464546, "epoch": 0.39473684, "global_step/max_steps": "30/380", "percentage": "7.89%", "elapsed_time": "3m 28s", "remaining_time": "40m 28s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.144127}
+{"loss": 0.77250299, "grad_norm": 3.08980622, "learning_rate": 1.84e-06, "token_acc": 0.76057017, "epoch": 0.46052632, "global_step/max_steps": "35/380", "percentage": "9.21%", "elapsed_time": "4m 2s", "remaining_time": "39m 49s", "memory(GiB)": 74.81, "train_speed(iter/s)": 0.144385}
+{"loss": 0.73685861, "grad_norm": 3.57330769, "learning_rate": 2e-06, "token_acc": 0.77234126, "epoch": 0.52631579, "global_step/max_steps": "40/380", "percentage": "10.53%", "elapsed_time": "4m 36s", "remaining_time": "39m 6s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.144868}
+{"loss": 0.70914588, "grad_norm": 3.00527963, "learning_rate": 2e-06, "token_acc": 0.77579774, "epoch": 0.59210526, "global_step/max_steps": "45/380", "percentage": "11.84%", "elapsed_time": "5m 7s", "remaining_time": "38m 10s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.146257}
+{"loss": 0.68160334, "grad_norm": 2.65479063, "learning_rate": 1.99e-06, "token_acc": 0.78457868, "epoch": 0.65789474, "global_step/max_steps": "50/380", "percentage": "13.16%", "elapsed_time": "5m 37s", "remaining_time": "37m 6s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.14823}
+{"loss": 0.64582319, "grad_norm": 2.63971591, "learning_rate": 1.99e-06, "token_acc": 0.79379981, "epoch": 0.72368421, "global_step/max_steps": "55/380", "percentage": "14.47%", "elapsed_time": "6m 9s", "remaining_time": "36m 22s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.14893}
+{"loss": 0.64523239, "grad_norm": 2.85482362, "learning_rate": 1.98e-06, "token_acc": 0.79411205, "epoch": 0.78947368, "global_step/max_steps": "60/380", "percentage": "15.79%", "elapsed_time": "6m 39s", "remaining_time": "35m 33s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.150011}
+{"loss": 0.66067114, "grad_norm": 2.89350093, "learning_rate": 1.97e-06, "token_acc": 0.78896789, "epoch": 0.85526316, "global_step/max_steps": "65/380", "percentage": "17.11%", "elapsed_time": "7m 11s", "remaining_time": "34m 50s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.150698}
+{"loss": 0.6494267, "grad_norm": 2.78378816, "learning_rate": 1.96e-06, "token_acc": 0.79045196, "epoch": 0.92105263, "global_step/max_steps": "70/380", "percentage": "18.42%", "elapsed_time": "7m 41s", "remaining_time": "34m 4s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.151656}
+{"loss": 0.64364486, "grad_norm": 2.79880995, "learning_rate": 1.94e-06, "token_acc": 0.79357107, "epoch": 0.98684211, "global_step/max_steps": "75/380", "percentage": "19.74%", "elapsed_time": "8m 12s", "remaining_time": "33m 22s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.152293}
+{"eval_loss": 0.616813, "eval_runtime": 14.5426, "eval_samples_per_second": 9.214, "eval_steps_per_second": 1.169, "eval_token_acc": 0.80259239, "epoch": 1.0, "global_step/max_steps": "76/380", "percentage": "20.00%", "elapsed_time": "8m 32s", "remaining_time": "34m 8s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.148396}
+{"loss": 0.57345514, "grad_norm": 2.8436763, "learning_rate": 1.93e-06, "token_acc": 0.8115053, "epoch": 1.05263158, "global_step/max_steps": "80/380", "percentage": "21.05%", "elapsed_time": "10m 28s", "remaining_time": "39m 16s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.127324}
+{"loss": 0.54538131, "grad_norm": 2.71717831, "learning_rate": 1.91e-06, "token_acc": 0.81794795, "epoch": 1.11842105, "global_step/max_steps": "85/380", "percentage": "22.37%", "elapsed_time": "10m 59s", "remaining_time": "38m 10s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.128807}
+{"loss": 0.54944105, "grad_norm": 3.04744118, "learning_rate": 1.89e-06, "token_acc": 0.82092924, "epoch": 1.18421053, "global_step/max_steps": "90/380", "percentage": "23.68%", "elapsed_time": "11m 35s", "remaining_time": "37m 20s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.129454}
+{"loss": 0.53823109, "grad_norm": 2.62641942, "learning_rate": 1.87e-06, "token_acc": 0.82090943, "epoch": 1.25, "global_step/max_steps": "95/380", "percentage": "25.00%", "elapsed_time": "12m 7s", "remaining_time": "36m 21s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.13065}
+{"loss": 0.55361595, "grad_norm": 2.72909405, "learning_rate": 1.84e-06, "token_acc": 0.81733602, "epoch": 1.31578947, "global_step/max_steps": "100/380", "percentage": "26.32%", "elapsed_time": "12m 38s", "remaining_time": "35m 23s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.131865}
+{"loss": 0.5331531, "grad_norm": 2.76724049, "learning_rate": 1.82e-06, "token_acc": 0.82397644, "epoch": 1.38157895, "global_step/max_steps": "105/380", "percentage": "27.63%", "elapsed_time": "13m 10s", "remaining_time": "34m 29s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.132881}
+{"loss": 0.53187194, "grad_norm": 2.73887619, "learning_rate": 1.79e-06, "token_acc": 0.82264057, "epoch": 1.44736842, "global_step/max_steps": "110/380", "percentage": "28.95%", "elapsed_time": "13m 41s", "remaining_time": "33m 36s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.133877}
+{"loss": 0.51434765, "grad_norm": 2.54186448, "learning_rate": 1.76e-06, "token_acc": 0.82807212, "epoch": 1.51315789, "global_step/max_steps": "115/380", "percentage": "30.26%", "elapsed_time": "14m 14s", "remaining_time": "32m 48s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.134596}
+{"loss": 0.52869568, "grad_norm": 2.75446568, "learning_rate": 1.73e-06, "token_acc": 0.82352778, "epoch": 1.57894737, "global_step/max_steps": "120/380", "percentage": "31.58%", "elapsed_time": "14m 44s", "remaining_time": "31m 56s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.135635}
+{"loss": 0.53881702, "grad_norm": 2.81852678, "learning_rate": 1.7e-06, "token_acc": 0.81733953, "epoch": 1.64473684, "global_step/max_steps": "125/380", "percentage": "32.89%", "elapsed_time": "15m 15s", "remaining_time": "31m 8s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.136503}
+{"loss": 0.52753901, "grad_norm": 2.57030985, "learning_rate": 1.66e-06, "token_acc": 0.82564791, "epoch": 1.71052632, "global_step/max_steps": "130/380", "percentage": "34.21%", "elapsed_time": "15m 48s", "remaining_time": "30m 24s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.137018}
+{"loss": 0.52426052, "grad_norm": 2.64228783, "learning_rate": 1.63e-06, "token_acc": 0.82591674, "epoch": 1.77631579, "global_step/max_steps": "135/380", "percentage": "35.53%", "elapsed_time": "16m 19s", "remaining_time": "29m 37s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.137812}
+{"loss": 0.53264208, "grad_norm": 2.81934902, "learning_rate": 1.59e-06, "token_acc": 0.82071431, "epoch": 1.84210526, "global_step/max_steps": "140/380", "percentage": "36.84%", "elapsed_time": "16m 50s", "remaining_time": "28m 52s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.138497}
+{"loss": 0.54039316, "grad_norm": 2.82923562, "learning_rate": 1.55e-06, "token_acc": 0.82191971, "epoch": 1.90789474, "global_step/max_steps": "145/380", "percentage": "38.16%", "elapsed_time": "17m 22s", "remaining_time": "28m 9s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.139098}
+{"loss": 0.52914052, "grad_norm": 2.52448152, "learning_rate": 1.52e-06, "token_acc": 0.82558171, "epoch": 1.97368421, "global_step/max_steps": "150/380", "percentage": "39.47%", "elapsed_time": "17m 53s", "remaining_time": "27m 25s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.139766}
+{"eval_loss": 0.58733511, "eval_runtime": 15.0089, "eval_samples_per_second": 8.928, "eval_steps_per_second": 1.133, "eval_token_acc": 0.81015351, "epoch": 2.0, "global_step/max_steps": "152/380", "percentage": "40.00%", "elapsed_time": "18m 19s", "remaining_time": "27m 28s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.13829}
+{"loss": 0.48025646, "grad_norm": 2.63400473, "learning_rate": 1.48e-06, "token_acc": 0.8391755, "epoch": 2.03947368, "global_step/max_steps": "155/380", "percentage": "40.79%", "elapsed_time": "20m 16s", "remaining_time": "29m 25s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.127426}
+{"loss": 0.42414885, "grad_norm": 2.45816391, "learning_rate": 1.44e-06, "token_acc": 0.85464293, "epoch": 2.10526316, "global_step/max_steps": "160/380", "percentage": "42.11%", "elapsed_time": "20m 50s", "remaining_time": "28m 39s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.127973}
+{"loss": 0.42393274, "grad_norm": 3.00720319, "learning_rate": 1.39e-06, "token_acc": 0.85665163, "epoch": 2.17105263, "global_step/max_steps": "165/380", "percentage": "43.42%", "elapsed_time": "21m 22s", "remaining_time": "27m 50s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.128692}
+{"loss": 0.42813063, "grad_norm": 2.76545273, "learning_rate": 1.35e-06, "token_acc": 0.85727011, "epoch": 2.23684211, "global_step/max_steps": "170/380", "percentage": "44.74%", "elapsed_time": "21m 54s", "remaining_time": "27m 3s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.129335}
+{"loss": 0.43173027, "grad_norm": 2.85190366, "learning_rate": 1.31e-06, "token_acc": 0.85392554, "epoch": 2.30263158, "global_step/max_steps": "175/380", "percentage": "46.05%", "elapsed_time": "22m 26s", "remaining_time": "26m 17s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.129973}
+{"loss": 0.42942295, "grad_norm": 2.90693169, "learning_rate": 1.26e-06, "token_acc": 0.85301548, "epoch": 2.36842105, "global_step/max_steps": "180/380", "percentage": "47.37%", "elapsed_time": "22m 57s", "remaining_time": "25m 30s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.130671}
+{"loss": 0.4209444, "grad_norm": 2.76699347, "learning_rate": 1.22e-06, "token_acc": 0.858088, "epoch": 2.43421053, "global_step/max_steps": "185/380", "percentage": "48.68%", "elapsed_time": "23m 29s", "remaining_time": "24m 46s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.131218}
+{"loss": 0.41085768, "grad_norm": 2.91027915, "learning_rate": 1.17e-06, "token_acc": 0.86128094, "epoch": 2.5, "global_step/max_steps": "190/380", "percentage": "50.00%", "elapsed_time": "24m 2s", "remaining_time": "24m 2s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.131735}
+{"loss": 0.42939186, "grad_norm": 2.70203438, "learning_rate": 1.13e-06, "token_acc": 0.85618811, "epoch": 2.56578947, "global_step/max_steps": "195/380", "percentage": "51.32%", "elapsed_time": "24m 32s", "remaining_time": "23m 16s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.132443}
+{"loss": 0.4140902, "grad_norm": 2.67213636, "learning_rate": 1.08e-06, "token_acc": 0.85952776, "epoch": 2.63157895, "global_step/max_steps": "200/380", "percentage": "52.63%", "elapsed_time": "25m 4s", "remaining_time": "22m 33s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.132957}
+{"loss": 0.41438055, "grad_norm": 2.6856131, "learning_rate": 1.04e-06, "token_acc": 0.85939182, "epoch": 2.69736842, "global_step/max_steps": "205/380", "percentage": "53.95%", "elapsed_time": "25m 35s", "remaining_time": "21m 50s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.1335}
+{"loss": 0.39745784, "grad_norm": 2.67865817, "learning_rate": 9.9e-07, "token_acc": 0.86575207, "epoch": 2.76315789, "global_step/max_steps": "210/380", "percentage": "55.26%", "elapsed_time": "26m 8s", "remaining_time": "21m 9s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.133884}
+{"loss": 0.40328641, "grad_norm": 2.85743769, "learning_rate": 9.4e-07, "token_acc": 0.86255549, "epoch": 2.82894737, "global_step/max_steps": "215/380", "percentage": "56.58%", "elapsed_time": "26m 39s", "remaining_time": "20m 27s", "memory(GiB)": 76.97, "train_speed(iter/s)": 0.134413}
+{"loss": 0.40736237, "grad_norm": 2.65353403, "learning_rate": 9e-07, "token_acc": 0.86017658, "epoch": 2.89473684, "global_step/max_steps": "220/380", "percentage": "57.89%", "elapsed_time": "27m 11s", "remaining_time": "19m 46s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.134867}
+{"loss": 0.40659695, "grad_norm": 2.70121207, "learning_rate": 8.5e-07, "token_acc": 0.85994319, "epoch": 2.96052632, "global_step/max_steps": "225/380", "percentage": "59.21%", "elapsed_time": "27m 42s", "remaining_time": "19m 5s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.135358}
+{"eval_loss": 0.59996831, "eval_runtime": 14.9348, "eval_samples_per_second": 8.972, "eval_steps_per_second": 1.138, "eval_token_acc": 0.81121731, "epoch": 3.0, "global_step/max_steps": "228/380", "percentage": "60.00%", "elapsed_time": "28m 13s", "remaining_time": "18m 49s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.134632}
+{"loss": 0.37590837, "grad_norm": 2.34303951, "learning_rate": 8.1e-07, "token_acc": 0.87325984, "epoch": 3.02631579, "global_step/max_steps": "230/380", "percentage": "60.53%", "elapsed_time": "29m 59s", "remaining_time": "19m 33s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.127781}
+{"loss": 0.34411049, "grad_norm": 2.64090227, "learning_rate": 7.6e-07, "token_acc": 0.88515155, "epoch": 3.09210526, "global_step/max_steps": "235/380", "percentage": "61.84%", "elapsed_time": "30m 34s", "remaining_time": "18m 52s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.128083}
+{"loss": 0.33349562, "grad_norm": 2.73673123, "learning_rate": 7.2e-07, "token_acc": 0.88579021, "epoch": 3.15789474, "global_step/max_steps": "240/380", "percentage": "63.16%", "elapsed_time": "31m 6s", "remaining_time": "18m 8s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.128578}
+{"loss": 0.33648844, "grad_norm": 2.79852526, "learning_rate": 6.8e-07, "token_acc": 0.88734995, "epoch": 3.22368421, "global_step/max_steps": "245/380", "percentage": "64.47%", "elapsed_time": "31m 38s", "remaining_time": "17m 26s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.129036}
+{"loss": 0.32933283, "grad_norm": 2.83857994, "learning_rate": 6.3e-07, "token_acc": 0.88551507, "epoch": 3.28947368, "global_step/max_steps": "250/380", "percentage": "65.79%", "elapsed_time": "32m 12s", "remaining_time": "16m 44s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.129356}
+{"loss": 0.33368144, "grad_norm": 2.91740683, "learning_rate": 5.9e-07, "token_acc": 0.88633695, "epoch": 3.35526316, "global_step/max_steps": "255/380", "percentage": "67.11%", "elapsed_time": "32m 46s", "remaining_time": "16m 3s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.12967}
+{"loss": 0.31928144, "grad_norm": 2.71597386, "learning_rate": 5.5e-07, "token_acc": 0.89106513, "epoch": 3.42105263, "global_step/max_steps": "260/380", "percentage": "68.42%", "elapsed_time": "33m 19s", "remaining_time": "15m 22s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.130035}
+{"loss": 0.32702599, "grad_norm": 2.63773504, "learning_rate": 5.1e-07, "token_acc": 0.8884918, "epoch": 3.48684211, "global_step/max_steps": "265/380", "percentage": "69.74%", "elapsed_time": "33m 50s", "remaining_time": "14m 41s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.130517}
+{"loss": 0.33068037, "grad_norm": 2.77826519, "learning_rate": 4.7e-07, "token_acc": 0.88732108, "epoch": 3.55263158, "global_step/max_steps": "270/380", "percentage": "71.05%", "elapsed_time": "34m 21s", "remaining_time": "13m 59s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.130953}
+{"loss": 0.34255319, "grad_norm": 2.82606344, "learning_rate": 4.3e-07, "token_acc": 0.88418658, "epoch": 3.61842105, "global_step/max_steps": "275/380", "percentage": "72.37%", "elapsed_time": "34m 54s", "remaining_time": "13m 19s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131291}
+{"loss": 0.33419976, "grad_norm": 2.7130524, "learning_rate": 3.9e-07, "token_acc": 0.88693176, "epoch": 3.68421053, "global_step/max_steps": "280/380", "percentage": "73.68%", "elapsed_time": "35m 25s", "remaining_time": "12m 39s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131751}
+{"loss": 0.33861413, "grad_norm": 2.86742049, "learning_rate": 3.6e-07, "token_acc": 0.8844872, "epoch": 3.75, "global_step/max_steps": "285/380", "percentage": "75.00%", "elapsed_time": "35m 56s", "remaining_time": "11m 58s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.13216}
+{"loss": 0.34265239, "grad_norm": 2.61177458, "learning_rate": 3.2e-07, "token_acc": 0.88458715, "epoch": 3.81578947, "global_step/max_steps": "290/380", "percentage": "76.32%", "elapsed_time": "36m 28s", "remaining_time": "11m 19s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.132536}
+{"loss": 0.32698245, "grad_norm": 2.78414977, "learning_rate": 2.9e-07, "token_acc": 0.88858972, "epoch": 3.88157895, "global_step/max_steps": "295/380", "percentage": "77.63%", "elapsed_time": "36m 59s", "remaining_time": "10m 39s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.132938}
+{"loss": 0.33692307, "grad_norm": 2.49466861, "learning_rate": 2.6e-07, "token_acc": 0.88669291, "epoch": 3.94736842, "global_step/max_steps": "300/380", "percentage": "78.95%", "elapsed_time": "37m 29s", "remaining_time": "9m 59s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.133335}
+{"eval_loss": 0.62609965, "eval_runtime": 15.7575, "eval_samples_per_second": 8.504, "eval_steps_per_second": 1.079, "eval_token_acc": 0.81034991, "epoch": 4.0, "global_step/max_steps": "304/380", "percentage": "80.00%", "elapsed_time": "38m 12s", "remaining_time": "9m 33s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.132633}
+{"loss": 0.33888826, "grad_norm": 2.44170868, "learning_rate": 2.3e-07, "token_acc": 0.88497406, "epoch": 4.01315789, "global_step/max_steps": "305/380", "percentage": "80.26%", "elapsed_time": "39m 49s", "remaining_time": "9m 47s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.127645}
+{"loss": 0.30261137, "grad_norm": 2.3470474, "learning_rate": 2e-07, "token_acc": 0.89888967, "epoch": 4.07894737, "global_step/max_steps": "310/380", "percentage": "81.58%", "elapsed_time": "40m 25s", "remaining_time": "9m 7s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.12782}
+{"loss": 0.29790361, "grad_norm": 2.54135867, "learning_rate": 1.7e-07, "token_acc": 0.89942274, "epoch": 4.14473684, "global_step/max_steps": "315/380", "percentage": "82.89%", "elapsed_time": "41m 2s", "remaining_time": "8m 28s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.127939}
+{"loss": 0.29341311, "grad_norm": 2.71093939, "learning_rate": 1.5e-07, "token_acc": 0.90098437, "epoch": 4.21052632, "global_step/max_steps": "320/380", "percentage": "84.21%", "elapsed_time": "41m 35s", "remaining_time": "7m 47s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.128234}
+{"loss": 0.28581276, "grad_norm": 2.40393774, "learning_rate": 1.2e-07, "token_acc": 0.90298033, "epoch": 4.27631579, "global_step/max_steps": "325/380", "percentage": "85.53%", "elapsed_time": "42m 6s", "remaining_time": "7m 7s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.12862}
+{"loss": 0.29638333, "grad_norm": 2.73543103, "learning_rate": 1e-07, "token_acc": 0.90127643, "epoch": 4.34210526, "global_step/max_steps": "330/380", "percentage": "86.84%", "elapsed_time": "42m 38s", "remaining_time": "6m 27s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.128998}
+{"loss": 0.29314277, "grad_norm": 2.50176047, "learning_rate": 8e-08, "token_acc": 0.90104935, "epoch": 4.40789474, "global_step/max_steps": "335/380", "percentage": "88.16%", "elapsed_time": "43m 11s", "remaining_time": "5m 48s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.129292}
+{"loss": 0.29729128, "grad_norm": 2.60715813, "learning_rate": 7e-08, "token_acc": 0.89942937, "epoch": 4.47368421, "global_step/max_steps": "340/380", "percentage": "89.47%", "elapsed_time": "43m 41s", "remaining_time": "5m 8s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.129676}
+{"loss": 0.29389222, "grad_norm": 2.59512918, "learning_rate": 5e-08, "token_acc": 0.90163595, "epoch": 4.53947368, "global_step/max_steps": "345/380", "percentage": "90.79%", "elapsed_time": "44m 13s", "remaining_time": "4m 29s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.129997}
+{"loss": 0.3116612, "grad_norm": 2.61069145, "learning_rate": 4e-08, "token_acc": 0.89649329, "epoch": 4.60526316, "global_step/max_steps": "350/380", "percentage": "92.11%", "elapsed_time": "44m 44s", "remaining_time": "3m 50s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.13038}
+{"loss": 0.29344308, "grad_norm": 2.47391232, "learning_rate": 3e-08, "token_acc": 0.89970942, "epoch": 4.67105263, "global_step/max_steps": "355/380", "percentage": "93.42%", "elapsed_time": "45m 17s", "remaining_time": "3m 11s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.130626}
+{"loss": 0.28615444, "grad_norm": 2.60012451, "learning_rate": 2e-08, "token_acc": 0.90323878, "epoch": 4.73684211, "global_step/max_steps": "360/380", "percentage": "94.74%", "elapsed_time": "45m 50s", "remaining_time": "2m 32s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.130877}
+{"loss": 0.29576859, "grad_norm": 2.43552294, "learning_rate": 1e-08, "token_acc": 0.90143775, "epoch": 4.80263158, "global_step/max_steps": "365/380", "percentage": "96.05%", "elapsed_time": "46m 21s", "remaining_time": "1m 54s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131228}
+{"loss": 0.29920444, "grad_norm": 2.7377967, "learning_rate": 0.0, "token_acc": 0.90092493, "epoch": 4.86842105, "global_step/max_steps": "370/380", "percentage": "97.37%", "elapsed_time": "46m 53s", "remaining_time": "1m 16s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131529}
+{"loss": 0.30521078, "grad_norm": 2.77751398, "learning_rate": 0.0, "token_acc": 0.89559328, "epoch": 4.93421053, "global_step/max_steps": "375/380", "percentage": "98.68%", "elapsed_time": "47m 24s", "remaining_time": "37s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131811}
+{"loss": 0.3077724, "grad_norm": 3.11641431, "learning_rate": 0.0, "token_acc": 0.8958139, "epoch": 5.0, "global_step/max_steps": "380/380", "percentage": "100.00%", "elapsed_time": "47m 53s", "remaining_time": "0s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.132226}
+{"eval_loss": 0.64305353, "eval_runtime": 15.4821, "eval_samples_per_second": 8.655, "eval_steps_per_second": 1.098, "eval_token_acc": 0.80995712, "epoch": 5.0, "global_step/max_steps": "380/380", "percentage": "100.00%", "elapsed_time": "48m 9s", "remaining_time": "0s", "memory(GiB)": 77.29, "train_speed(iter/s)": 0.131517}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/args.json

@@ -0,0 +1,384 @@
+{
+  "output_dir": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 2,
+  "per_device_eval_batch_size": 2,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 5e-07,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 10.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.1,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": 5,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": null,
+  "dataloader_num_workers": 4,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": true,
+  "metric_for_best_model": "eval_loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "swanlab"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": null,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "metric": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "use_flash_ckpt": false,
+  "model": "/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct",
+  "model_type": "qwen2_5_vl",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5_vl",
+  "system": null,
+  "max_length": 32768,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "/mnt/data/users/liamding/data/3AM_Plus/final/ood_split/ambi_normal_train_aug_messages.json"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.1,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "lazy_tokenize": true,
+  "cached_dataset": [],
+  "custom_register_path": [],
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [
+    "model.visual",
+    "model.visual.merger"
+  ],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "swanlab_lark_webhook_url": null,
+  "swanlab_lark_secret": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "early_stop_interval": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "Qwen2.5-VL-7B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2_5_vl', model_dir='/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=128000, quant_method=None, quant_bits=None, rope_scaling={'type': 'default', 'mrope_section': [16, 24, 24], 'rope_type': 'default'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5_vl', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen2_5_vl', get_function=<function get_model_tokenizer_qwen2_5_vl at 0x7f64da1f1cf0>, model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen2_5_VLForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'], tags=['vision', 'video'])",
+  "model_dir": "/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=5e-07, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=10.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.1, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=5, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=None, dataloader_num_workers=4, dataloader_prefetch_factor=10, past_index=-1, run_name='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=True, metric_for_best_model='eval_loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['swanlab'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
+}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/args.json

@@ -0,0 +1,384 @@
+{
+  "output_dir": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 2,
+  "per_device_eval_batch_size": 2,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 5e-07,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 10.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.1,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": 5,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": null,
+  "dataloader_num_workers": 4,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": true,
+  "metric_for_best_model": "eval_loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "swanlab"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": null,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "metric": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "use_flash_ckpt": false,
+  "model": "/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct",
+  "model_type": "qwen2_5_vl",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5_vl",
+  "system": null,
+  "max_length": 32768,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "/mnt/data/users/liamding/data/3AM_Plus/final/ood_split/ambi_normal_train_aug_messages.json"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.1,
+  "dataset_num_proc": 1,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "lazy_tokenize": true,
+  "cached_dataset": [],
+  "custom_register_path": [],
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [
+    "model.visual",
+    "model.visual.merger"
+  ],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120",
+  "swanlab_lark_webhook_url": null,
+  "swanlab_lark_secret": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "early_stop_interval": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "Qwen2.5-VL-7B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2_5_vl', model_dir='/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=128000, quant_method=None, quant_bits=None, rope_scaling={'type': 'default', 'mrope_section': [16, 24, 24], 'rope_type': 'default'}, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5_vl', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen2_5_vl', get_function=<function get_model_tokenizer_qwen2_5_vl at 0x7f64da1f1cf0>, model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), architectures=['Qwen2_5_VLForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'], tags=['vision', 'video'])",
+  "model_dir": "/mnt/data/users/liamding/data/models/Qwen2.5-VL-7B-Instruct",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=5e-07, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=10.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.1, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=5, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=None, dataloader_num_workers=4, dataloader_prefetch_factor=10, past_index=-1, run_name='/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=True, metric_for_best_model='eval_loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['swanlab'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, train_type='full', local_repo_path=None, galore_config=None)"
+}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/config.json

@@ -0,0 +1,138 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 28,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "text_config": {
+    "architectures": [
+      "Qwen2_5_VLForConditionalGeneration"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151645,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "image_token_id": null,
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 128000,
+    "max_window_layers": 28,
+    "model_type": "qwen2_5_vl_text",
+    "num_attention_heads": 28,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "mrope_section": [
+        16,
+        24,
+        24
+      ],
+      "rope_type": "default",
+      "type": "default"
+    },
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16",
+    "use_cache": false,
+    "use_sliding_window": false,
+    "video_token_id": null,
+    "vision_end_token_id": 151653,
+    "vision_start_token_id": 151652,
+    "vision_token_id": 151654,
+    "vocab_size": 152064
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 3584,
+    "pad_token_id": 151643,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 152064
+}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.55.4"
+}

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304/latest

@@ -0,0 +1 @@
+global_step304

ood/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/logging.jsonl

@@ -0,0 +1,166 @@
+{"loss": 1.02941561, "grad_norm": 17.3159181, "learning_rate": 1e-08, "token_acc": 0.6969697, "epoch": 0.01315789, "global_step/max_steps": "1/760", "percentage": "0.13%", "elapsed_time": "12s", "remaining_time": "2h 40m 37s", "memory(GiB)": 50.34, "train_speed(iter/s)": 0.078752}
+{"loss": 0.90563303, "grad_norm": 16.54827763, "learning_rate": 3e-08, "token_acc": 0.75214724, "epoch": 0.06578947, "global_step/max_steps": "5/760", "percentage": "0.66%", "elapsed_time": "29s", "remaining_time": "1h 14m 5s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.169836}
+{"loss": 1.022719, "grad_norm": 15.42344434, "learning_rate": 7e-08, "token_acc": 0.72828096, "epoch": 0.13157895, "global_step/max_steps": "10/760", "percentage": "1.32%", "elapsed_time": "49s", "remaining_time": "1h 1m 48s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.20226}
+{"loss": 0.92649784, "grad_norm": 16.17717353, "learning_rate": 1e-07, "token_acc": 0.76236749, "epoch": 0.19736842, "global_step/max_steps": "15/760", "percentage": "1.97%", "elapsed_time": "1m 9s", "remaining_time": "57m 20s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.216527}
+{"loss": 0.94114761, "grad_norm": 16.91098064, "learning_rate": 1.3e-07, "token_acc": 0.74158326, "epoch": 0.26315789, "global_step/max_steps": "20/760", "percentage": "2.63%", "elapsed_time": "1m 29s", "remaining_time": "55m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.22377}
+{"loss": 0.897752, "grad_norm": 17.10921014, "learning_rate": 1.6e-07, "token_acc": 0.76012146, "epoch": 0.32894737, "global_step/max_steps": "25/760", "percentage": "3.29%", "elapsed_time": "1m 48s", "remaining_time": "53m 10s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.230368}
+{"loss": 0.9137682, "grad_norm": 17.10710801, "learning_rate": 2e-07, "token_acc": 0.74127126, "epoch": 0.39473684, "global_step/max_steps": "30/760", "percentage": "3.95%", "elapsed_time": "2m 8s", "remaining_time": "52m 12s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.233052}
+{"loss": 0.84548731, "grad_norm": 19.22796394, "learning_rate": 2.3e-07, "token_acc": 0.76445211, "epoch": 0.46052632, "global_step/max_steps": "35/760", "percentage": "4.61%", "elapsed_time": "2m 28s", "remaining_time": "51m 19s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.235414}
+{"loss": 0.78939257, "grad_norm": 16.20044305, "learning_rate": 2.6e-07, "token_acc": 0.76708633, "epoch": 0.52631579, "global_step/max_steps": "40/760", "percentage": "5.26%", "elapsed_time": "2m 48s", "remaining_time": "50m 40s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.236828}
+{"loss": 0.87271767, "grad_norm": 12.81356622, "learning_rate": 3e-07, "token_acc": 0.7574171, "epoch": 0.59210526, "global_step/max_steps": "45/760", "percentage": "5.92%", "elapsed_time": "3m 8s", "remaining_time": "50m 2s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.238149}
+{"loss": 0.86138668, "grad_norm": 23.73855973, "learning_rate": 3.3e-07, "token_acc": 0.76438849, "epoch": 0.65789474, "global_step/max_steps": "50/760", "percentage": "6.58%", "elapsed_time": "3m 28s", "remaining_time": "49m 26s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.239373}
+{"loss": 0.91995277, "grad_norm": 18.20488127, "learning_rate": 3.6e-07, "token_acc": 0.73500448, "epoch": 0.72368421, "global_step/max_steps": "55/760", "percentage": "7.24%", "elapsed_time": "3m 48s", "remaining_time": "48m 43s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.241188}
+{"loss": 0.81555462, "grad_norm": 16.53984575, "learning_rate": 3.9e-07, "token_acc": 0.76759966, "epoch": 0.78947368, "global_step/max_steps": "60/760", "percentage": "7.89%", "elapsed_time": "4m 7s", "remaining_time": "48m 12s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.241968}
+{"loss": 0.87123632, "grad_norm": 15.36130527, "learning_rate": 4.3e-07, "token_acc": 0.75, "epoch": 0.85526316, "global_step/max_steps": "65/760", "percentage": "8.55%", "elapsed_time": "4m 26s", "remaining_time": "47m 31s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.243726}
+{"loss": 0.78044271, "grad_norm": 14.5636902, "learning_rate": 4.6e-07, "token_acc": 0.78176796, "epoch": 0.92105263, "global_step/max_steps": "70/760", "percentage": "9.21%", "elapsed_time": "4m 45s", "remaining_time": "46m 54s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.245171}
+{"loss": 0.85664997, "grad_norm": 13.72730304, "learning_rate": 4.9e-07, "token_acc": 0.7480916, "epoch": 0.98684211, "global_step/max_steps": "75/760", "percentage": "9.87%", "elapsed_time": "5m 4s", "remaining_time": "46m 20s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.246347}
+{"eval_loss": 0.82916719, "eval_runtime": 10.7249, "eval_samples_per_second": 12.587, "eval_steps_per_second": 1.585, "eval_token_acc": 0.75931842, "epoch": 1.0, "global_step/max_steps": "76/760", "percentage": "10.00%", "elapsed_time": "5m 19s", "remaining_time": "47m 52s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.238079}
+{"loss": 0.84156723, "grad_norm": 13.78746715, "learning_rate": 5e-07, "token_acc": 0.75711575, "epoch": 1.05263158, "global_step/max_steps": "80/760", "percentage": "10.53%", "elapsed_time": "7m 18s", "remaining_time": "1h 2m 10s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182271}
+{"loss": 0.92336359, "grad_norm": 13.70548477, "learning_rate": 5e-07, "token_acc": 0.73505798, "epoch": 1.11842105, "global_step/max_steps": "85/760", "percentage": "11.18%", "elapsed_time": "7m 39s", "remaining_time": "1h 0m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185183}
+{"loss": 0.83351564, "grad_norm": 13.82552424, "learning_rate": 5e-07, "token_acc": 0.76068376, "epoch": 1.18421053, "global_step/max_steps": "90/760", "percentage": "11.84%", "elapsed_time": "8m 0s", "remaining_time": "59m 35s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187369}
+{"loss": 0.74494662, "grad_norm": 13.81295197, "learning_rate": 5e-07, "token_acc": 0.77972028, "epoch": 1.25, "global_step/max_steps": "95/760", "percentage": "12.50%", "elapsed_time": "8m 19s", "remaining_time": "58m 18s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190055}
+{"loss": 0.78937058, "grad_norm": 12.97976986, "learning_rate": 5e-07, "token_acc": 0.76831683, "epoch": 1.31578947, "global_step/max_steps": "100/760", "percentage": "13.16%", "elapsed_time": "8m 39s", "remaining_time": "57m 8s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.192501}
+{"loss": 0.78472328, "grad_norm": 11.91064209, "learning_rate": 5e-07, "token_acc": 0.76811594, "epoch": 1.38157895, "global_step/max_steps": "105/760", "percentage": "13.82%", "elapsed_time": "8m 59s", "remaining_time": "56m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.194687}
+{"loss": 0.73886271, "grad_norm": 15.20236435, "learning_rate": 5e-07, "token_acc": 0.77837838, "epoch": 1.44736842, "global_step/max_steps": "110/760", "percentage": "14.47%", "elapsed_time": "9m 19s", "remaining_time": "55m 3s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.196777}
+{"loss": 0.78518505, "grad_norm": 13.92003311, "learning_rate": 5e-07, "token_acc": 0.76475478, "epoch": 1.51315789, "global_step/max_steps": "115/760", "percentage": "15.13%", "elapsed_time": "9m 38s", "remaining_time": "54m 3s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.198856}
+{"loss": 0.76190891, "grad_norm": 12.38341881, "learning_rate": 4.9e-07, "token_acc": 0.76888489, "epoch": 1.57894737, "global_step/max_steps": "120/760", "percentage": "15.79%", "elapsed_time": "9m 57s", "remaining_time": "53m 7s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.200796}
+{"loss": 0.74719434, "grad_norm": 15.64056532, "learning_rate": 4.9e-07, "token_acc": 0.78358209, "epoch": 1.64473684, "global_step/max_steps": "125/760", "percentage": "16.45%", "elapsed_time": "10m 16s", "remaining_time": "52m 14s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.202603}
+{"loss": 0.79086919, "grad_norm": 13.49205976, "learning_rate": 4.9e-07, "token_acc": 0.78280961, "epoch": 1.71052632, "global_step/max_steps": "130/760", "percentage": "17.11%", "elapsed_time": "10m 36s", "remaining_time": "51m 23s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.204315}
+{"loss": 0.78124485, "grad_norm": 14.52953203, "learning_rate": 4.9e-07, "token_acc": 0.77809798, "epoch": 1.77631579, "global_step/max_steps": "135/760", "percentage": "17.76%", "elapsed_time": "10m 55s", "remaining_time": "50m 34s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.205987}
+{"loss": 0.76170378, "grad_norm": 13.77716196, "learning_rate": 4.9e-07, "token_acc": 0.78044444, "epoch": 1.84210526, "global_step/max_steps": "140/760", "percentage": "18.42%", "elapsed_time": "11m 15s", "remaining_time": "49m 53s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.207103}
+{"loss": 0.80421877, "grad_norm": 11.94680249, "learning_rate": 4.9e-07, "token_acc": 0.77209302, "epoch": 1.90789474, "global_step/max_steps": "145/760", "percentage": "19.08%", "elapsed_time": "11m 35s", "remaining_time": "49m 7s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.208619}
+{"loss": 0.78462038, "grad_norm": 14.27554967, "learning_rate": 4.9e-07, "token_acc": 0.78860294, "epoch": 1.97368421, "global_step/max_steps": "150/760", "percentage": "19.74%", "elapsed_time": "11m 54s", "remaining_time": "48m 25s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.20994}
+{"eval_loss": 0.79742974, "eval_runtime": 11.3855, "eval_samples_per_second": 11.857, "eval_steps_per_second": 1.493, "eval_token_acc": 0.76996805, "epoch": 2.0, "global_step/max_steps": "152/760", "percentage": "20.00%", "elapsed_time": "12m 13s", "remaining_time": "48m 54s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.207198}
+{"loss": 0.70446115, "grad_norm": 12.20828183, "learning_rate": 4.8e-07, "token_acc": 0.77549272, "epoch": 2.03947368, "global_step/max_steps": "155/760", "percentage": "20.39%", "elapsed_time": "14m 8s", "remaining_time": "55m 12s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182634}
+{"loss": 0.74434495, "grad_norm": 12.86816501, "learning_rate": 4.8e-07, "token_acc": 0.78288868, "epoch": 2.10526316, "global_step/max_steps": "160/760", "percentage": "21.05%", "elapsed_time": "14m 29s", "remaining_time": "54m 19s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184089}
+{"loss": 0.69011607, "grad_norm": 13.15553517, "learning_rate": 4.8e-07, "token_acc": 0.78975741, "epoch": 2.17105263, "global_step/max_steps": "165/760", "percentage": "21.71%", "elapsed_time": "14m 48s", "remaining_time": "53m 24s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185681}
+{"loss": 0.71562028, "grad_norm": 11.69440391, "learning_rate": 4.8e-07, "token_acc": 0.8040153, "epoch": 2.23684211, "global_step/max_steps": "170/760", "percentage": "22.37%", "elapsed_time": "15m 7s", "remaining_time": "52m 30s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187272}
+{"loss": 0.71393661, "grad_norm": 15.77797348, "learning_rate": 4.7e-07, "token_acc": 0.79822222, "epoch": 2.30263158, "global_step/max_steps": "175/760", "percentage": "23.03%", "elapsed_time": "15m 27s", "remaining_time": "51m 40s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188693}
+{"loss": 0.65476294, "grad_norm": 11.31427353, "learning_rate": 4.7e-07, "token_acc": 0.79822835, "epoch": 2.36842105, "global_step/max_steps": "180/760", "percentage": "23.68%", "elapsed_time": "15m 46s", "remaining_time": "50m 50s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190163}
+{"loss": 0.66260881, "grad_norm": 12.04881686, "learning_rate": 4.7e-07, "token_acc": 0.82744702, "epoch": 2.43421053, "global_step/max_steps": "185/760", "percentage": "24.34%", "elapsed_time": "16m 5s", "remaining_time": "50m 1s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.191577}
+{"loss": 0.77575922, "grad_norm": 13.59218371, "learning_rate": 4.7e-07, "token_acc": 0.7822374, "epoch": 2.5, "global_step/max_steps": "190/760", "percentage": "25.00%", "elapsed_time": "16m 25s", "remaining_time": "49m 15s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.192878}
+{"loss": 0.69286623, "grad_norm": 13.28972823, "learning_rate": 4.6e-07, "token_acc": 0.78975265, "epoch": 2.56578947, "global_step/max_steps": "195/760", "percentage": "25.66%", "elapsed_time": "16m 44s", "remaining_time": "48m 30s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.194104}
+{"loss": 0.6376442, "grad_norm": 16.08999576, "learning_rate": 4.6e-07, "token_acc": 0.81484876, "epoch": 2.63157895, "global_step/max_steps": "200/760", "percentage": "26.32%", "elapsed_time": "17m 3s", "remaining_time": "47m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.195422}
+{"loss": 0.68135052, "grad_norm": 14.32753154, "learning_rate": 4.6e-07, "token_acc": 0.79595427, "epoch": 2.69736842, "global_step/max_steps": "205/760", "percentage": "26.97%", "elapsed_time": "17m 22s", "remaining_time": "47m 3s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.196564}
+{"loss": 0.68764391, "grad_norm": 14.97718993, "learning_rate": 4.5e-07, "token_acc": 0.80453258, "epoch": 2.76315789, "global_step/max_steps": "210/760", "percentage": "27.63%", "elapsed_time": "17m 42s", "remaining_time": "46m 23s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.197612}
+{"loss": 0.7278162, "grad_norm": 13.09284339, "learning_rate": 4.5e-07, "token_acc": 0.781491, "epoch": 2.82894737, "global_step/max_steps": "215/760", "percentage": "28.29%", "elapsed_time": "18m 2s", "remaining_time": "45m 44s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.198614}
+{"loss": 0.73977337, "grad_norm": 14.42389646, "learning_rate": 4.5e-07, "token_acc": 0.78738116, "epoch": 2.89473684, "global_step/max_steps": "220/760", "percentage": "28.95%", "elapsed_time": "18m 21s", "remaining_time": "45m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.199684}
+{"loss": 0.74037175, "grad_norm": 14.00628898, "learning_rate": 4.4e-07, "token_acc": 0.77495463, "epoch": 2.96052632, "global_step/max_steps": "225/760", "percentage": "29.61%", "elapsed_time": "18m 40s", "remaining_time": "44m 25s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.200741}
+{"eval_loss": 0.78277975, "eval_runtime": 11.2217, "eval_samples_per_second": 12.03, "eval_steps_per_second": 1.515, "eval_token_acc": 0.7715655, "epoch": 3.0, "global_step/max_steps": "228/760", "percentage": "30.00%", "elapsed_time": "19m 3s", "remaining_time": "44m 29s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.19931}
+{"loss": 0.64765458, "grad_norm": 12.11796553, "learning_rate": 4.4e-07, "token_acc": 0.80246914, "epoch": 3.02631579, "global_step/max_steps": "230/760", "percentage": "30.26%", "elapsed_time": "21m 1s", "remaining_time": "48m 27s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182256}
+{"loss": 0.58833332, "grad_norm": 10.63458162, "learning_rate": 4.4e-07, "token_acc": 0.82829181, "epoch": 3.09210526, "global_step/max_steps": "235/760", "percentage": "30.92%", "elapsed_time": "21m 25s", "remaining_time": "47m 52s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182795}
+{"loss": 0.62636294, "grad_norm": 13.97241496, "learning_rate": 4.3e-07, "token_acc": 0.8032345, "epoch": 3.15789474, "global_step/max_steps": "240/760", "percentage": "31.58%", "elapsed_time": "21m 45s", "remaining_time": "47m 7s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183883}
+{"loss": 0.59209342, "grad_norm": 11.77574647, "learning_rate": 4.3e-07, "token_acc": 0.82899306, "epoch": 3.22368421, "global_step/max_steps": "245/760", "percentage": "32.24%", "elapsed_time": "22m 4s", "remaining_time": "46m 24s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184953}
+{"loss": 0.61780977, "grad_norm": 12.85864133, "learning_rate": 4.2e-07, "token_acc": 0.80993897, "epoch": 3.28947368, "global_step/max_steps": "250/760", "percentage": "32.89%", "elapsed_time": "22m 25s", "remaining_time": "45m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185746}
+{"loss": 0.62229028, "grad_norm": 11.99203912, "learning_rate": 4.2e-07, "token_acc": 0.81783681, "epoch": 3.35526316, "global_step/max_steps": "255/760", "percentage": "33.55%", "elapsed_time": "22m 45s", "remaining_time": "45m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186754}
+{"loss": 0.60951276, "grad_norm": 14.32377195, "learning_rate": 4.2e-07, "token_acc": 0.81622678, "epoch": 3.42105263, "global_step/max_steps": "260/760", "percentage": "34.21%", "elapsed_time": "23m 4s", "remaining_time": "44m 22s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187827}
+{"loss": 0.68783875, "grad_norm": 13.08060299, "learning_rate": 4.1e-07, "token_acc": 0.7913351, "epoch": 3.48684211, "global_step/max_steps": "265/760", "percentage": "34.87%", "elapsed_time": "23m 23s", "remaining_time": "43m 41s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188795}
+{"loss": 0.64720039, "grad_norm": 12.92149821, "learning_rate": 4.1e-07, "token_acc": 0.81897386, "epoch": 3.55263158, "global_step/max_steps": "270/760", "percentage": "35.53%", "elapsed_time": "23m 42s", "remaining_time": "43m 2s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.189755}
+{"loss": 0.58908606, "grad_norm": 12.37553316, "learning_rate": 4e-07, "token_acc": 0.81750466, "epoch": 3.61842105, "global_step/max_steps": "275/760", "percentage": "36.18%", "elapsed_time": "24m 1s", "remaining_time": "42m 22s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190778}
+{"loss": 0.66937218, "grad_norm": 13.12920206, "learning_rate": 4e-07, "token_acc": 0.81687612, "epoch": 3.68421053, "global_step/max_steps": "280/760", "percentage": "36.84%", "elapsed_time": "24m 20s", "remaining_time": "41m 43s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.191723}
+{"loss": 0.68171711, "grad_norm": 14.40747301, "learning_rate": 3.9e-07, "token_acc": 0.81320591, "epoch": 3.75, "global_step/max_steps": "285/760", "percentage": "37.50%", "elapsed_time": "24m 40s", "remaining_time": "41m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.192559}
+{"loss": 0.61392231, "grad_norm": 13.66264232, "learning_rate": 3.9e-07, "token_acc": 0.83055556, "epoch": 3.81578947, "global_step/max_steps": "290/760", "percentage": "38.16%", "elapsed_time": "24m 58s", "remaining_time": "40m 29s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.193474}
+{"loss": 0.5279911, "grad_norm": 11.47237183, "learning_rate": 3.8e-07, "token_acc": 0.8325667, "epoch": 3.88157895, "global_step/max_steps": "295/760", "percentage": "38.82%", "elapsed_time": "25m 18s", "remaining_time": "39m 53s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.194304}
+{"loss": 0.58143449, "grad_norm": 13.61124969, "learning_rate": 3.8e-07, "token_acc": 0.82510823, "epoch": 3.94736842, "global_step/max_steps": "300/760", "percentage": "39.47%", "elapsed_time": "25m 38s", "remaining_time": "39m 18s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.195026}
+{"eval_loss": 0.78271794, "eval_runtime": 11.4603, "eval_samples_per_second": 11.78, "eval_steps_per_second": 1.483, "eval_token_acc": 0.77582535, "epoch": 4.0, "global_step/max_steps": "304/760", "percentage": "40.00%", "elapsed_time": "26m 4s", "remaining_time": "39m 7s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.194284}
+{"loss": 0.49745297, "grad_norm": 12.23548274, "learning_rate": 3.7e-07, "token_acc": 0.85315315, "epoch": 4.01315789, "global_step/max_steps": "305/760", "percentage": "40.13%", "elapsed_time": "27m 57s", "remaining_time": "41m 42s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181799}
+{"loss": 0.52832932, "grad_norm": 11.58635468, "learning_rate": 3.7e-07, "token_acc": 0.83865248, "epoch": 4.07894737, "global_step/max_steps": "310/760", "percentage": "40.79%", "elapsed_time": "28m 19s", "remaining_time": "41m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182421}
+{"loss": 0.51454048, "grad_norm": 12.06320791, "learning_rate": 3.6e-07, "token_acc": 0.83317168, "epoch": 4.14473684, "global_step/max_steps": "315/760", "percentage": "41.45%", "elapsed_time": "28m 39s", "remaining_time": "40m 28s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183241}
+{"loss": 0.52731872, "grad_norm": 12.35438662, "learning_rate": 3.6e-07, "token_acc": 0.83301158, "epoch": 4.21052632, "global_step/max_steps": "320/760", "percentage": "42.11%", "elapsed_time": "28m 59s", "remaining_time": "39m 51s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184009}
+{"loss": 0.49958558, "grad_norm": 11.79500267, "learning_rate": 3.5e-07, "token_acc": 0.85548012, "epoch": 4.27631579, "global_step/max_steps": "325/760", "percentage": "42.76%", "elapsed_time": "29m 18s", "remaining_time": "39m 13s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184861}
+{"loss": 0.53519773, "grad_norm": 12.79872778, "learning_rate": 3.5e-07, "token_acc": 0.85056543, "epoch": 4.34210526, "global_step/max_steps": "330/760", "percentage": "43.42%", "elapsed_time": "29m 38s", "remaining_time": "38m 37s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185562}
+{"loss": 0.51555848, "grad_norm": 12.73069417, "learning_rate": 3.4e-07, "token_acc": 0.85452962, "epoch": 4.40789474, "global_step/max_steps": "335/760", "percentage": "44.08%", "elapsed_time": "29m 56s", "remaining_time": "37m 59s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186424}
+{"loss": 0.53922853, "grad_norm": 14.48429791, "learning_rate": 3.4e-07, "token_acc": 0.83053839, "epoch": 4.47368421, "global_step/max_steps": "340/760", "percentage": "44.74%", "elapsed_time": "30m 16s", "remaining_time": "37m 24s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187155}
+{"loss": 0.47321472, "grad_norm": 11.49103641, "learning_rate": 3.3e-07, "token_acc": 0.86215539, "epoch": 4.53947368, "global_step/max_steps": "345/760", "percentage": "45.39%", "elapsed_time": "30m 36s", "remaining_time": "36m 49s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187828}
+{"loss": 0.56955152, "grad_norm": 11.33748545, "learning_rate": 3.3e-07, "token_acc": 0.83121597, "epoch": 4.60526316, "global_step/max_steps": "350/760", "percentage": "46.05%", "elapsed_time": "30m 55s", "remaining_time": "36m 13s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188626}
+{"loss": 0.46773705, "grad_norm": 12.56964612, "learning_rate": 3.2e-07, "token_acc": 0.86182903, "epoch": 4.67105263, "global_step/max_steps": "355/760", "percentage": "46.71%", "elapsed_time": "31m 15s", "remaining_time": "35m 39s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.189326}
+{"loss": 0.55848804, "grad_norm": 15.81450431, "learning_rate": 3.2e-07, "token_acc": 0.838, "epoch": 4.73684211, "global_step/max_steps": "360/760", "percentage": "47.37%", "elapsed_time": "31m 34s", "remaining_time": "35m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190067}
+{"loss": 0.50942087, "grad_norm": 12.02623901, "learning_rate": 3.1e-07, "token_acc": 0.8604878, "epoch": 4.80263158, "global_step/max_steps": "365/760", "percentage": "48.03%", "elapsed_time": "31m 52s", "remaining_time": "34m 29s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190829}
+{"loss": 0.49252219, "grad_norm": 13.29445423, "learning_rate": 3e-07, "token_acc": 0.86591696, "epoch": 4.86842105, "global_step/max_steps": "370/760", "percentage": "48.68%", "elapsed_time": "32m 12s", "remaining_time": "33m 56s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.191494}
+{"loss": 0.54908113, "grad_norm": 12.42201337, "learning_rate": 3e-07, "token_acc": 0.84361037, "epoch": 4.93421053, "global_step/max_steps": "375/760", "percentage": "49.34%", "elapsed_time": "32m 31s", "remaining_time": "33m 23s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.192179}
+{"loss": 0.53364162, "grad_norm": 15.73394585, "learning_rate": 2.9e-07, "token_acc": 0.84782609, "epoch": 5.0, "global_step/max_steps": "380/760", "percentage": "50.00%", "elapsed_time": "32m 50s", "remaining_time": "32m 50s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.192865}
+{"eval_loss": 0.80552876, "eval_runtime": 11.9734, "eval_samples_per_second": 11.275, "eval_steps_per_second": 1.42, "eval_token_acc": 0.77316294, "epoch": 5.0, "global_step/max_steps": "380/760", "percentage": "50.00%", "elapsed_time": "33m 2s", "remaining_time": "33m 2s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.191698}
+{"loss": 0.40581384, "grad_norm": 11.57931837, "learning_rate": 2.9e-07, "token_acc": 0.88921002, "epoch": 5.06578947, "global_step/max_steps": "385/760", "percentage": "50.66%", "elapsed_time": "35m 13s", "remaining_time": "34m 18s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182185}
+{"loss": 0.4946116, "grad_norm": 13.46633141, "learning_rate": 2.8e-07, "token_acc": 0.85820204, "epoch": 5.13157895, "global_step/max_steps": "390/760", "percentage": "51.32%", "elapsed_time": "35m 32s", "remaining_time": "33m 43s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182884}
+{"loss": 0.44744673, "grad_norm": 12.70964931, "learning_rate": 2.8e-07, "token_acc": 0.85558853, "epoch": 5.19736842, "global_step/max_steps": "395/760", "percentage": "51.97%", "elapsed_time": "35m 52s", "remaining_time": "33m 9s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183508}
+{"loss": 0.39708123, "grad_norm": 10.08172646, "learning_rate": 2.7e-07, "token_acc": 0.87523105, "epoch": 5.26315789, "global_step/max_steps": "400/760", "percentage": "52.63%", "elapsed_time": "36m 13s", "remaining_time": "32m 35s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184054}
+{"loss": 0.44630985, "grad_norm": 13.12329007, "learning_rate": 2.6e-07, "token_acc": 0.86736475, "epoch": 5.32894737, "global_step/max_steps": "405/760", "percentage": "53.29%", "elapsed_time": "36m 32s", "remaining_time": "32m 1s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184738}
+{"loss": 0.44240837, "grad_norm": 12.55092676, "learning_rate": 2.6e-07, "token_acc": 0.8709369, "epoch": 5.39473684, "global_step/max_steps": "410/760", "percentage": "53.95%", "elapsed_time": "36m 52s", "remaining_time": "31m 28s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185309}
+{"loss": 0.41203389, "grad_norm": 12.15700096, "learning_rate": 2.5e-07, "token_acc": 0.88372093, "epoch": 5.46052632, "global_step/max_steps": "415/760", "percentage": "54.61%", "elapsed_time": "37m 11s", "remaining_time": "30m 55s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185936}
+{"loss": 0.39818454, "grad_norm": 11.79770449, "learning_rate": 2.5e-07, "token_acc": 0.87344029, "epoch": 5.52631579, "global_step/max_steps": "420/760", "percentage": "55.26%", "elapsed_time": "37m 30s", "remaining_time": "30m 21s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18662}
+{"loss": 0.45466576, "grad_norm": 12.74542293, "learning_rate": 2.4e-07, "token_acc": 0.87047101, "epoch": 5.59210526, "global_step/max_steps": "425/760", "percentage": "55.92%", "elapsed_time": "37m 50s", "remaining_time": "29m 49s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187161}
+{"loss": 0.45757322, "grad_norm": 13.32476096, "learning_rate": 2.4e-07, "token_acc": 0.87181904, "epoch": 5.65789474, "global_step/max_steps": "430/760", "percentage": "56.58%", "elapsed_time": "38m 9s", "remaining_time": "29m 17s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187815}
+{"loss": 0.45713186, "grad_norm": 12.04309303, "learning_rate": 2.3e-07, "token_acc": 0.8784965, "epoch": 5.72368421, "global_step/max_steps": "435/760", "percentage": "57.24%", "elapsed_time": "38m 29s", "remaining_time": "28m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188389}
+{"loss": 0.43521323, "grad_norm": 15.65857642, "learning_rate": 2.2e-07, "token_acc": 0.88613407, "epoch": 5.78947368, "global_step/max_steps": "440/760", "percentage": "57.89%", "elapsed_time": "38m 47s", "remaining_time": "28m 12s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.189029}
+{"loss": 0.49906354, "grad_norm": 11.14674096, "learning_rate": 2.2e-07, "token_acc": 0.86254296, "epoch": 5.85526316, "global_step/max_steps": "445/760", "percentage": "58.55%", "elapsed_time": "39m 6s", "remaining_time": "27m 41s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.189638}
+{"loss": 0.43380098, "grad_norm": 10.82007543, "learning_rate": 2.1e-07, "token_acc": 0.86727123, "epoch": 5.92105263, "global_step/max_steps": "450/760", "percentage": "59.21%", "elapsed_time": "39m 25s", "remaining_time": "27m 9s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.190252}
+{"loss": 0.37601833, "grad_norm": 10.22116608, "learning_rate": 2.1e-07, "token_acc": 0.89473684, "epoch": 5.98684211, "global_step/max_steps": "455/760", "percentage": "59.87%", "elapsed_time": "39m 43s", "remaining_time": "26m 37s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.19087}
+{"eval_loss": 0.8343057, "eval_runtime": 11.7159, "eval_samples_per_second": 11.523, "eval_steps_per_second": 1.451, "eval_token_acc": 0.77263046, "epoch": 6.0, "global_step/max_steps": "456/760", "percentage": "60.00%", "elapsed_time": "39m 59s", "remaining_time": "26m 39s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.19007}
+{"loss": 0.4425354, "grad_norm": 13.36737778, "learning_rate": 2e-07, "token_acc": 0.88156638, "epoch": 6.05263158, "global_step/max_steps": "460/760", "percentage": "60.53%", "elapsed_time": "42m 11s", "remaining_time": "27m 30s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181719}
+{"loss": 0.38719044, "grad_norm": 13.88518015, "learning_rate": 2e-07, "token_acc": 0.88651163, "epoch": 6.11842105, "global_step/max_steps": "465/760", "percentage": "61.18%", "elapsed_time": "42m 31s", "remaining_time": "26m 58s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18228}
+{"loss": 0.39121766, "grad_norm": 12.74952726, "learning_rate": 1.9e-07, "token_acc": 0.87396352, "epoch": 6.18421053, "global_step/max_steps": "470/760", "percentage": "61.84%", "elapsed_time": "42m 52s", "remaining_time": "26m 27s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18268}
+{"loss": 0.3415463, "grad_norm": 11.11411976, "learning_rate": 1.9e-07, "token_acc": 0.89875836, "epoch": 6.25, "global_step/max_steps": "475/760", "percentage": "62.50%", "elapsed_time": "43m 12s", "remaining_time": "25m 55s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183206}
+{"loss": 0.38265395, "grad_norm": 9.05417936, "learning_rate": 1.8e-07, "token_acc": 0.8994709, "epoch": 6.31578947, "global_step/max_steps": "480/760", "percentage": "63.16%", "elapsed_time": "43m 31s", "remaining_time": "25m 23s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183785}
+{"loss": 0.34448581, "grad_norm": 12.76382944, "learning_rate": 1.7e-07, "token_acc": 0.89911504, "epoch": 6.38157895, "global_step/max_steps": "485/760", "percentage": "63.82%", "elapsed_time": "43m 51s", "remaining_time": "24m 51s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18434}
+{"loss": 0.32860575, "grad_norm": 13.03501464, "learning_rate": 1.7e-07, "token_acc": 0.90650779, "epoch": 6.44736842, "global_step/max_steps": "490/760", "percentage": "64.47%", "elapsed_time": "44m 9s", "remaining_time": "24m 19s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184939}
+{"loss": 0.32854912, "grad_norm": 10.83126851, "learning_rate": 1.6e-07, "token_acc": 0.90980052, "epoch": 6.51315789, "global_step/max_steps": "495/760", "percentage": "65.13%", "elapsed_time": "44m 29s", "remaining_time": "23m 48s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18546}
+{"loss": 0.33108892, "grad_norm": 9.93697018, "learning_rate": 1.6e-07, "token_acc": 0.90366972, "epoch": 6.57894737, "global_step/max_steps": "500/760", "percentage": "65.79%", "elapsed_time": "44m 48s", "remaining_time": "23m 17s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185998}
+{"loss": 0.41943026, "grad_norm": 13.7162715, "learning_rate": 1.5e-07, "token_acc": 0.87477954, "epoch": 6.64473684, "global_step/max_steps": "505/760", "percentage": "66.45%", "elapsed_time": "45m 6s", "remaining_time": "22m 46s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18656}
+{"loss": 0.42898965, "grad_norm": 13.39988271, "learning_rate": 1.5e-07, "token_acc": 0.88362919, "epoch": 6.71052632, "global_step/max_steps": "510/760", "percentage": "67.11%", "elapsed_time": "45m 25s", "remaining_time": "22m 16s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187118}
+{"loss": 0.38254173, "grad_norm": 12.18274469, "learning_rate": 1.4e-07, "token_acc": 0.88393686, "epoch": 6.77631579, "global_step/max_steps": "515/760", "percentage": "67.76%", "elapsed_time": "45m 43s", "remaining_time": "21m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187696}
+{"loss": 0.34045658, "grad_norm": 9.70718661, "learning_rate": 1.4e-07, "token_acc": 0.89765101, "epoch": 6.84210526, "global_step/max_steps": "520/760", "percentage": "68.42%", "elapsed_time": "46m 3s", "remaining_time": "21m 15s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188197}
+{"loss": 0.35535841, "grad_norm": 11.16728077, "learning_rate": 1.3e-07, "token_acc": 0.89400922, "epoch": 6.90789474, "global_step/max_steps": "525/760", "percentage": "69.08%", "elapsed_time": "46m 21s", "remaining_time": "20m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188723}
+{"loss": 0.39788885, "grad_norm": 12.33942622, "learning_rate": 1.3e-07, "token_acc": 0.89342629, "epoch": 6.97368421, "global_step/max_steps": "530/760", "percentage": "69.74%", "elapsed_time": "46m 40s", "remaining_time": "20m 15s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.189226}
+{"eval_loss": 0.87638384, "eval_runtime": 11.7441, "eval_samples_per_second": 11.495, "eval_steps_per_second": 1.448, "eval_token_acc": 0.76943557, "epoch": 7.0, "global_step/max_steps": "532/760", "percentage": "70.00%", "elapsed_time": "47m 0s", "remaining_time": "20m 8s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188651}
+{"loss": 0.35242004, "grad_norm": 7.7182241, "learning_rate": 1.2e-07, "token_acc": 0.89974511, "epoch": 7.03947368, "global_step/max_steps": "535/760", "percentage": "70.39%", "elapsed_time": "49m 1s", "remaining_time": "20m 36s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181909}
+{"loss": 0.39486873, "grad_norm": 14.45331168, "learning_rate": 1.2e-07, "token_acc": 0.91481481, "epoch": 7.10526316, "global_step/max_steps": "540/760", "percentage": "71.05%", "elapsed_time": "49m 21s", "remaining_time": "20m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182363}
+{"loss": 0.34441402, "grad_norm": 11.37036945, "learning_rate": 1.1e-07, "token_acc": 0.89748549, "epoch": 7.17105263, "global_step/max_steps": "545/760", "percentage": "71.71%", "elapsed_time": "49m 39s", "remaining_time": "19m 35s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182904}
+{"loss": 0.32244649, "grad_norm": 9.90080573, "learning_rate": 1.1e-07, "token_acc": 0.90873016, "epoch": 7.23684211, "global_step/max_steps": "550/760", "percentage": "72.37%", "elapsed_time": "49m 59s", "remaining_time": "19m 5s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183391}
+{"loss": 0.33222942, "grad_norm": 14.14349121, "learning_rate": 1e-07, "token_acc": 0.90391791, "epoch": 7.30263158, "global_step/max_steps": "555/760", "percentage": "73.03%", "elapsed_time": "50m 18s", "remaining_time": "18m 34s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18388}
+{"loss": 0.30484335, "grad_norm": 11.05255347, "learning_rate": 1e-07, "token_acc": 0.91179291, "epoch": 7.36842105, "global_step/max_steps": "560/760", "percentage": "73.68%", "elapsed_time": "50m 37s", "remaining_time": "18m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184381}
+{"loss": 0.35793195, "grad_norm": 11.36206043, "learning_rate": 9e-08, "token_acc": 0.90137221, "epoch": 7.43421053, "global_step/max_steps": "565/760", "percentage": "74.34%", "elapsed_time": "50m 56s", "remaining_time": "17m 34s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184848}
+{"loss": 0.34636035, "grad_norm": 11.5419839, "learning_rate": 9e-08, "token_acc": 0.89878893, "epoch": 7.5, "global_step/max_steps": "570/760", "percentage": "75.00%", "elapsed_time": "51m 15s", "remaining_time": "17m 5s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185334}
+{"loss": 0.26542242, "grad_norm": 13.11391942, "learning_rate": 8e-08, "token_acc": 0.91387126, "epoch": 7.56578947, "global_step/max_steps": "575/760", "percentage": "75.66%", "elapsed_time": "51m 34s", "remaining_time": "16m 35s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185809}
+{"loss": 0.3173897, "grad_norm": 11.94004068, "learning_rate": 8e-08, "token_acc": 0.9028777, "epoch": 7.63157895, "global_step/max_steps": "580/760", "percentage": "76.32%", "elapsed_time": "51m 53s", "remaining_time": "16m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186273}
+{"loss": 0.31118855, "grad_norm": 14.66870429, "learning_rate": 8e-08, "token_acc": 0.91055276, "epoch": 7.69736842, "global_step/max_steps": "585/760", "percentage": "76.97%", "elapsed_time": "52m 13s", "remaining_time": "15m 37s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186708}
+{"loss": 0.32157631, "grad_norm": 12.06775911, "learning_rate": 7e-08, "token_acc": 0.90810811, "epoch": 7.76315789, "global_step/max_steps": "590/760", "percentage": "77.63%", "elapsed_time": "52m 32s", "remaining_time": "15m 8s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18717}
+{"loss": 0.30963364, "grad_norm": 11.37855567, "learning_rate": 7e-08, "token_acc": 0.90719861, "epoch": 7.82894737, "global_step/max_steps": "595/760", "percentage": "78.29%", "elapsed_time": "52m 50s", "remaining_time": "14m 39s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187665}
+{"loss": 0.35222101, "grad_norm": 10.968135, "learning_rate": 6e-08, "token_acc": 0.9011178, "epoch": 7.89473684, "global_step/max_steps": "600/760", "percentage": "78.95%", "elapsed_time": "53m 9s", "remaining_time": "14m 10s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188114}
+{"loss": 0.38019013, "grad_norm": 13.62190038, "learning_rate": 6e-08, "token_acc": 0.87465438, "epoch": 7.96052632, "global_step/max_steps": "605/760", "percentage": "79.61%", "elapsed_time": "53m 28s", "remaining_time": "13m 42s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188561}
+{"eval_loss": 0.90744567, "eval_runtime": 11.7559, "eval_samples_per_second": 11.484, "eval_steps_per_second": 1.446, "eval_token_acc": 0.76783813, "epoch": 8.0, "global_step/max_steps": "608/760", "percentage": "80.00%", "elapsed_time": "53m 51s", "remaining_time": "13m 27s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.188122}
+{"loss": 0.328544, "grad_norm": 11.09485671, "learning_rate": 6e-08, "token_acc": 0.90344249, "epoch": 8.02631579, "global_step/max_steps": "610/760", "percentage": "80.26%", "elapsed_time": "55m 51s", "remaining_time": "13m 44s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181997}
+{"loss": 0.36517785, "grad_norm": 13.5692201, "learning_rate": 5e-08, "token_acc": 0.90467626, "epoch": 8.09210526, "global_step/max_steps": "615/760", "percentage": "80.92%", "elapsed_time": "56m 12s", "remaining_time": "13m 15s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182366}
+{"loss": 0.33747931, "grad_norm": 12.34830502, "learning_rate": 5e-08, "token_acc": 0.89779736, "epoch": 8.15789474, "global_step/max_steps": "620/760", "percentage": "81.58%", "elapsed_time": "56m 32s", "remaining_time": "12m 46s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182734}
+{"loss": 0.31133468, "grad_norm": 13.37335466, "learning_rate": 5e-08, "token_acc": 0.90362511, "epoch": 8.22368421, "global_step/max_steps": "625/760", "percentage": "82.24%", "elapsed_time": "56m 52s", "remaining_time": "12m 17s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183125}
+{"loss": 0.32176852, "grad_norm": 9.70805811, "learning_rate": 4e-08, "token_acc": 0.9171123, "epoch": 8.28947368, "global_step/max_steps": "630/760", "percentage": "82.89%", "elapsed_time": "57m 12s", "remaining_time": "11m 48s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18353}
+{"loss": 0.32019682, "grad_norm": 11.08719854, "learning_rate": 4e-08, "token_acc": 0.91584602, "epoch": 8.35526316, "global_step/max_steps": "635/760", "percentage": "83.55%", "elapsed_time": "57m 31s", "remaining_time": "11m 19s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183973}
+{"loss": 0.33460722, "grad_norm": 11.68847763, "learning_rate": 4e-08, "token_acc": 0.9139265, "epoch": 8.42105263, "global_step/max_steps": "640/760", "percentage": "84.21%", "elapsed_time": "57m 50s", "remaining_time": "10m 50s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184391}
+{"loss": 0.25271811, "grad_norm": 10.1263874, "learning_rate": 3e-08, "token_acc": 0.92458101, "epoch": 8.48684211, "global_step/max_steps": "645/760", "percentage": "84.87%", "elapsed_time": "58m 10s", "remaining_time": "10m 22s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184813}
+{"loss": 0.28289604, "grad_norm": 10.57567859, "learning_rate": 3e-08, "token_acc": 0.91559001, "epoch": 8.55263158, "global_step/max_steps": "650/760", "percentage": "85.53%", "elapsed_time": "58m 28s", "remaining_time": "9m 53s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185247}
+{"loss": 0.30494127, "grad_norm": 11.25375302, "learning_rate": 3e-08, "token_acc": 0.90987536, "epoch": 8.61842105, "global_step/max_steps": "655/760", "percentage": "86.18%", "elapsed_time": "58m 48s", "remaining_time": "9m 25s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185649}
+{"loss": 0.26085703, "grad_norm": 9.62444181, "learning_rate": 3e-08, "token_acc": 0.92429577, "epoch": 8.68421053, "global_step/max_steps": "660/760", "percentage": "86.84%", "elapsed_time": "59m 7s", "remaining_time": "8m 57s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186035}
+{"loss": 0.31142015, "grad_norm": 14.0443236, "learning_rate": 2e-08, "token_acc": 0.90163934, "epoch": 8.75, "global_step/max_steps": "665/760", "percentage": "87.50%", "elapsed_time": "59m 26s", "remaining_time": "8m 29s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186436}
+{"loss": 0.33683982, "grad_norm": 14.93347267, "learning_rate": 2e-08, "token_acc": 0.90386521, "epoch": 8.81578947, "global_step/max_steps": "670/760", "percentage": "88.16%", "elapsed_time": "59m 45s", "remaining_time": "8m 1s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18686}
+{"loss": 0.30484743, "grad_norm": 12.3242763, "learning_rate": 2e-08, "token_acc": 0.91292639, "epoch": 8.88157895, "global_step/max_steps": "675/760", "percentage": "88.82%", "elapsed_time": "1h 0m 5s", "remaining_time": "7m 33s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187226}
+{"loss": 0.34318013, "grad_norm": 14.10461969, "learning_rate": 2e-08, "token_acc": 0.89927405, "epoch": 8.94736842, "global_step/max_steps": "680/760", "percentage": "89.47%", "elapsed_time": "1h 0m 24s", "remaining_time": "7m 6s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187631}
+{"eval_loss": 0.92236823, "eval_runtime": 11.8005, "eval_samples_per_second": 11.44, "eval_steps_per_second": 1.441, "eval_token_acc": 0.76411076, "epoch": 9.0, "global_step/max_steps": "684/760", "percentage": "90.00%", "elapsed_time": "1h 0m 50s", "remaining_time": "6m 45s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187389}
+{"loss": 0.30799294, "grad_norm": 13.72248565, "learning_rate": 1e-08, "token_acc": 0.9157197, "epoch": 9.01315789, "global_step/max_steps": "685/760", "percentage": "90.13%", "elapsed_time": "1h 2m 48s", "remaining_time": "6m 52s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181788}
+{"loss": 0.32543039, "grad_norm": 11.08344386, "learning_rate": 1e-08, "token_acc": 0.90828924, "epoch": 9.07894737, "global_step/max_steps": "690/760", "percentage": "90.79%", "elapsed_time": "1h 3m 7s", "remaining_time": "6m 24s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182179}
+{"loss": 0.29225836, "grad_norm": 11.90729093, "learning_rate": 1e-08, "token_acc": 0.92395105, "epoch": 9.14473684, "global_step/max_steps": "695/760", "percentage": "91.45%", "elapsed_time": "1h 3m 27s", "remaining_time": "5m 56s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182555}
+{"loss": 0.30806551, "grad_norm": 10.73859046, "learning_rate": 1e-08, "token_acc": 0.91734052, "epoch": 9.21052632, "global_step/max_steps": "700/760", "percentage": "92.11%", "elapsed_time": "1h 3m 47s", "remaining_time": "5m 28s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.182896}
+{"loss": 0.32995567, "grad_norm": 11.6545818, "learning_rate": 1e-08, "token_acc": 0.91465677, "epoch": 9.27631579, "global_step/max_steps": "705/760", "percentage": "92.76%", "elapsed_time": "1h 4m 6s", "remaining_time": "5m 0s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.183297}
+{"loss": 0.35297151, "grad_norm": 10.71563719, "learning_rate": 1e-08, "token_acc": 0.88929889, "epoch": 9.34210526, "global_step/max_steps": "710/760", "percentage": "93.42%", "elapsed_time": "1h 4m 24s", "remaining_time": "4m 32s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.18371}
+{"loss": 0.29446287, "grad_norm": 14.8354412, "learning_rate": 1e-08, "token_acc": 0.90686736, "epoch": 9.40789474, "global_step/max_steps": "715/760", "percentage": "94.08%", "elapsed_time": "1h 4m 43s", "remaining_time": "4m 4s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184104}
+{"loss": 0.27518668, "grad_norm": 13.32190376, "learning_rate": 0.0, "token_acc": 0.92389381, "epoch": 9.47368421, "global_step/max_steps": "720/760", "percentage": "94.74%", "elapsed_time": "1h 5m 3s", "remaining_time": "3m 36s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184429}
+{"loss": 0.28573508, "grad_norm": 10.25606923, "learning_rate": 0.0, "token_acc": 0.9112478, "epoch": 9.53947368, "global_step/max_steps": "725/760", "percentage": "95.39%", "elapsed_time": "1h 5m 22s", "remaining_time": "3m 9s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.184828}
+{"loss": 0.31120999, "grad_norm": 12.03400386, "learning_rate": 0.0, "token_acc": 0.9164396, "epoch": 9.60526316, "global_step/max_steps": "730/760", "percentage": "96.05%", "elapsed_time": "1h 5m 41s", "remaining_time": "2m 41s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185186}
+{"loss": 0.27546597, "grad_norm": 12.60717672, "learning_rate": 0.0, "token_acc": 0.93406593, "epoch": 9.67105263, "global_step/max_steps": "735/760", "percentage": "96.71%", "elapsed_time": "1h 6m 1s", "remaining_time": "2m 14s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185559}
+{"loss": 0.29992547, "grad_norm": 14.49252127, "learning_rate": 0.0, "token_acc": 0.91187739, "epoch": 9.73684211, "global_step/max_steps": "740/760", "percentage": "97.37%", "elapsed_time": "1h 6m 20s", "remaining_time": "1m 47s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.185929}
+{"loss": 0.28825083, "grad_norm": 10.32137161, "learning_rate": 0.0, "token_acc": 0.91651206, "epoch": 9.80263158, "global_step/max_steps": "745/760", "percentage": "98.03%", "elapsed_time": "1h 6m 38s", "remaining_time": "1m 20s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186314}
+{"loss": 0.29370143, "grad_norm": 11.55867552, "learning_rate": 0.0, "token_acc": 0.91425993, "epoch": 9.86842105, "global_step/max_steps": "750/760", "percentage": "98.68%", "elapsed_time": "1h 6m 57s", "remaining_time": "53s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186691}
+{"loss": 0.27887554, "grad_norm": 10.06715286, "learning_rate": 0.0, "token_acc": 0.92258652, "epoch": 9.93421053, "global_step/max_steps": "755/760", "percentage": "99.34%", "elapsed_time": "1h 7m 16s", "remaining_time": "26s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187024}
+{"loss": 0.33659291, "grad_norm": 16.67109379, "learning_rate": 0.0, "token_acc": 0.90573372, "epoch": 10.0, "global_step/max_steps": "760/760", "percentage": "100.00%", "elapsed_time": "1h 7m 35s", "remaining_time": "0s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.187412}
+{"eval_loss": 0.92296737, "eval_runtime": 11.7093, "eval_samples_per_second": 11.529, "eval_steps_per_second": 1.452, "eval_token_acc": 0.76464324, "epoch": 10.0, "global_step/max_steps": "760/760", "percentage": "100.00%", "elapsed_time": "1h 7m 46s", "remaining_time": "0s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.186872}
+{"eval_loss": 0.92296737, "eval_runtime": 14.567, "eval_samples_per_second": 9.268, "eval_steps_per_second": 1.167, "eval_token_acc": 0.76464324, "epoch": 10.0, "global_step/max_steps": "760/760", "percentage": "100.00%", "elapsed_time": "1h 9m 53s", "remaining_time": "0s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.181228}
+{"train_runtime": 4384.4417, "train_samples_per_second": 2.776, "train_steps_per_second": 0.173, "total_flos": 69848077049856.0, "train_loss": 0.52662418, "epoch": 10.0, "global_step/max_steps": "760/760", "percentage": "100.00%", "elapsed_time": "1h 12m 59s", "remaining_time": "0s", "memory(GiB)": 54.13, "train_speed(iter/s)": 0.173556}
+{"model_parameter_info": "Qwen2_5_VLForConditionalGeneration: 8292.1667M Params (7615.6165M Trainable [91.8411%]), 0.0019M Buffers.", "last_model_checkpoint": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-760", "best_model_checkpoint": "/mnt/data/users/liamding/data/MMMT/lora/qwen2.5vl-7b-full_sft_ood/v1-20251004-154120/checkpoint-304", "best_metric": 0.78271794, "global_step": 760, "log_history": [{"loss": 1.0294156074523926, "grad_norm": 17.31591809508792, "learning_rate": 6.578947368421052e-09, "token_acc": 0.696969696969697, "epoch": 0.013157894736842105, "step": 1}, {"loss": 0.9056330323219299, "grad_norm": 16.548277625750544, "learning_rate": 3.289473684210526e-08, "token_acc": 0.7521472392638037, "epoch": 0.06578947368421052, "step": 5}, {"loss": 1.0227190017700196, "grad_norm": 15.423444339129516, "learning_rate": 6.578947368421052e-08, "token_acc": 0.7282809611829945, "epoch": 0.13157894736842105, "step": 10}, {"loss": 0.9264978408813477, "grad_norm": 16.177173530018614, "learning_rate": 9.868421052631579e-08, "token_acc": 0.7623674911660777, "epoch": 0.19736842105263158, "step": 15}, {"loss": 0.9411476135253907, "grad_norm": 16.91098064197238, "learning_rate": 1.3157894736842104e-07, "token_acc": 0.7415832575068244, "epoch": 0.2631578947368421, "step": 20}, {"loss": 0.8977519989013671, "grad_norm": 17.10921014210045, "learning_rate": 1.6447368421052632e-07, "token_acc": 0.7601214574898786, "epoch": 0.32894736842105265, "step": 25}, {"loss": 0.913768196105957, "grad_norm": 17.1071080088263, "learning_rate": 1.9736842105263157e-07, "token_acc": 0.7412712623097583, "epoch": 0.39473684210526316, "step": 30}, {"loss": 0.8454873085021972, "grad_norm": 19.227963936500046, "learning_rate": 2.3026315789473683e-07, "token_acc": 0.7644521138912856, "epoch": 0.4605263157894737, "step": 35}, {"loss": 0.7893925666809082, "grad_norm": 16.200443052460418, "learning_rate": 2.631578947368421e-07, "token_acc": 0.7670863309352518, "epoch": 0.5263157894736842, "step": 40}, {"loss": 0.8727176666259766, "grad_norm": 12.81356621670082, "learning_rate": 2.960526315789473e-07, "token_acc": 0.7574171029668412, "epoch": 0.5921052631578947, "step": 45}, {"loss": 0.8613866806030274, "grad_norm": 23.738559726620743, "learning_rate": 3.2894736842105264e-07, "token_acc": 0.7643884892086331, "epoch": 0.6578947368421053, "step": 50}, {"loss": 0.9199527740478516, "grad_norm": 18.20488127129436, "learning_rate": 3.618421052631579e-07, "token_acc": 0.7350044762757386, "epoch": 0.7236842105263158, "step": 55}, {"loss": 0.8155546188354492, "grad_norm": 16.53984575296178, "learning_rate": 3.9473684210526315e-07, "token_acc": 0.7675996607294318, "epoch": 0.7894736842105263, "step": 60}, {"loss": 0.8712363243103027, "grad_norm": 15.361305270466637, "learning_rate": 4.2763157894736837e-07, "token_acc": 0.75, "epoch": 0.8552631578947368, "step": 65}, {"loss": 0.7804427146911621, "grad_norm": 14.563690201290349, "learning_rate": 4.6052631578947365e-07, "token_acc": 0.7817679558011049, "epoch": 0.9210526315789473, "step": 70}, {"loss": 0.8566499710083008, "grad_norm": 13.727303038868559, "learning_rate": 4.934210526315789e-07, "token_acc": 0.7480916030534351, "epoch": 0.9868421052631579, "step": 75}, {"eval_loss": 0.8291671872138977, "eval_runtime": 10.7249, "eval_samples_per_second": 12.587, "eval_steps_per_second": 1.585, "eval_token_acc": 0.759318423855165, "epoch": 1.0, "step": 76}, {"loss": 0.8415672302246093, "grad_norm": 13.787467147156026, "learning_rate": 4.999578104083306e-07, "token_acc": 0.7571157495256167, "epoch": 1.0526315789473684, "step": 80}, {"loss": 0.9233635902404785, "grad_norm": 13.705484769366395, "learning_rate": 4.997864395968252e-07, "token_acc": 0.7350579839429081, "epoch": 1.118421052631579, "step": 85}, {"loss": 0.8335156440734863, "grad_norm": 13.825524241955167, "learning_rate": 4.994833410208486e-07, "token_acc": 0.7606837606837606, "epoch": 1.1842105263157894, "step": 90}, {"loss": 0.7449466228485108, "grad_norm": 13.812951965959309, "learning_rate": 4.990486745229364e-07, "token_acc": 0.7797202797202797, "epoch": 1.25, "step": 95}, {"loss": 0.7893705844879151, "grad_norm": 12.979769860353171, "learning_rate": 4.984826693294873e-07, "token_acc": 0.7683168316831683, "epoch": 1.3157894736842106, "step": 100}, {"loss": 0.7847232818603516, "grad_norm": 11.910642093630177, "learning_rate": 4.977856239298789e-07, "token_acc": 0.7681159420289855, "epoch": 1.381578947368421, "step": 105}, {"loss": 0.7388627052307128, "grad_norm": 15.20236434989543, "learning_rate": 4.969579059190548e-07, "token_acc": 0.7783783783783784, "epoch": 1.4473684210526316, "step": 110}, {"loss": 0.7851850509643554, "grad_norm": 13.920033113899239, "learning_rate": 4.9599995180367e-07, "token_acc": 0.7647547797173733, "epoch": 1.513157894736842, "step": 115}, {"loss": 0.7619089126586914, "grad_norm": 12.383418809687024, "learning_rate": 4.949122667718934e-07, "token_acc": 0.7688848920863309, "epoch": 1.5789473684210527, "step": 120}, {"loss": 0.7471943378448487, "grad_norm": 15.64056532443421, "learning_rate": 4.936954244269917e-07, "token_acc": 0.7835820895522388, "epoch": 1.6447368421052633, "step": 125}, {"loss": 0.7908691883087158, "grad_norm": 13.492059761704322, "learning_rate": 4.923500664848326e-07, "token_acc": 0.7828096118299446, "epoch": 1.7105263157894737, "step": 130}, {"loss": 0.7812448501586914, "grad_norm": 14.529532033698874, "learning_rate": 4.908769024354683e-07, "token_acc": 0.7780979827089337, "epoch": 1.776315789473684, "step": 135}, {"loss": 0.7617037773132325, "grad_norm": 13.777161961650751, "learning_rate": 4.892767091689785e-07, "token_acc": 0.7804444444444445, "epoch": 1.8421052631578947, "step": 140}, {"loss": 0.8042187690734863, "grad_norm": 11.946802493317874, "learning_rate": 4.875503305657677e-07, "token_acc": 0.772093023255814, "epoch": 1.9078947368421053, "step": 145}, {"loss": 0.7846203804016113, "grad_norm": 14.275549665617376, "learning_rate": 4.856986770515357e-07, "token_acc": 0.7886029411764706, "epoch": 1.973684210526316, "step": 150}, {"eval_loss": 0.7974297404289246, "eval_runtime": 11.3855, "eval_samples_per_second": 11.857, "eval_steps_per_second": 1.493, "eval_token_acc": 0.7699680511182109, "epoch": 2.0, "step": 152}, {"loss": 0.704461145401001, "grad_norm": 12.208281829044838, "learning_rate": 4.837227251171537e-07, "token_acc": 0.7754927163667523, "epoch": 2.039473684210526, "step": 155}, {"loss": 0.74434494972229, "grad_norm": 12.86816500951879, "learning_rate": 4.816235168037004e-07, "token_acc": 0.7828886844526219, "epoch": 2.1052631578947367, "step": 160}, {"loss": 0.6901160717010498, "grad_norm": 13.155535172747632, "learning_rate": 4.794021591529302e-07, "token_acc": 0.7897574123989218, "epoch": 2.1710526315789473, "step": 165}, {"loss": 0.7156202793121338, "grad_norm": 11.694403910193117, "learning_rate": 4.770598236234616e-07, "token_acc": 0.8040152963671128, "epoch": 2.236842105263158, "step": 170}, {"loss": 0.7139366149902344, "grad_norm": 15.777973481351275, "learning_rate": 4.745977454729947e-07, "token_acc": 0.7982222222222223, "epoch": 2.3026315789473686, "step": 175}, {"loss": 0.6547629356384277, "grad_norm": 11.314273525019125, "learning_rate": 4.720172231068844e-07, "token_acc": 0.7982283464566929, "epoch": 2.3684210526315788, "step": 180}, {"loss": 0.6626088142395019, "grad_norm": 12.048816862642347, "learning_rate": 4.693196173934107e-07, "token_acc": 0.82744702320888, "epoch": 2.4342105263157894, "step": 185}, {"loss": 0.775759220123291, "grad_norm": 13.592183711828891, "learning_rate": 4.6650635094610966e-07, "token_acc": 0.7822374039282665, "epoch": 2.5, "step": 190}, {"loss": 0.6928662300109864, "grad_norm": 13.289728232283066, "learning_rate": 4.635789073735412e-07, "token_acc": 0.7897526501766784, "epoch": 2.5657894736842106, "step": 195}, {"loss": 0.6376441955566406, "grad_norm": 16.08999575789933, "learning_rate": 4.605388304968914e-07, "token_acc": 0.8148487626031164, "epoch": 2.6315789473684212, "step": 200}, {"loss": 0.6813505172729493, "grad_norm": 14.327531535921459, "learning_rate": 4.5738772353582033e-07, "token_acc": 0.7959542656112577, "epoch": 2.6973684210526314, "step": 205}, {"loss": 0.6876439094543457, "grad_norm": 14.977189931535413, "learning_rate": 4.541272482629857e-07, "token_acc": 0.8045325779036827, "epoch": 2.763157894736842, "step": 210}, {"loss": 0.7278162002563476, "grad_norm": 13.092843394229808, "learning_rate": 4.507591241276879e-07, "token_acc": 0.781491002570694, "epoch": 2.8289473684210527, "step": 215}, {"loss": 0.7397733688354492, "grad_norm": 14.423896457474601, "learning_rate": 4.472851273490984e-07, "token_acc": 0.7873811581676751, "epoch": 2.8947368421052633, "step": 220}, {"loss": 0.7403717517852784, "grad_norm": 14.00628897732342, "learning_rate": 4.437070899795503e-07, "token_acc": 0.7749546279491834, "epoch": 2.9605263157894735, "step": 225}, {"eval_loss": 0.7827797532081604, "eval_runtime": 11.2217, "eval_samples_per_second": 12.03, "eval_steps_per_second": 1.515, "eval_token_acc": 0.7715654952076677, "epoch": 3.0, "step": 228}, {"loss": 0.6476545810699463, "grad_norm": 12.117965531246165, "learning_rate": 4.4002689893838405e-07, "token_acc": 0.8024691358024691, "epoch": 3.026315789473684, "step": 230}, {"loss": 0.5883333206176757, "grad_norm": 10.634581616763157, "learning_rate": 4.3624649501685923e-07, "token_acc": 0.8282918149466192, "epoch": 3.0921052631578947, "step": 235}, {"loss": 0.626362943649292, "grad_norm": 13.972414955051068, "learning_rate": 4.323678718546552e-07, "token_acc": 0.8032345013477089, "epoch": 3.1578947368421053, "step": 240}, {"loss": 0.5920934200286865, "grad_norm": 11.77574647255366, "learning_rate": 4.2839307488850264e-07, "token_acc": 0.8289930555555556, "epoch": 3.223684210526316, "step": 245}, {"loss": 0.6178097724914551, "grad_norm": 12.858641327934963, "learning_rate": 4.243242002734988e-07, "token_acc": 0.8099389712292938, "epoch": 3.2894736842105265, "step": 250}, {"loss": 0.6222902774810791, "grad_norm": 11.992039120911935, "learning_rate": 4.201633937776759e-07, "token_acc": 0.8178368121442126, "epoch": 3.3552631578947367, "step": 255}, {"loss": 0.6095127582550048, "grad_norm": 14.323771950489155, "learning_rate": 4.159128496504053e-07, "token_acc": 0.8162267839687195, "epoch": 3.4210526315789473, "step": 260}, {"loss": 0.6878387451171875, "grad_norm": 13.080602987140287, "learning_rate": 4.115748094652352e-07, "token_acc": 0.7913351016799293, "epoch": 3.486842105263158, "step": 265}, {"loss": 0.6472003936767579, "grad_norm": 12.921498214011155, "learning_rate": 4.071515609377705e-07, "token_acc": 0.818973862536302, "epoch": 3.5526315789473686, "step": 270}, {"loss": 0.5890860557556152, "grad_norm": 12.375533162441062, "learning_rate": 4.026454367192199e-07, "token_acc": 0.8175046554934823, "epoch": 3.6184210526315788, "step": 275}, {"loss": 0.6693721771240234, "grad_norm": 13.129202055657805, "learning_rate": 3.9805881316624503e-07, "token_acc": 0.8168761220825853, "epoch": 3.6842105263157894, "step": 280}, {"loss": 0.6817171096801757, "grad_norm": 14.407473010348072, "learning_rate": 3.933941090877615e-07, "token_acc": 0.8132059079061685, "epoch": 3.75, "step": 285}, {"loss": 0.6139223098754882, "grad_norm": 13.662642322032315, "learning_rate": 3.8865378446935217e-07, "token_acc": 0.8305555555555556, "epoch": 3.8157894736842106, "step": 290}, {"loss": 0.5279911041259766, "grad_norm": 11.472371827393802, "learning_rate": 3.8384033917596515e-07, "token_acc": 0.8325666973321068, "epoch": 3.8815789473684212, "step": 295}, {"loss": 0.5814344882965088, "grad_norm": 13.611249691490164, "learning_rate": 3.78956311633581e-07, "token_acc": 0.8251082251082251, "epoch": 3.9473684210526314, "step": 300}, {"eval_loss": 0.7827179431915283, "eval_runtime": 11.4603, "eval_samples_per_second": 11.78, "eval_steps_per_second": 1.483, "eval_token_acc": 0.775825346112886, "epoch": 4.0, "step": 304}, {"loss": 0.497452974319458, "grad_norm": 12.235482741873291, "learning_rate": 3.740042774905449e-07, "token_acc": 0.8531531531531531, "epoch": 4.0131578947368425, "step": 305}, {"loss": 0.52832932472229, "grad_norm": 11.58635467783027, "learning_rate": 3.689868482592684e-07, "token_acc": 0.8386524822695035, "epoch": 4.078947368421052, "step": 310}, {"loss": 0.5145404815673829, "grad_norm": 12.063207912616575, "learning_rate": 3.6390666993901826e-07, "token_acc": 0.8331716779825412, "epoch": 4.144736842105263, "step": 315}, {"loss": 0.5273187160491943, "grad_norm": 12.354386618427899, "learning_rate": 3.587664216205183e-07, "token_acc": 0.833011583011583, "epoch": 4.2105263157894735, "step": 320}, {"loss": 0.4995855808258057, "grad_norm": 11.795002670717809, "learning_rate": 3.535688140730997e-07, "token_acc": 0.8554801163918526, "epoch": 4.276315789473684, "step": 325}, {"loss": 0.5351977348327637, "grad_norm": 12.798727775273417, "learning_rate": 3.4831658831514575e-07, "token_acc": 0.8505654281098546, "epoch": 4.342105263157895, "step": 330}, {"loss": 0.5155584812164307, "grad_norm": 12.730694168346282, "learning_rate": 3.4301251416858403e-07, "token_acc": 0.8545296167247387, "epoch": 4.407894736842105, "step": 335}, {"loss": 0.5392285346984863, "grad_norm": 14.484297908634066, "learning_rate": 3.376593887981886e-07, "token_acc": 0.8305383936451898, "epoch": 4.473684210526316, "step": 340}, {"loss": 0.4732147216796875, "grad_norm": 11.491036408986366, "learning_rate": 3.322600352364633e-07, "token_acc": 0.8621553884711779, "epoch": 4.5394736842105265, "step": 345}, {"loss": 0.5695515155792237, "grad_norm": 11.33748544855005, "learning_rate": 3.268173008948826e-07, "token_acc": 0.8312159709618875, "epoch": 4.605263157894737, "step": 350}, {"loss": 0.4677370548248291, "grad_norm": 12.56964612455139, "learning_rate": 3.213340560622763e-07, "token_acc": 0.8618290258449304, "epoch": 4.671052631578947, "step": 355}, {"loss": 0.5584880352020264, "grad_norm": 15.814504311164637, "learning_rate": 3.1581319239114976e-07, "token_acc": 0.838, "epoch": 4.7368421052631575, "step": 360}, {"loss": 0.5094208717346191, "grad_norm": 12.026239005665621, "learning_rate": 3.1025762137273735e-07, "token_acc": 0.8604878048780488, "epoch": 4.802631578947368, "step": 365}, {"loss": 0.4925221920013428, "grad_norm": 13.29445423103592, "learning_rate": 3.0467027280159394e-07, "token_acc": 0.865916955017301, "epoch": 4.868421052631579, "step": 370}, {"loss": 0.5490811347961426, "grad_norm": 12.422013374565926, "learning_rate": 2.99054093230534e-07, "token_acc": 0.8436103663985701, "epoch": 4.934210526315789, "step": 375}, {"loss": 0.5336416244506836, "grad_norm": 15.73394585068487, "learning_rate": 2.934120444167326e-07, "token_acc": 0.8478260869565217, "epoch": 5.0, "step": 380}, {"eval_loss": 0.8055287599563599, "eval_runtime": 11.9734, "eval_samples_per_second": 11.275, "eval_steps_per_second": 1.42, "eval_token_acc": 0.7731629392971247, "epoch": 5.0, "step": 380}, {"loss": 0.4058138370513916, "grad_norm": 11.579318373185998, "learning_rate": 2.877471017598092e-07, "token_acc": 0.8892100192678227, "epoch": 5.065789473684211, "step": 385}, {"loss": 0.4946115970611572, "grad_norm": 13.466331406979641, "learning_rate": 2.820622527327158e-07, "token_acc": 0.8582020389249305, "epoch": 5.131578947368421, "step": 390}, {"loss": 0.44744672775268557, "grad_norm": 12.70964930976301, "learning_rate": 2.7636049530625844e-07, "token_acc": 0.8555885262116716, "epoch": 5.197368421052632, "step": 395}, {"loss": 0.39708123207092283, "grad_norm": 10.081726462821392, "learning_rate": 2.706448363680831e-07, "token_acc": 0.8752310536044362, "epoch": 5.2631578947368425, "step": 400}, {"loss": 0.44630985260009765, "grad_norm": 13.123290073548652, "learning_rate": 2.649182901369585e-07, "token_acc": 0.8673647469458988, "epoch": 5.328947368421053, "step": 405}, {"loss": 0.4424083709716797, "grad_norm": 12.55092675524107, "learning_rate": 2.591838765731931e-07, "token_acc": 0.8709369024856597, "epoch": 5.394736842105263, "step": 410}, {"loss": 0.4120338916778564, "grad_norm": 12.157000958625412, "learning_rate": 2.5344461978602413e-07, "token_acc": 0.8837209302325582, "epoch": 5.4605263157894735, "step": 415}, {"loss": 0.39818453788757324, "grad_norm": 11.797704490185449, "learning_rate": 2.477035464388184e-07, "token_acc": 0.8734402852049911, "epoch": 5.526315789473684, "step": 420}, {"loss": 0.4546657562255859, "grad_norm": 12.745422925981462, "learning_rate": 2.419636841529271e-07, "token_acc": 0.8704710144927537, "epoch": 5.592105263157895, "step": 425}, {"loss": 0.45757322311401366, "grad_norm": 13.324760956982862, "learning_rate": 2.3622805991103358e-07, "token_acc": 0.8718190386427899, "epoch": 5.657894736842105, "step": 430}, {"loss": 0.45713186264038086, "grad_norm": 12.043093034906175, "learning_rate": 2.3049969846084053e-07, "token_acc": 0.8784965034965035, "epoch": 5.723684210526316, "step": 435}, {"loss": 0.4352132320404053, "grad_norm": 15.658576422078957, "learning_rate": 2.2478162071993296e-07, "token_acc": 0.8861340679522498, "epoch": 5.7894736842105265, "step": 440}, {"loss": 0.49906353950500487, "grad_norm": 11.146740957754833, "learning_rate": 2.1907684218266306e-07, "token_acc": 0.8625429553264605, "epoch": 5.855263157894737, "step": 445}, {"loss": 0.43380098342895507, "grad_norm": 10.820075426555121, "learning_rate": 2.133883713298946e-07, "token_acc": 0.8672712283594394, "epoch": 5.921052631578947, "step": 450}, {"loss": 0.3760183334350586, "grad_norm": 10.22116607799743, "learning_rate": 2.0771920804244537e-07, "token_acc": 0.8947368421052632, "epoch": 5.9868421052631575, "step": 455}, {"eval_loss": 0.8343057036399841, "eval_runtime": 11.7159, "eval_samples_per_second": 11.523, "eval_steps_per_second": 1.451, "eval_token_acc": 0.7726304579339723, "epoch": 6.0, "step": 456}, {"loss": 0.442535400390625, "grad_norm": 13.367377775022812, "learning_rate": 2.0207234201906545e-07, "token_acc": 0.8815663801337154, "epoch": 6.052631578947368, "step": 460}, {"loss": 0.3871904373168945, "grad_norm": 13.885180152614968, "learning_rate": 1.9645075119978528e-07, "token_acc": 0.8865116279069768, "epoch": 6.118421052631579, "step": 465}, {"loss": 0.3912176609039307, "grad_norm": 12.749527263262742, "learning_rate": 1.9085740019546436e-07, "token_acc": 0.8739635157545605, "epoch": 6.184210526315789, "step": 470}, {"loss": 0.34154629707336426, "grad_norm": 11.114119762408144, "learning_rate": 1.8529523872436977e-07, "token_acc": 0.8987583572110793, "epoch": 6.25, "step": 475}, {"loss": 0.38265395164489746, "grad_norm": 9.05417935997834, "learning_rate": 1.7976720005660767e-07, "token_acc": 0.8994708994708994, "epoch": 6.315789473684211, "step": 480}, {"loss": 0.34448580741882323, "grad_norm": 12.763829441208104, "learning_rate": 1.742761994672297e-07, "token_acc": 0.8991150442477877, "epoch": 6.381578947368421, "step": 485}, {"loss": 0.32860574722290037, "grad_norm": 13.035014643295293, "learning_rate": 1.6882513269882913e-07, "token_acc": 0.9065077910174152, "epoch": 6.447368421052632, "step": 490}, {"loss": 0.32854912281036375, "grad_norm": 10.831268513096093, "learning_rate": 1.6341687443443738e-07, "token_acc": 0.9098005203816132, "epoch": 6.5131578947368425, "step": 495}, {"loss": 0.331088924407959, "grad_norm": 9.93697018265633, "learning_rate": 1.5805427678152674e-07, "token_acc": 0.9036697247706422, "epoch": 6.578947368421053, "step": 500}, {"loss": 0.4194302558898926, "grad_norm": 13.716271495126469, "learning_rate": 1.527401677679185e-07, "token_acc": 0.8747795414462081, "epoch": 6.644736842105263, "step": 505}, {"loss": 0.4289896488189697, "grad_norm": 13.399882710387233, "learning_rate": 1.4747734985039e-07, "token_acc": 0.883629191321499, "epoch": 6.7105263157894735, "step": 510}, {"loss": 0.38254172801971437, "grad_norm": 12.182744689817845, "learning_rate": 1.422685984367664e-07, "token_acc": 0.8839368616527391, "epoch": 6.776315789473684, "step": 515}, {"loss": 0.34045658111572263, "grad_norm": 9.707186610053322, "learning_rate": 1.371166604222777e-07, "token_acc": 0.8976510067114094, "epoch": 6.842105263157895, "step": 520}, {"loss": 0.3553584098815918, "grad_norm": 11.167280766500607, "learning_rate": 1.3202425274095116e-07, "token_acc": 0.8940092165898618, "epoch": 6.907894736842105, "step": 525}, {"loss": 0.3978888511657715, "grad_norm": 12.339426222895359, "learning_rate": 1.2699406093280545e-07, "token_acc": 0.8934262948207171, "epoch": 6.973684210526316, "step": 530}, {"eval_loss": 0.8763838410377502, "eval_runtime": 11.7441, "eval_samples_per_second": 11.495, "eval_steps_per_second": 1.448, "eval_token_acc": 0.7694355697550586, "epoch": 7.0, "step": 532}, {"loss": 0.3524200439453125, "grad_norm": 7.718224097938739, "learning_rate": 1.220287377275998e-07, "token_acc": 0.8997451146983857, "epoch": 7.0394736842105265, "step": 535}, {"loss": 0.39486873149871826, "grad_norm": 14.453311680486095, "learning_rate": 1.1713090164588606e-07, "token_acc": 0.9148148148148149, "epoch": 7.105263157894737, "step": 540}, {"loss": 0.3444140195846558, "grad_norm": 11.37036944935638, "learning_rate": 1.1230313561810215e-07, "token_acc": 0.8974854932301741, "epoch": 7.171052631578948, "step": 545}, {"loss": 0.32244648933410647, "grad_norm": 9.900805732362533, "learning_rate": 1.0754798562243344e-07, "token_acc": 0.9087301587301587, "epoch": 7.2368421052631575, "step": 550}, {"loss": 0.3322294235229492, "grad_norm": 14.143491211947243, "learning_rate": 1.0286795934216203e-07, "token_acc": 0.9039179104477612, "epoch": 7.302631578947368, "step": 555}, {"loss": 0.3048433542251587, "grad_norm": 11.052553470405742, "learning_rate": 9.826552484321085e-08, "token_acc": 0.9117929050814957, "epoch": 7.368421052631579, "step": 560}, {"loss": 0.35793194770812986, "grad_norm": 11.362060430165748, "learning_rate": 9.374310927258038e-08, "token_acc": 0.9013722126929674, "epoch": 7.434210526315789, "step": 565}, {"loss": 0.3463603496551514, "grad_norm": 11.541983896518747, "learning_rate": 8.930309757836516e-08, "token_acc": 0.8987889273356401, "epoch": 7.5, "step": 570}, {"loss": 0.2654224157333374, "grad_norm": 13.113919421742889, "learning_rate": 8.49478312520235e-08, "token_acc": 0.9138712601994561, "epoch": 7.565789473684211, "step": 575}, {"loss": 0.317389702796936, "grad_norm": 11.94004067837314, "learning_rate": 8.067960709356478e-08, "token_acc": 0.9028776978417267, "epoch": 7.631578947368421, "step": 580}, {"loss": 0.3111885547637939, "grad_norm": 14.668704290159432, "learning_rate": 7.650067600030583e-08, "token_acc": 0.9105527638190954, "epoch": 7.697368421052632, "step": 585}, {"loss": 0.3215763092041016, "grad_norm": 12.067759105613176, "learning_rate": 7.241324177983399e-08, "token_acc": 0.9081081081081082, "epoch": 7.7631578947368425, "step": 590}, {"loss": 0.3096336364746094, "grad_norm": 11.378555670601354, "learning_rate": 6.841945998780374e-08, "token_acc": 0.9071986123156982, "epoch": 7.828947368421053, "step": 595}, {"loss": 0.3522210121154785, "grad_norm": 10.968135004904445, "learning_rate": 6.452143679117964e-08, "token_acc": 0.9011177987962167, "epoch": 7.894736842105263, "step": 600}, {"loss": 0.3801901340484619, "grad_norm": 13.621900375202348, "learning_rate": 6.072122785752448e-08, "token_acc": 0.8746543778801843, "epoch": 7.9605263157894735, "step": 605}, {"eval_loss": 0.9074456691741943, "eval_runtime": 11.7559, "eval_samples_per_second": 11.484, "eval_steps_per_second": 1.446, "eval_token_acc": 0.7678381256656017, "epoch": 8.0, "step": 608}, {"loss": 0.32854399681091306, "grad_norm": 11.094856706759074, "learning_rate": 5.702083727091977e-08, "token_acc": 0.9034424853064652, "epoch": 8.026315789473685, "step": 610}, {"loss": 0.365177845954895, "grad_norm": 13.56922009827688, "learning_rate": 5.34222164750886e-08, "token_acc": 0.9046762589928058, "epoch": 8.092105263157896, "step": 615}, {"loss": 0.337479305267334, "grad_norm": 12.348305022353873, "learning_rate": 4.992726324427901e-08, "token_acc": 0.8977973568281938, "epoch": 8.157894736842104, "step": 620}, {"loss": 0.3113346815109253, "grad_norm": 13.373354658665798, "learning_rate": 4.653782068245127e-08, "token_acc": 0.9036251105216623, "epoch": 8.223684210526315, "step": 625}, {"loss": 0.32176852226257324, "grad_norm": 9.70805810993063, "learning_rate": 4.325567625129545e-08, "token_acc": 0.9171122994652406, "epoch": 8.289473684210526, "step": 630}, {"loss": 0.32019681930541993, "grad_norm": 11.087198542630968, "learning_rate": 4.008256082759284e-08, "token_acc": 0.9158460161145927, "epoch": 8.355263157894736, "step": 635}, {"loss": 0.3346072196960449, "grad_norm": 11.68847762915729, "learning_rate": 3.702014779041826e-08, "token_acc": 0.913926499032882, "epoch": 8.421052631578947, "step": 640}, {"loss": 0.2527181148529053, "grad_norm": 10.12638740099469, "learning_rate": 3.407005213866393e-08, "token_acc": 0.9245810055865922, "epoch": 8.486842105263158, "step": 645}, {"loss": 0.2828960418701172, "grad_norm": 10.575678593551583, "learning_rate": 3.123382963935156e-08, "token_acc": 0.9155900086132644, "epoch": 8.552631578947368, "step": 650}, {"loss": 0.3049412727355957, "grad_norm": 11.253753023963641, "learning_rate": 2.85129760071805e-08, "token_acc": 0.909875359539789, "epoch": 8.618421052631579, "step": 655}, {"loss": 0.2608570337295532, "grad_norm": 9.624441812742798, "learning_rate": 2.5908926115744994e-08, "token_acc": 0.9242957746478874, "epoch": 8.68421052631579, "step": 660}, {"loss": 0.3114201545715332, "grad_norm": 14.044323598527022, "learning_rate": 2.3423053240837514e-08, "token_acc": 0.9016393442622951, "epoch": 8.75, "step": 665}, {"loss": 0.3368398189544678, "grad_norm": 14.93347266826451, "learning_rate": 2.1056668336235623e-08, "token_acc": 0.9038652130822596, "epoch": 8.81578947368421, "step": 670}, {"loss": 0.30484743118286134, "grad_norm": 12.32427630355762, "learning_rate": 1.8811019342355433e-08, "token_acc": 0.9129263913824057, "epoch": 8.881578947368421, "step": 675}, {"loss": 0.34318013191223146, "grad_norm": 14.104619692060185, "learning_rate": 1.6687290528135722e-08, "token_acc": 0.8992740471869328, "epoch": 8.947368421052632, "step": 680}, {"eval_loss": 0.9223682284355164, "eval_runtime": 11.8005, "eval_samples_per_second": 11.44, "eval_steps_per_second": 1.441, "eval_token_acc": 0.7641107561235356, "epoch": 9.0, "step": 684}, {"loss": 0.30799293518066406, "grad_norm": 13.722485651177253, "learning_rate": 1.4686601866500115e-08, "token_acc": 0.915719696969697, "epoch": 9.013157894736842, "step": 685}, {"loss": 0.32543039321899414, "grad_norm": 11.083443862370641, "learning_rate": 1.2810008443726456e-08, "token_acc": 0.908289241622575, "epoch": 9.078947368421053, "step": 690}, {"loss": 0.29225835800170896, "grad_norm": 11.90729092801513, "learning_rate": 1.105849990303473e-08, "token_acc": 0.923951048951049, "epoch": 9.144736842105264, "step": 695}, {"loss": 0.30806550979614256, "grad_norm": 10.738590457271043, "learning_rate": 9.432999922687396e-09, "token_acc": 0.917340521114106, "epoch": 9.210526315789474, "step": 700}, {"loss": 0.3299556732177734, "grad_norm": 11.654581799055522, "learning_rate": 7.934365728877068e-09, "token_acc": 0.9146567717996289, "epoch": 9.276315789473685, "step": 705}, {"loss": 0.3529715061187744, "grad_norm": 10.715637193504373, "learning_rate": 6.563387643658075e-09, "token_acc": 0.8892988929889298, "epoch": 9.342105263157896, "step": 710}, {"loss": 0.2944628715515137, "grad_norm": 14.835441198761986, "learning_rate": 5.3207886681613804e-09, "token_acc": 0.9068673565380997, "epoch": 9.407894736842104, "step": 715}, {"loss": 0.27518668174743655, "grad_norm": 13.321903757763652, "learning_rate": 4.207224101311246e-09, "token_acc": 0.9238938053097345, "epoch": 9.473684210526315, "step": 720}, {"loss": 0.2857350826263428, "grad_norm": 10.256069230948793, "learning_rate": 3.223281194245975e-09, "token_acc": 0.9112478031634447, "epoch": 9.539473684210526, "step": 725}, {"loss": 0.3112099885940552, "grad_norm": 12.03400386189763, "learning_rate": 2.3694788406241894e-09, "token_acc": 0.9164396003633061, "epoch": 9.605263157894736, "step": 730}, {"loss": 0.2754659652709961, "grad_norm": 12.607176716250494, "learning_rate": 1.6462673029802587e-09, "token_acc": 0.9340659340659341, "epoch": 9.671052631578947, "step": 735}, {"loss": 0.2999254703521729, "grad_norm": 14.492521266612545, "learning_rate": 1.0540279752731252e-09, "token_acc": 0.9118773946360154, "epoch": 9.736842105263158, "step": 740}, {"loss": 0.28825082778930666, "grad_norm": 10.321371605871532, "learning_rate": 5.930731817538893e-10, "token_acc": 0.9165120593692022, "epoch": 9.802631578947368, "step": 745}, {"loss": 0.29370143413543703, "grad_norm": 11.558675521812575, "learning_rate": 2.636460122578399e-10, "token_acc": 0.9142599277978339, "epoch": 9.868421052631579, "step": 750}, {"loss": 0.27887554168701173, "grad_norm": 10.067152860690346, "learning_rate": 6.592019400841753e-11, "token_acc": 0.9225865209471766, "epoch": 9.93421052631579, "step": 755}, {"loss": 0.3365929126739502, "grad_norm": 16.67109379317691, "learning_rate": 0.0, "token_acc": 0.9057337220602527, "epoch": 10.0, "step": 760}, {"eval_loss": 0.9229673743247986, "eval_runtime": 11.7093, "eval_samples_per_second": 11.529, "eval_steps_per_second": 1.452, "eval_token_acc": 0.7646432374866879, "epoch": 10.0, "step": 760}, {"eval_loss": 0.9229673743247986, "eval_runtime": 14.567, "eval_samples_per_second": 9.268, "eval_steps_per_second": 1.167, "eval_token_acc": 0.7646432374866879, "epoch": 10.0, "step": 760}, {"train_runtime": 4384.4417, "train_samples_per_second": 2.776, "train_steps_per_second": 0.173, "total_flos": 69848077049856.0, "train_loss": 0.5266241832783348, "epoch": 10.0, "step": 760}], "memory": 54.130859375}