fulekkk commited on Jan 27, 2025

Commit

d44137e

1 Parent(s): 84c5ce4

提交信息

Browse files

Files changed (35) hide show

InternVL101/.ipynb_checkpoints/demo-checkpoint.py +0 -119
InternVL101/configuration.json +0 -1
InternVL101/demo.py +0 -119
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/.ipynb_checkpoints/internvl_v2_internlm2_2b_lora_finetune_food-checkpoint.py +0 -139
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/20241230_145825.log +0 -517
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/20241230_145825.json +0 -64
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/config.py +0 -139
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/scalars.json +0 -64
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/internvl_v2_internlm2_2b_lora_finetune_food.py +0 -139
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_128.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_192.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_256.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_320.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_384.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_448.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_512.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_576.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_64.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_640.pth +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/last_checkpoint +0 -1
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/added_tokens.json +0 -11
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/config.json +0 -199
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_intern_vit.py +0 -119
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_internlm2.py +0 -150
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_internvl_chat.py +0 -96
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/conversation.py +0 -393
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/generation_config.json +0 -4
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/model.safetensors +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_intern_vit.py +0 -435
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_internlm2.py +0 -1415
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_internvl_chat.py +0 -345
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/special_tokens_map.json +0 -47
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenization_internlm2.py +0 -235
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenizer.model +0 -3
InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenizer_config.json +0 -179

InternVL101/.ipynb_checkpoints/demo-checkpoint.py DELETED Viewed

@@ -1,119 +0,0 @@
-import os
-import random
-import numpy as np
-import torch
-import torch.backends.cudnn as cudnn
-import gradio as gr
-from utils import load_json, init_logger
-from demo import ConversationalAgent, CustomTheme
-FOOD_EXAMPLES = "demo/food_for_demo.json"
-MODEL_PATH = "/root/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10"
-# MODEL_PATH = "/root/xtuner/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10"
-OUTPUT_PATH = "./outputs"
-def setup_seeds():
-    seed = 42
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    cudnn.benchmark = False
-    cudnn.deterministic = True
-def main():
-    setup_seeds()
-    # logging
-    init_logger(OUTPUT_PATH)
-    # food examples
-    food_examples = load_json(FOOD_EXAMPLES)
-    agent = ConversationalAgent(model_path=MODEL_PATH,
-                                outputs_dir=OUTPUT_PATH)
-    theme = CustomTheme()
-    titles = [
-        """<center><B><font face="Comic Sans MS" size=10>书生大模型实战营</font></B></center>"""  ## Kalam:wght@700
-        """<center><B><font face="Courier" size=5>「进阶岛」InternVL 多模态模型部署微调实践</font></B></center>"""
-    ]
-    language = """Language: 中文 and English"""
-    with gr.Blocks(theme) as demo_chatbot:
-        for title in titles:
-            gr.Markdown(title)
-        # gr.Markdown(article)
-        gr.Markdown(language)
-        with gr.Row():
-            with gr.Column(scale=3):
-                start_btn = gr.Button("Start Chat", variant="primary", interactive=True)
-                clear_btn = gr.Button("Clear Context", interactive=False)
-                image = gr.Image(type="pil", interactive=False)
-                upload_btn = gr.Button("🖼️ Upload Image", interactive=False)
-                with gr.Accordion("Generation Settings"):
-                    top_p = gr.Slider(minimum=0, maximum=1, step=0.1,
-                                      value=0.8,
-                                      interactive=True,
-                                      label='top-p value',
-                                      visible=True)
-                    temperature = gr.Slider(minimum=0, maximum=1.5, step=0.1,
-                                            value=0.8,
-                                            interactive=True,
-                                            label='temperature',
-                                            visible=True)
-            with gr.Column(scale=7):
-                chat_state = gr.State()
-                chatbot = gr.Chatbot(label='InternVL2', height=800, avatar_images=((os.path.join(os.path.dirname(__file__), 'demo/user.png')), (os.path.join(os.path.dirname(__file__), "demo/bot.png"))))
-                text_input = gr.Textbox(label='User', placeholder="Please click the <Start Chat> button to start chat!", interactive=False)
-                gr.Markdown("### 输入示例")
-                def on_text_change(text):
-                    return gr.update(interactive=True)
-                text_input.change(fn=on_text_change, inputs=text_input, outputs=text_input)
-                gr.Examples(
-                    examples=[["图片中的食物通常属于哪个菜系?"],
-                              ["如果让你简单形容一下品尝图片中的食物的滋味，你会描述它"],
-                              ["去哪个地方游玩时应该品尝当地的特色美食图片中的食物?"],
-                              ["食用图片中的食物时，一般它上菜或摆盘时的特点是?"]],
-                    inputs=[text_input]
-                )
-        with gr.Row():
-            gr.Markdown("### 食物快捷栏")
-        with gr.Row():
-            example_xinjiang_food = gr.Examples(examples=food_examples["新疆菜"], inputs=image, label="新疆菜")
-            example_sichuan_food = gr.Examples(examples=food_examples["川菜（四川，重庆）"], inputs=image, label="川菜（四川，重庆）")
-            example_xibei_food = gr.Examples(examples=food_examples["西北菜 （陕西，甘肃等地）"], inputs=image, label="西北菜 （陕西，甘肃等地）")
-        with gr.Row():
-            example_guizhou_food = gr.Examples(examples=food_examples["黔菜 (贵州）"], inputs=image, label="黔菜 (贵州）")
-            example_jiangsu_food = gr.Examples(examples=food_examples["苏菜（江苏）"], inputs=image, label="苏菜（江苏）")
-            example_guangdong_food = gr.Examples(examples=food_examples["粤菜（广东等地）"], inputs=image, label="粤菜（广东等地）")
-        with gr.Row():
-            example_hunan_food = gr.Examples(examples=food_examples["湘菜（湖南）"], inputs=image, label="湘菜（湖南）")
-            example_fujian_food = gr.Examples(examples=food_examples["闽菜（福建）"], inputs=image, label="闽菜（福建）")
-            example_zhejiang_food = gr.Examples(examples=food_examples["浙菜（浙江）"], inputs=image, label="浙菜（浙江）")
-        with gr.Row():
-            example_dongbei_food = gr.Examples(examples=food_examples["东北菜 （黑龙江等地）"], inputs=image, label="东北菜 （黑龙江等地）")
-        start_btn.click(agent.start_chat, [chat_state], [text_input, start_btn, clear_btn, image, upload_btn, chat_state])
-        clear_btn.click(agent.restart_chat, [chat_state], [chatbot, text_input, start_btn, clear_btn, image, upload_btn, chat_state], queue=False)
-        upload_btn.click(agent.upload_image, [image, chatbot, chat_state], [image, chatbot, chat_state])
-        text_input.submit(
-            agent.respond,
-            inputs=[text_input, image, chatbot, top_p, temperature, chat_state],
-            outputs=[text_input, image, chatbot, chat_state]
-        )
-    demo_chatbot.launch(share=True, server_name="127.0.0.1", server_port=1096, allowed_paths=['./'])
-    demo_chatbot.queue()
-if __name__ == "__main__":
-    main()

InternVL101/configuration.json DELETED Viewed

	@@ -1 +0,0 @@
1	- {"framework":"other","task":"other"}

InternVL101/demo.py DELETED Viewed

@@ -1,119 +0,0 @@
-import os
-import random
-import numpy as np
-import torch
-import torch.backends.cudnn as cudnn
-import gradio as gr
-from utils import load_json, init_logger
-from demo import ConversationalAgent, CustomTheme
-FOOD_EXAMPLES = "demo/food_for_demo.json"
-MODEL_PATH = "/root/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10"
-# MODEL_PATH = "/root/xtuner/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10"
-OUTPUT_PATH = "./outputs"
-def setup_seeds():
-    seed = 42
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    cudnn.benchmark = False
-    cudnn.deterministic = True
-def main():
-    setup_seeds()
-    # logging
-    init_logger(OUTPUT_PATH)
-    # food examples
-    food_examples = load_json(FOOD_EXAMPLES)
-    agent = ConversationalAgent(model_path=MODEL_PATH,
-                                outputs_dir=OUTPUT_PATH)
-    theme = CustomTheme()
-    titles = [
-        """<center><B><font face="Comic Sans MS" size=10>书生大模型实战营</font></B></center>"""  ## Kalam:wght@700
-        """<center><B><font face="Courier" size=5>「进阶岛」InternVL 多模态模型部署微调实践</font></B></center>"""
-    ]
-    language = """Language: 中文 and English"""
-    with gr.Blocks(theme) as demo_chatbot:
-        for title in titles:
-            gr.Markdown(title)
-        # gr.Markdown(article)
-        gr.Markdown(language)
-        with gr.Row():
-            with gr.Column(scale=3):
-                start_btn = gr.Button("Start Chat", variant="primary", interactive=True)
-                clear_btn = gr.Button("Clear Context", interactive=False)
-                image = gr.Image(type="pil", interactive=False)
-                upload_btn = gr.Button("🖼️ Upload Image", interactive=False)
-                with gr.Accordion("Generation Settings"):
-                    top_p = gr.Slider(minimum=0, maximum=1, step=0.1,
-                                      value=0.8,
-                                      interactive=True,
-                                      label='top-p value',
-                                      visible=True)
-                    temperature = gr.Slider(minimum=0, maximum=1.5, step=0.1,
-                                            value=0.8,
-                                            interactive=True,
-                                            label='temperature',
-                                            visible=True)
-            with gr.Column(scale=7):
-                chat_state = gr.State()
-                chatbot = gr.Chatbot(label='InternVL2', height=800, avatar_images=((os.path.join(os.path.dirname(__file__), 'demo/user.png')), (os.path.join(os.path.dirname(__file__), "demo/bot.png"))))
-                text_input = gr.Textbox(label='User', placeholder="Please click the <Start Chat> button to start chat!", interactive=False)
-                gr.Markdown("### 输入示例")
-                def on_text_change(text):
-                    return gr.update(interactive=True)
-                text_input.change(fn=on_text_change, inputs=text_input, outputs=text_input)
-                gr.Examples(
-                    examples=[["图片中的食物通常属于哪个菜系?"],
-                              ["如果让你简单形容一下品尝图片中的食物的滋味，你会描述它"],
-                              ["去哪个地方游玩时应该品尝当地的特色美食图片中的食物?"],
-                              ["食用图片中的食物时，一般它上菜或摆盘时的特点是?"]],
-                    inputs=[text_input]
-                )
-        with gr.Row():
-            gr.Markdown("### 食物快捷栏")
-        with gr.Row():
-            example_xinjiang_food = gr.Examples(examples=food_examples["新疆菜"], inputs=image, label="新疆菜")
-            example_sichuan_food = gr.Examples(examples=food_examples["川菜（四川，重庆）"], inputs=image, label="川菜（四川，重庆）")
-            example_xibei_food = gr.Examples(examples=food_examples["西北菜 （陕西，甘肃等地）"], inputs=image, label="西北菜 （陕西，甘肃等地）")
-        with gr.Row():
-            example_guizhou_food = gr.Examples(examples=food_examples["黔菜 (贵州）"], inputs=image, label="黔菜 (贵州）")
-            example_jiangsu_food = gr.Examples(examples=food_examples["苏菜（江苏）"], inputs=image, label="苏菜（江苏）")
-            example_guangdong_food = gr.Examples(examples=food_examples["粤菜（广东等地）"], inputs=image, label="粤菜（广东等地）")
-        with gr.Row():
-            example_hunan_food = gr.Examples(examples=food_examples["湘菜（湖南）"], inputs=image, label="湘菜（湖南）")
-            example_fujian_food = gr.Examples(examples=food_examples["闽菜（福建）"], inputs=image, label="闽菜（福建）")
-            example_zhejiang_food = gr.Examples(examples=food_examples["浙菜（浙江）"], inputs=image, label="浙菜（浙江）")
-        with gr.Row():
-            example_dongbei_food = gr.Examples(examples=food_examples["东北菜 （黑龙江等地）"], inputs=image, label="东北菜 （黑龙江等地）")
-        start_btn.click(agent.start_chat, [chat_state], [text_input, start_btn, clear_btn, image, upload_btn, chat_state])
-        clear_btn.click(agent.restart_chat, [chat_state], [chatbot, text_input, start_btn, clear_btn, image, upload_btn, chat_state], queue=False)
-        upload_btn.click(agent.upload_image, [image, chatbot, chat_state], [image, chatbot, chat_state])
-        text_input.submit(
-            agent.respond,
-            inputs=[text_input, image, chatbot, top_p, temperature, chat_state],
-            outputs=[text_input, image, chatbot, chat_state]
-        )
-    demo_chatbot.launch(share=True, server_name="127.0.0.1", server_port=1096, allowed_paths=['./'])
-    demo_chatbot.queue()
-if __name__ == "__main__":
-    main()

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/.ipynb_checkpoints/internvl_v2_internlm2_2b_lora_finetune_food-checkpoint.py DELETED Viewed

@@ -1,139 +0,0 @@
-accumulative_counts = 2
-batch_size = 4
-betas = (
-    0.9,
-    0.999,
-)
-custom_hooks = [
-    dict(
-        tokenizer=dict(
-            pretrained_model_name_or_path=
-            '/root/share/new_models/OpenGVLab/InternVL2-2B',
-            trust_remote_code=True,
-            type='transformers.AutoTokenizer.from_pretrained'),
-        type='xtuner.engine.hooks.DatasetInfoHook'),
-]
-data_path = '/root/share/datasets/FoodieQA/sivqa_llava.json'
-data_root = '/root/share/datasets/FoodieQA/'
-dataloader_num_workers = 4
-default_hooks = dict(
-    checkpoint=dict(
-        by_epoch=False,
-        interval=64,
-        max_keep_ckpts=-1,
-        save_optimizer=False,
-        type='mmengine.hooks.CheckpointHook'),
-    logger=dict(
-        interval=10,
-        log_metric_by_epoch=False,
-        type='mmengine.hooks.LoggerHook'),
-    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
-    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
-    timer=dict(type='mmengine.hooks.IterTimerHook'))
-env_cfg = dict(
-    cudnn_benchmark=False,
-    dist_cfg=dict(backend='nccl'),
-    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
-image_folder = '/root/share/datasets/FoodieQA/'
-launcher = 'none'
-llava_dataset = dict(
-    data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-    image_folders='/root/share/datasets/FoodieQA/',
-    max_length=8192,
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-    type='xtuner.dataset.InternVL_V1_5_Dataset')
-load_from = None
-log_level = 'INFO'
-log_processor = dict(by_epoch=False)
-lr = 3e-05
-max_epochs = 10
-max_length = 8192
-max_norm = 1
-model = dict(
-    freeze_llm=True,
-    freeze_visual_encoder=True,
-    llm_lora=dict(
-        lora_alpha=256,
-        lora_dropout=0.05,
-        r=128,
-        target_modules=None,
-        task_type='CAUSAL_LM',
-        type='peft.LoraConfig'),
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    type='xtuner.model.InternVL_V1_5')
-optim_type = 'torch.optim.AdamW'
-optim_wrapper = dict(
-    optimizer=dict(
-        betas=(
-            0.9,
-            0.999,
-        ),
-        lr=3e-05,
-        type='torch.optim.AdamW',
-        weight_decay=0.05),
-    type='DeepSpeedOptimWrapper')
-param_scheduler = [
-    dict(
-        begin=0,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=0.3,
-        start_factor=1e-05,
-        type='mmengine.optim.LinearLR'),
-    dict(
-        begin=0.3,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=10,
-        eta_min=0.0,
-        type='mmengine.optim.CosineAnnealingLR'),
-]
-path = '/root/share/new_models/OpenGVLab/InternVL2-2B'
-prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat'
-randomness = dict(deterministic=False, seed=None)
-resume = False
-runner_type = 'FlexibleRunner'
-save_steps = 64
-save_total_limit = -1
-strategy = dict(
-    config=dict(
-        bf16=dict(enabled=True),
-        fp16=dict(enabled=False, initial_scale_power=16),
-        gradient_accumulation_steps='auto',
-        gradient_clipping='auto',
-        train_micro_batch_size_per_gpu='auto',
-        zero_allow_untested_optimizer=True,
-        zero_force_ds_cpu_optimizer=False,
-        zero_optimization=dict(overlap_comm=True, stage=2)),
-    exclude_frozen_parameters=True,
-    gradient_accumulation_steps=2,
-    gradient_clipping=1,
-    sequence_parallel_size=1,
-    train_micro_batch_size_per_gpu=4,
-    type='xtuner.engine.DeepSpeedStrategy')
-tokenizer = dict(
-    pretrained_model_name_or_path=
-    '/root/share/new_models/OpenGVLab/InternVL2-2B',
-    trust_remote_code=True,
-    type='transformers.AutoTokenizer.from_pretrained')
-train_cfg = dict(max_epochs=10, type='xtuner.engine.runner.TrainLoop')
-train_dataloader = dict(
-    batch_size=4,
-    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
-    dataset=dict(
-        data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-        image_folders='/root/share/datasets/FoodieQA/',
-        max_length=8192,
-        model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-        template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-        type='xtuner.dataset.InternVL_V1_5_Dataset'),
-    num_workers=4,
-    sampler=dict(
-        length_property='modality_length',
-        per_device_batch_size=8,
-        type='xtuner.dataset.samplers.LengthGroupedSampler'))
-visualizer = None
-warmup_ratio = 0.03
-weight_decay = 0.05
-work_dir = './work_dirs/internvl_v2_internlm2_2b_lora_finetune_food'

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/20241230_145825.log DELETED Viewed

@@ -1,517 +0,0 @@
-2024/12/30 14:58:26 - mmengine - INFO -
-------------------------------------------------------------
-System environment:
-    sys.platform: linux
-    Python: 3.10.16 (main, Dec 11 2024, 16:24:50) [GCC 11.2.0]
-    CUDA available: True
-    MUSA available: False
-    numpy_random_seed: 1369483566
-    GPU 0: NVIDIA A100-SXM4-80GB
-    CUDA_HOME: /usr/local/cuda
-    NVCC: Cuda compilation tools, release 12.2, V12.2.140
-    GCC: gcc (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0
-    PyTorch: 2.4.1+cu121
-    PyTorch compiling details: PyTorch built with:
-  - GCC 9.3
-  - C++ Version: 201703
-  - Intel(R) oneAPI Math Kernel Library Version 2022.2-Product Build 20220804 for Intel(R) 64 architecture applications
-  - Intel(R) MKL-DNN v3.4.2 (Git Hash 1137e04ec0b5251ca2b4400a4fd3c667ce843d67)
-  - OpenMP 201511 (a.k.a. OpenMP 4.5)
-  - LAPACK is enabled (usually provided by MKL)
-  - NNPACK is enabled
-  - CPU capability usage: AVX512
-  - CUDA Runtime 12.1
-  - NVCC architecture flags: -gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_90,code=sm_90
-  - CuDNN 90.1  (built against CUDA 12.4)
-  - Magma 2.6.1
-  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=12.1, CUDNN_VERSION=9.1.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-stringop-overflow -Wsuggest-override -Wno-psabi -Wno-error=pedantic -Wno-error=old-style-cast -Wno-missing-braces -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=2.4.1, USE_CUDA=ON, USE_CUDNN=ON, USE_CUSPARSELT=1, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_GLOO=ON, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, USE_ROCM_KERNEL_ASSERT=OFF,
-    TorchVision: 0.19.1+cu121
-    OpenCV: 4.10.0
-    MMEngine: 0.10.5
-Runtime environment:
-    launcher: none
-    randomness: {'seed': None, 'deterministic': False}
-    cudnn_benchmark: False
-    mp_cfg: {'mp_start_method': 'fork', 'opencv_num_threads': 0}
-    dist_cfg: {'backend': 'nccl'}
-    seed: None
-    deterministic: False
-    Distributed launcher: none
-    Distributed training: False
-    GPU number: 1
-------------------------------------------------------------
-2024/12/30 14:58:26 - mmengine - INFO - Config:
-accumulative_counts = 2
-batch_size = 4
-betas = (
-    0.9,
-    0.999,
-)
-custom_hooks = [
-    dict(
-        tokenizer=dict(
-            pretrained_model_name_or_path=
-            '/root/share/new_models/OpenGVLab/InternVL2-2B',
-            trust_remote_code=True,
-            type='transformers.AutoTokenizer.from_pretrained'),
-        type='xtuner.engine.hooks.DatasetInfoHook'),
-]
-data_path = '/root/share/datasets/FoodieQA/sivqa_llava.json'
-data_root = '/root/share/datasets/FoodieQA/'
-dataloader_num_workers = 4
-default_hooks = dict(
-    checkpoint=dict(
-        by_epoch=False,
-        interval=64,
-        max_keep_ckpts=-1,
-        save_optimizer=False,
-        type='mmengine.hooks.CheckpointHook'),
-    logger=dict(
-        interval=10,
-        log_metric_by_epoch=False,
-        type='mmengine.hooks.LoggerHook'),
-    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
-    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
-    timer=dict(type='mmengine.hooks.IterTimerHook'))
-env_cfg = dict(
-    cudnn_benchmark=False,
-    dist_cfg=dict(backend='nccl'),
-    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
-image_folder = '/root/share/datasets/FoodieQA/'
-launcher = 'none'
-llava_dataset = dict(
-    data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-    image_folders='/root/share/datasets/FoodieQA/',
-    max_length=8192,
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-    type='xtuner.dataset.InternVL_V1_5_Dataset')
-load_from = None
-log_level = 'INFO'
-log_processor = dict(by_epoch=False)
-lr = 3e-05
-max_epochs = 10
-max_length = 8192
-max_norm = 1
-model = dict(
-    freeze_llm=True,
-    freeze_visual_encoder=True,
-    llm_lora=dict(
-        lora_alpha=256,
-        lora_dropout=0.05,
-        r=128,
-        target_modules=None,
-        task_type='CAUSAL_LM',
-        type='peft.LoraConfig'),
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    type='xtuner.model.InternVL_V1_5')
-optim_type = 'torch.optim.AdamW'
-optim_wrapper = dict(
-    optimizer=dict(
-        betas=(
-            0.9,
-            0.999,
-        ),
-        lr=3e-05,
-        type='torch.optim.AdamW',
-        weight_decay=0.05),
-    type='DeepSpeedOptimWrapper')
-param_scheduler = [
-    dict(
-        begin=0,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=0.3,
-        start_factor=1e-05,
-        type='mmengine.optim.LinearLR'),
-    dict(
-        begin=0.3,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=10,
-        eta_min=0.0,
-        type='mmengine.optim.CosineAnnealingLR'),
-]
-path = '/root/share/new_models/OpenGVLab/InternVL2-2B'
-prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat'
-randomness = dict(deterministic=False, seed=None)
-resume = False
-runner_type = 'FlexibleRunner'
-save_steps = 64
-save_total_limit = -1
-strategy = dict(
-    config=dict(
-        bf16=dict(enabled=True),
-        fp16=dict(enabled=False, initial_scale_power=16),
-        gradient_accumulation_steps='auto',
-        gradient_clipping='auto',
-        train_micro_batch_size_per_gpu='auto',
-        zero_allow_untested_optimizer=True,
-        zero_force_ds_cpu_optimizer=False,
-        zero_optimization=dict(overlap_comm=True, stage=2)),
-    exclude_frozen_parameters=True,
-    gradient_accumulation_steps=2,
-    gradient_clipping=1,
-    sequence_parallel_size=1,
-    train_micro_batch_size_per_gpu=4,
-    type='xtuner.engine.DeepSpeedStrategy')
-tokenizer = dict(
-    pretrained_model_name_or_path=
-    '/root/share/new_models/OpenGVLab/InternVL2-2B',
-    trust_remote_code=True,
-    type='transformers.AutoTokenizer.from_pretrained')
-train_cfg = dict(max_epochs=10, type='xtuner.engine.runner.TrainLoop')
-train_dataloader = dict(
-    batch_size=4,
-    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
-    dataset=dict(
-        data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-        image_folders='/root/share/datasets/FoodieQA/',
-        max_length=8192,
-        model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-        template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-        type='xtuner.dataset.InternVL_V1_5_Dataset'),
-    num_workers=4,
-    sampler=dict(
-        length_property='modality_length',
-        per_device_batch_size=8,
-        type='xtuner.dataset.samplers.LengthGroupedSampler'))
-visualizer = None
-warmup_ratio = 0.03
-weight_decay = 0.05
-work_dir = './work_dirs/internvl_v2_internlm2_2b_lora_finetune_food'
-2024/12/30 14:58:26 - mmengine - WARNING - Failed to search registry with scope "mmengine" in the "builder" registry tree. As a workaround, the current "builder" registry in "xtuner" is used to build instance. This may cause unexpected failure when running the built modules. Please check whether "mmengine" is a correct scope, or whether the registry is initialized.
-2024/12/30 14:58:27 - mmengine - INFO - Hooks will be executed in the following order:
-before_run:
-(VERY_HIGH   ) RuntimeInfoHook
-(BELOW_NORMAL) LoggerHook
- --------------------
-before_train:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
-(NORMAL      ) DatasetInfoHook
-(VERY_LOW    ) CheckpointHook
- --------------------
-before_train_epoch:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
-(NORMAL      ) DistSamplerSeedHook
- --------------------
-before_train_iter:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
- --------------------
-after_train_iter:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
-(BELOW_NORMAL) LoggerHook
-(LOW         ) ParamSchedulerHook
-(VERY_LOW    ) CheckpointHook
- --------------------
-after_train_epoch:
-(NORMAL      ) IterTimerHook
-(LOW         ) ParamSchedulerHook
-(VERY_LOW    ) CheckpointHook
- --------------------
-before_val:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) DatasetInfoHook
- --------------------
-before_val_epoch:
-(NORMAL      ) IterTimerHook
- --------------------
-before_val_iter:
-(NORMAL      ) IterTimerHook
- --------------------
-after_val_iter:
-(NORMAL      ) IterTimerHook
-(BELOW_NORMAL) LoggerHook
- --------------------
-after_val_epoch:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
-(BELOW_NORMAL) LoggerHook
-(LOW         ) ParamSchedulerHook
-(VERY_LOW    ) CheckpointHook
- --------------------
-after_val:
-(VERY_HIGH   ) RuntimeInfoHook
- --------------------
-after_train:
-(VERY_HIGH   ) RuntimeInfoHook
-(VERY_LOW    ) CheckpointHook
- --------------------
-before_test:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) DatasetInfoHook
- --------------------
-before_test_epoch:
-(NORMAL      ) IterTimerHook
- --------------------
-before_test_iter:
-(NORMAL      ) IterTimerHook
- --------------------
-after_test_iter:
-(NORMAL      ) IterTimerHook
-(BELOW_NORMAL) LoggerHook
- --------------------
-after_test_epoch:
-(VERY_HIGH   ) RuntimeInfoHook
-(NORMAL      ) IterTimerHook
-(BELOW_NORMAL) LoggerHook
- --------------------
-after_test:
-(VERY_HIGH   ) RuntimeInfoHook
- --------------------
-after_run:
-(BELOW_NORMAL) LoggerHook
- --------------------
-2024/12/30 14:58:27 - mmengine - INFO - Starting to loading data and calc length
-2024/12/30 14:58:27 - mmengine - INFO - =======Starting to process /root/share/datasets/FoodieQA/sivqa_llava.json =======
-2024/12/30 14:58:27 - mmengine - INFO - =======total 256 samples of /root/share/datasets/FoodieQA/sivqa_llava.json=======
-2024/12/30 14:58:27 - mmengine - INFO - end loading data and calc length
-2024/12/30 14:58:27 - mmengine - INFO - =======total 256 samples=======
-2024/12/30 14:58:27 - mmengine - INFO - LengthGroupedSampler is used.
-2024/12/30 14:58:27 - mmengine - INFO - LengthGroupedSampler construction is complete, and the selected attribute is modality_length
-2024/12/30 14:58:27 - mmengine - WARNING - Dataset InternVL_V1_5_Dataset has no metainfo. ``dataset_meta`` in visualizer will be None.
-2024/12/30 14:58:28 - mmengine - INFO - Start to load InternVL_V1_5 model.
-2024/12/30 14:58:55 - mmengine - INFO - InternVL_V1_5(
-  (data_preprocessor): BaseDataPreprocessor()
-  (model): InternVLChatModel(
-    (vision_model): InternVisionModel(
-      (embeddings): InternVisionEmbeddings(
-        (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))
-      )
-      (encoder): InternVisionEncoder(
-        (layers): ModuleList(
-          (0-23): 24 x InternVisionEncoderLayer(
-            (attn): InternAttention(
-              (qkv): Linear(in_features=1024, out_features=3072, bias=True)
-              (attn_drop): Dropout(p=0.0, inplace=False)
-              (proj_drop): Dropout(p=0.0, inplace=False)
-              (proj): Linear(in_features=1024, out_features=1024, bias=True)
-            )
-            (mlp): InternMLP(
-              (act): GELUActivation()
-              (fc1): Linear(in_features=1024, out_features=4096, bias=True)
-              (fc2): Linear(in_features=4096, out_features=1024, bias=True)
-            )
-            (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
-            (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)
-            (drop_path1): Identity()
-            (drop_path2): Identity()
-          )
-        )
-      )
-    )
-    (language_model): PeftModelForCausalLM(
-      (base_model): LoraModel(
-        (model): InternLM2ForCausalLM(
-          (model): InternLM2Model(
-            (tok_embeddings): Embedding(92553, 2048, padding_idx=2)
-            (layers): ModuleList(
-              (0-23): 24 x InternLM2DecoderLayer(
-                (attention): InternLM2Attention(
-                  (wqkv): lora.Linear(
-                    (base_layer): Linear(in_features=2048, out_features=4096, bias=False)
-                    (lora_dropout): ModuleDict(
-                      (default): Dropout(p=0.05, inplace=False)
-                    )
-                    (lora_A): ModuleDict(
-                      (default): Linear(in_features=2048, out_features=128, bias=False)
-                    )
-                    (lora_B): ModuleDict(
-                      (default): Linear(in_features=128, out_features=4096, bias=False)
-                    )
-                    (lora_embedding_A): ParameterDict()
-                    (lora_embedding_B): ParameterDict()
-                    (lora_magnitude_vector): ModuleDict()
-                  )
-                  (wo): lora.Linear(
-                    (base_layer): Linear(in_features=2048, out_features=2048, bias=False)
-                    (lora_dropout): ModuleDict(
-                      (default): Dropout(p=0.05, inplace=False)
-                    )
-                    (lora_A): ModuleDict(
-                      (default): Linear(in_features=2048, out_features=128, bias=False)
-                    )
-                    (lora_B): ModuleDict(
-                      (default): Linear(in_features=128, out_features=2048, bias=False)
-                    )
-                    (lora_embedding_A): ParameterDict()
-                    (lora_embedding_B): ParameterDict()
-                    (lora_magnitude_vector): ModuleDict()
-                  )
-                  (rotary_emb): InternLM2DynamicNTKScalingRotaryEmbedding()
-                )
-                (feed_forward): InternLM2MLP(
-                  (w1): lora.Linear(
-                    (base_layer): Linear(in_features=2048, out_features=8192, bias=False)
-                    (lora_dropout): ModuleDict(
-                      (default): Dropout(p=0.05, inplace=False)
-                    )
-                    (lora_A): ModuleDict(
-                      (default): Linear(in_features=2048, out_features=128, bias=False)
-                    )
-                    (lora_B): ModuleDict(
-                      (default): Linear(in_features=128, out_features=8192, bias=False)
-                    )
-                    (lora_embedding_A): ParameterDict()
-                    (lora_embedding_B): ParameterDict()
-                    (lora_magnitude_vector): ModuleDict()
-                  )
-                  (w3): lora.Linear(
-                    (base_layer): Linear(in_features=2048, out_features=8192, bias=False)
-                    (lora_dropout): ModuleDict(
-                      (default): Dropout(p=0.05, inplace=False)
-                    )
-                    (lora_A): ModuleDict(
-                      (default): Linear(in_features=2048, out_features=128, bias=False)
-                    )
-                    (lora_B): ModuleDict(
-                      (default): Linear(in_features=128, out_features=8192, bias=False)
-                    )
-                    (lora_embedding_A): ParameterDict()
-                    (lora_embedding_B): ParameterDict()
-                    (lora_magnitude_vector): ModuleDict()
-                  )
-                  (w2): lora.Linear(
-                    (base_layer): Linear(in_features=8192, out_features=2048, bias=False)
-                    (lora_dropout): ModuleDict(
-                      (default): Dropout(p=0.05, inplace=False)
-                    )
-                    (lora_A): ModuleDict(
-                      (default): Linear(in_features=8192, out_features=128, bias=False)
-                    )
-                    (lora_B): ModuleDict(
-                      (default): Linear(in_features=128, out_features=2048, bias=False)
-                    )
-                    (lora_embedding_A): ParameterDict()
-                    (lora_embedding_B): ParameterDict()
-                    (lora_magnitude_vector): ModuleDict()
-                  )
-                  (act_fn): SiLU()
-                )
-                (attention_norm): InternLM2RMSNorm()
-                (ffn_norm): InternLM2RMSNorm()
-              )
-            )
-            (norm): InternLM2RMSNorm()
-          )
-          (output): lora.Linear(
-            (base_layer): Linear(in_features=2048, out_features=92553, bias=False)
-            (lora_dropout): ModuleDict(
-              (default): Dropout(p=0.05, inplace=False)
-            )
-            (lora_A): ModuleDict(
-              (default): Linear(in_features=2048, out_features=128, bias=False)
-            )
-            (lora_B): ModuleDict(
-              (default): Linear(in_features=128, out_features=92553, bias=False)
-            )
-            (lora_embedding_A): ParameterDict()
-            (lora_embedding_B): ParameterDict()
-            (lora_magnitude_vector): ModuleDict()
-          )
-        )
-      )
-    )
-    (mlp1): Sequential(
-      (0): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)
-      (1): Linear(in_features=4096, out_features=2048, bias=True)
-      (2): GELU(approximate='none')
-      (3): Linear(in_features=2048, out_features=2048, bias=True)
-    )
-  )
-)
-2024/12/30 14:58:55 - mmengine - INFO - InternVL_V1_5 construction is complete
-2024/12/30 14:59:14 - mmengine - INFO - Num train samples 256
-2024/12/30 14:59:14 - mmengine - INFO - train example:
-2024/12/30 14:59:15 - mmengine - INFO -  <s><|im_start|> system
-You are an AI assistant whose name is InternLM (书生·浦语).<|im_end|><|im_start|>user
- <img> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> <IMG_CONTEXT> </img>
-图片中的食物通常属于哪个菜系?<|im_end|><|im_start|> assistant
-新疆菜，图中的菜是烤羊肉串<|im_end|>
-2024/12/30 14:59:15 - mmengine - WARNING - "FileClient" will be deprecated in future. Please use io functions in https://mmengine.readthedocs.io/en/latest/api/fileio.html#file-io
-2024/12/30 14:59:15 - mmengine - WARNING - "HardDiskBackend" is the alias of "LocalBackend" and the former will be deprecated in future.
-2024/12/30 14:59:15 - mmengine - INFO - Checkpoints will be saved to /root/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food.
-2024/12/30 15:00:56 - mmengine - INFO - Iter(train) [ 10/640]  lr: 1.5000e-05  eta: 1:45:35  time: 10.0558  data_time: 0.0208  memory: 25140  loss: 5.2053
-2024/12/30 15:02:19 - mmengine - INFO - Iter(train) [ 20/640]  lr: 3.0000e-05  eta: 1:34:54  time: 8.3120  data_time: 0.0361  memory: 25130  loss: 2.9355
-2024/12/30 15:03:41 - mmengine - INFO - Iter(train) [ 30/640]  lr: 2.9981e-05  eta: 1:30:06  time: 8.2209  data_time: 0.0271  memory: 25135  loss: 1.6512
-2024/12/30 15:05:04 - mmengine - INFO - Iter(train) [ 40/640]  lr: 2.9923e-05  eta: 1:27:05  time: 8.2512  data_time: 0.0260  memory: 25158  loss: 1.2375
-2024/12/30 15:06:28 - mmengine - INFO - Iter(train) [ 50/640]  lr: 2.9828e-05  eta: 1:25:00  time: 8.3884  data_time: 0.0271  memory: 25135  loss: 0.8768
-2024/12/30 15:07:51 - mmengine - INFO - Iter(train) [ 60/640]  lr: 2.9694e-05  eta: 1:23:05  time: 8.3412  data_time: 0.0290  memory: 25144  loss: 0.9339
-2024/12/30 15:08:25 - mmengine - INFO - Exp name: internvl_v2_internlm2_2b_lora_finetune_food_20241230_145825
-2024/12/30 15:08:25 - mmengine - INFO - Saving checkpoint at 64 iterations
-2024/12/30 15:08:27 - mmengine - WARNING - Reach the end of the dataloader, it will be restarted and continue to iterate. It is recommended to use `mmengine.dataset.InfiniteSampler` to enable the dataloader to iterate infinitely.
-2024/12/30 15:09:21 - mmengine - INFO - Iter(train) [ 70/640]  lr: 2.9523e-05  eta: 1:22:12  time: 9.0074  data_time: 0.6512  memory: 25149  loss: 0.5485
-2024/12/30 15:10:43 - mmengine - INFO - Iter(train) [ 80/640]  lr: 2.9314e-05  eta: 1:20:14  time: 8.1989  data_time: 0.0244  memory: 25144  loss: 0.4231
-2024/12/30 15:12:05 - mmengine - INFO - Iter(train) [ 90/640]  lr: 2.9069e-05  eta: 1:18:25  time: 8.2213  data_time: 0.0265  memory: 25174  loss: 0.4846
-2024/12/30 15:13:25 - mmengine - INFO - Iter(train) [100/640]  lr: 2.8788e-05  eta: 1:16:29  time: 7.9919  data_time: 0.0247  memory: 25158  loss: 0.3924
-2024/12/30 15:14:49 - mmengine - INFO - Iter(train) [110/640]  lr: 2.8472e-05  eta: 1:14:57  time: 8.3518  data_time: 0.0260  memory: 25140  loss: 0.3984
-2024/12/30 15:16:12 - mmengine - INFO - Iter(train) [120/640]  lr: 2.8121e-05  eta: 1:13:25  time: 8.3317  data_time: 0.0264  memory: 25135  loss: 0.4056
-2024/12/30 15:17:19 - mmengine - INFO - Saving checkpoint at 128 iterations
-2024/12/30 15:17:50 - mmengine - INFO - Iter(train) [130/640]  lr: 2.7737e-05  eta: 1:12:53  time: 9.8130  data_time: 1.4211  memory: 25174  loss: 0.4113
-2024/12/30 15:19:13 - mmengine - INFO - Iter(train) [140/640]  lr: 2.7320e-05  eta: 1:11:17  time: 8.2921  data_time: 0.0306  memory: 25140  loss: 0.1910
-2024/12/30 15:20:35 - mmengine - INFO - Iter(train) [150/640]  lr: 2.6871e-05  eta: 1:09:41  time: 8.2371  data_time: 0.0313  memory: 25135  loss: 0.1305
-2024/12/30 15:21:58 - mmengine - INFO - Iter(train) [160/640]  lr: 2.6393e-05  eta: 1:08:09  time: 8.2850  data_time: 0.0298  memory: 25140  loss: 0.2068
-2024/12/30 15:23:21 - mmengine - INFO - Iter(train) [170/640]  lr: 2.5885e-05  eta: 1:06:37  time: 8.2801  data_time: 0.0290  memory: 25140  loss: 0.1408
-2024/12/30 15:24:45 - mmengine - INFO - Iter(train) [180/640]  lr: 2.5349e-05  eta: 1:05:08  time: 8.3631  data_time: 0.0311  memory: 25125  loss: 0.1690
-2024/12/30 15:26:08 - mmengine - INFO - Iter(train) [190/640]  lr: 2.4786e-05  eta: 1:03:39  time: 8.3238  data_time: 0.0293  memory: 25144  loss: 0.1948
-2024/12/30 15:26:25 - mmengine - INFO - Saving checkpoint at 192 iterations
-2024/12/30 15:27:37 - mmengine - INFO - Iter(train) [200/640]  lr: 2.4199e-05  eta: 1:02:22  time: 8.8643  data_time: 0.6447  memory: 25140  loss: 0.0559
-2024/12/30 15:28:59 - mmengine - INFO - Iter(train) [210/640]  lr: 2.3588e-05  eta: 1:00:53  time: 8.2712  data_time: 0.0301  memory: 25174  loss: 0.0558
-2024/12/30 15:30:26 - mmengine - INFO - Iter(train) [220/640]  lr: 2.2955e-05  eta: 0:59:31  time: 8.6658  data_time: 0.1482  memory: 25135  loss: 0.0843
-2024/12/30 15:31:50 - mmengine - INFO - Iter(train) [230/640]  lr: 2.2302e-05  eta: 0:58:04  time: 8.3904  data_time: 0.0266  memory: 25158  loss: 0.0581
-2024/12/30 15:33:14 - mmengine - INFO - Iter(train) [240/640]  lr: 2.1630e-05  eta: 0:56:38  time: 8.4367  data_time: 0.0307  memory: 25140  loss: 0.0512
-2024/12/30 15:34:38 - mmengine - INFO - Iter(train) [250/640]  lr: 2.0941e-05  eta: 0:55:10  time: 8.3334  data_time: 0.0260  memory: 25144  loss: 0.0549
-2024/12/30 15:35:28 - mmengine - INFO - Saving checkpoint at 256 iterations
-2024/12/30 15:36:08 - mmengine - INFO - Iter(train) [260/640]  lr: 2.0237e-05  eta: 0:53:54  time: 9.0811  data_time: 0.7423  memory: 25158  loss: 0.0478
-2024/12/30 15:37:32 - mmengine - INFO - Iter(train) [270/640]  lr: 1.9520e-05  eta: 0:52:27  time: 8.3447  data_time: 0.0310  memory: 25140  loss: 0.0144
-2024/12/30 15:38:55 - mmengine - INFO - Iter(train) [280/640]  lr: 1.8791e-05  eta: 0:50:59  time: 8.3361  data_time: 0.0309  memory: 25144  loss: 0.0216
-2024/12/30 15:40:18 - mmengine - INFO - Iter(train) [290/640]  lr: 1.8052e-05  eta: 0:49:32  time: 8.2659  data_time: 0.0294  memory: 25135  loss: 0.0485
-2024/12/30 15:41:40 - mmengine - INFO - Iter(train) [300/640]  lr: 1.7305e-05  eta: 0:48:04  time: 8.2496  data_time: 0.0327  memory: 25158  loss: 0.0397
-2024/12/30 15:43:03 - mmengine - INFO - Iter(train) [310/640]  lr: 1.6553e-05  eta: 0:46:37  time: 8.2489  data_time: 0.0281  memory: 25154  loss: 0.0201
-2024/12/30 15:44:26 - mmengine - INFO - Iter(train) [320/640]  lr: 1.5796e-05  eta: 0:45:10  time: 8.3446  data_time: 0.0301  memory: 25135  loss: 0.0355
-2024/12/30 15:44:27 - mmengine - INFO - Saving checkpoint at 320 iterations
-2024/12/30 15:45:59 - mmengine - INFO - Iter(train) [330/640]  lr: 1.5038e-05  eta: 0:43:54  time: 9.3116  data_time: 1.0012  memory: 25116  loss: 0.0034
-2024/12/30 15:47:23 - mmengine - INFO - Iter(train) [340/640]  lr: 1.4279e-05  eta: 0:42:27  time: 8.3089  data_time: 0.0299  memory: 25121  loss: 0.0041
-2024/12/30 15:48:46 - mmengine - INFO - Iter(train) [350/640]  lr: 1.3523e-05  eta: 0:41:01  time: 8.3649  data_time: 0.0296  memory: 25140  loss: 0.0037
-2024/12/30 15:50:10 - mmengine - INFO - Iter(train) [360/640]  lr: 1.2770e-05  eta: 0:39:35  time: 8.3647  data_time: 0.0283  memory: 25158  loss: 0.0222
-2024/12/30 15:51:35 - mmengine - INFO - Iter(train) [370/640]  lr: 1.2022e-05  eta: 0:38:10  time: 8.4992  data_time: 0.0547  memory: 25140  loss: 0.0122
-2024/12/30 15:53:00 - mmengine - INFO - Iter(train) [380/640]  lr: 1.1283e-05  eta: 0:36:46  time: 8.4771  data_time: 0.0360  memory: 25140  loss: 0.0112
-2024/12/30 15:53:33 - mmengine - INFO - Saving checkpoint at 384 iterations
-2024/12/30 15:54:30 - mmengine - INFO - Iter(train) [390/640]  lr: 1.0553e-05  eta: 0:35:24  time: 9.0590  data_time: 0.7136  memory: 25140  loss: 0.0016
-2024/12/30 15:55:55 - mmengine - INFO - Iter(train) [400/640]  lr: 9.8341e-06  eta: 0:33:59  time: 8.4440  data_time: 0.0298  memory: 25144  loss: 0.0017
-2024/12/30 15:57:19 - mmengine - INFO - Iter(train) [410/640]  lr: 9.1286e-06  eta: 0:32:34  time: 8.3998  data_time: 0.0293  memory: 25140  loss: 0.0007
-2024/12/30 15:58:41 - mmengine - INFO - Iter(train) [420/640]  lr: 8.4381e-06  eta: 0:31:07  time: 8.2591  data_time: 0.0287  memory: 25135  loss: 0.0004
-2024/12/30 16:00:05 - mmengine - INFO - Iter(train) [430/640]  lr: 7.7644e-06  eta: 0:29:42  time: 8.3715  data_time: 0.0271  memory: 25158  loss: 0.0116
-2024/12/30 16:01:27 - mmengine - INFO - Iter(train) [440/640]  lr: 7.1092e-06  eta: 0:28:16  time: 8.2560  data_time: 0.0318  memory: 25158  loss: 0.0004
-2024/12/30 16:02:34 - mmengine - INFO - Saving checkpoint at 448 iterations
-2024/12/30 16:02:58 - mmengine - INFO - Iter(train) [450/640]  lr: 6.4742e-06  eta: 0:26:54  time: 9.0731  data_time: 0.8085  memory: 25158  loss: 0.0010
-2024/12/30 16:04:21 - mmengine - INFO - Iter(train) [460/640]  lr: 5.8611e-06  eta: 0:25:28  time: 8.2784  data_time: 0.0277  memory: 25140  loss: 0.0006
-2024/12/30 16:05:43 - mmengine - INFO - Iter(train) [470/640]  lr: 5.2713e-06  eta: 0:24:02  time: 8.2471  data_time: 0.0323  memory: 25144  loss: 0.0004
-2024/12/30 16:07:07 - mmengine - INFO - Iter(train) [480/640]  lr: 4.7064e-06  eta: 0:22:37  time: 8.3357  data_time: 0.0299  memory: 25144  loss: 0.0003
-2024/12/30 16:08:31 - mmengine - INFO - Iter(train) [490/640]  lr: 4.1678e-06  eta: 0:21:12  time: 8.3910  data_time: 0.0467  memory: 25154  loss: 0.0003
-2024/12/30 16:09:53 - mmengine - INFO - Iter(train) [500/640]  lr: 3.6570e-06  eta: 0:19:46  time: 8.2112  data_time: 0.0303  memory: 25102  loss: 0.0085
-2024/12/30 16:11:16 - mmengine - INFO - Iter(train) [510/640]  lr: 3.1752e-06  eta: 0:18:21  time: 8.3088  data_time: 0.0284  memory: 25140  loss: 0.0005
-2024/12/30 16:11:32 - mmengine - INFO - Saving checkpoint at 512 iterations
-2024/12/30 16:12:47 - mmengine - INFO - Iter(train) [520/640]  lr: 2.7236e-06  eta: 0:16:58  time: 9.1451  data_time: 0.7830  memory: 25154  loss: 0.0002
-2024/12/30 16:14:09 - mmengine - INFO - Iter(train) [530/640]  lr: 2.3035e-06  eta: 0:15:32  time: 8.1316  data_time: 0.0280  memory: 25135  loss: 0.0003
-2024/12/30 16:15:29 - mmengine - INFO - Iter(train) [540/640]  lr: 1.9158e-06  eta: 0:14:06  time: 8.0338  data_time: 0.0279  memory: 25158  loss: 0.0004
-2024/12/30 16:16:51 - mmengine - INFO - Iter(train) [550/640]  lr: 1.5616e-06  eta: 0:12:41  time: 8.1674  data_time: 0.0274  memory: 25149  loss: 0.0003
-2024/12/30 16:18:14 - mmengine - INFO - Iter(train) [560/640]  lr: 1.2418e-06  eta: 0:11:16  time: 8.3422  data_time: 0.0255  memory: 25140  loss: 0.0003
-2024/12/30 16:19:39 - mmengine - INFO - Iter(train) [570/640]  lr: 9.5724e-07  eta: 0:09:52  time: 8.4454  data_time: 0.0324  memory: 25144  loss: 0.0002
-2024/12/30 16:20:29 - mmengine - INFO - Saving checkpoint at 576 iterations
-2024/12/30 16:21:11 - mmengine - INFO - Iter(train) [580/640]  lr: 7.0858e-07  eta: 0:08:28  time: 9.2532  data_time: 0.7713  memory: 25130  loss: 0.0003
-2024/12/30 16:22:37 - mmengine - INFO - Iter(train) [590/640]  lr: 4.9649e-07  eta: 0:07:03  time: 8.5854  data_time: 0.0300  memory: 25135  loss: 0.0002
-2024/12/30 16:24:02 - mmengine - INFO - Iter(train) [600/640]  lr: 3.2151e-07  eta: 0:05:39  time: 8.5478  data_time: 0.0282  memory: 25140  loss: 0.0003
-2024/12/30 16:25:27 - mmengine - INFO - Iter(train) [610/640]  lr: 1.8408e-07  eta: 0:04:14  time: 8.4821  data_time: 0.0291  memory: 25135  loss: 0.0004
-2024/12/30 16:26:52 - mmengine - INFO - Iter(train) [620/640]  lr: 8.4568e-08  eta: 0:02:49  time: 8.4243  data_time: 0.0291  memory: 25149  loss: 0.0003
-2024/12/30 16:28:14 - mmengine - INFO - Iter(train) [630/640]  lr: 2.3219e-08  eta: 0:01:24  time: 8.2402  data_time: 0.0280  memory: 25144  loss: 0.0003
-2024/12/30 16:29:35 - mmengine - INFO - Iter(train) [640/640]  lr: 1.9195e-10  eta: 0:00:00  time: 8.1402  data_time: 0.0300  memory: 25154  loss: 0.0002
-2024/12/30 16:29:35 - mmengine - INFO - Saving checkpoint at 640 iterations

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/20241230_145825.json DELETED Viewed

@@ -1,64 +0,0 @@
-{"lr": 1.500015e-05, "data_time": 0.020780611038208007, "loss": 5.205293273925781, "time": 10.055773782730103, "iter": 10, "memory": 25140, "step": 10}
-{"lr": 3e-05, "data_time": 0.03608603477478027, "loss": 2.9355292201042174, "time": 8.311998891830445, "iter": 20, "memory": 25130, "step": 20}
-{"lr": 2.99808095489134e-05, "data_time": 0.027062010765075684, "loss": 1.6511537075042724, "time": 8.220871329307556, "iter": 30, "memory": 25135, "step": 30}
-{"lr": 2.9923287298775314e-05, "data_time": 0.02596433162689209, "loss": 1.2374991178512573, "time": 8.251240038871765, "iter": 40, "memory": 25158, "step": 40}
-{"lr": 2.9827580433309446e-05, "data_time": 0.027050209045410157, "loss": 0.8767685353755951, "time": 8.38843400478363, "iter": 50, "memory": 25135, "step": 50}
-{"lr": 2.9693933840238504e-05, "data_time": 0.02903637886047363, "loss": 0.9338608592748642, "time": 8.341249108314514, "iter": 60, "memory": 25144, "step": 60}
-{"lr": 2.952268948468346e-05, "data_time": 0.6511948108673096, "loss": 0.5485243916511535, "time": 9.00740897655487, "iter": 70, "memory": 25149, "step": 70}
-{"lr": 2.9314285534168186e-05, "data_time": 0.024419164657592772, "loss": 0.42309151738882067, "time": 8.19894368648529, "iter": 80, "memory": 25144, "step": 80}
-{"lr": 2.90692552374685e-05, "data_time": 0.026521754264831544, "loss": 0.48458234667778016, "time": 8.221326875686646, "iter": 90, "memory": 25174, "step": 90}
-{"lr": 2.8788225560174216e-05, "data_time": 0.024730491638183593, "loss": 0.3923974633216858, "time": 7.991903710365295, "iter": 100, "memory": 25158, "step": 100}
-{"lr": 2.847191558045545e-05, "data_time": 0.026031899452209472, "loss": 0.398375840485096, "time": 8.351808667182922, "iter": 110, "memory": 25140, "step": 110}
-{"lr": 2.8121134649138086e-05, "data_time": 0.026431798934936523, "loss": 0.40561383217573166, "time": 8.331733465194702, "iter": 120, "memory": 25135, "step": 120}
-{"lr": 2.7736780318796056e-05, "data_time": 1.4211087226867676, "loss": 0.41130477637052537, "time": 9.81301429271698, "iter": 130, "memory": 25174, "step": 130}
-{"lr": 2.7319836047159543e-05, "data_time": 0.03063185214996338, "loss": 0.1909895658493042, "time": 8.292147588729858, "iter": 140, "memory": 25140, "step": 140}
-{"lr": 2.68713686807153e-05, "data_time": 0.03131427764892578, "loss": 0.13053411729633807, "time": 8.237112450599671, "iter": 150, "memory": 25135, "step": 150}
-{"lr": 2.639252572493797e-05, "data_time": 0.02983086109161377, "loss": 0.20683301314711572, "time": 8.285038375854493, "iter": 160, "memory": 25140, "step": 160}
-{"lr": 2.5884532408136998e-05, "data_time": 0.028959155082702637, "loss": 0.14075656346976756, "time": 8.280118131637574, "iter": 170, "memory": 25140, "step": 170}
-{"lr": 2.534868854643217e-05, "data_time": 0.03114640712738037, "loss": 0.16897681429982186, "time": 8.363134860992432, "iter": 180, "memory": 25125, "step": 180}
-{"lr": 2.4786365217879254e-05, "data_time": 0.029333734512329103, "loss": 0.1947656821459532, "time": 8.32378842830658, "iter": 190, "memory": 25144, "step": 190}
-{"lr": 2.419900125425576e-05, "data_time": 0.6446581840515136, "loss": 0.05592906894162297, "time": 8.864284300804139, "iter": 200, "memory": 25140, "step": 200}
-{"lr": 2.3588099559483543e-05, "data_time": 0.030133438110351563, "loss": 0.055754791107028724, "time": 8.271190738677978, "iter": 210, "memory": 25174, "step": 210}
-{"lr": 2.2955223264108254e-05, "data_time": 0.14820406436920167, "loss": 0.08434767709113658, "time": 8.665787267684937, "iter": 220, "memory": 25135, "step": 220}
-{"lr": 2.2301991725675243e-05, "data_time": 0.02662975788116455, "loss": 0.05806073579005897, "time": 8.390367531776429, "iter": 230, "memory": 25158, "step": 230}
-{"lr": 2.163007638523606e-05, "data_time": 0.030708694458007814, "loss": 0.05118440636433661, "time": 8.436671686172485, "iter": 240, "memory": 25140, "step": 240}
-{"lr": 2.094119649058736e-05, "data_time": 0.026027894020080565, "loss": 0.05493427044712007, "time": 8.333375477790833, "iter": 250, "memory": 25144, "step": 250}
-{"lr": 2.0237114697185536e-05, "data_time": 0.7423177719116211, "loss": 0.04782075872644782, "time": 9.081082081794738, "iter": 260, "memory": 25158, "step": 260}
-{"lr": 1.9519632557992884e-05, "data_time": 0.031020736694335936, "loss": 0.014392800262430682, "time": 8.344713401794433, "iter": 270, "memory": 25140, "step": 270}
-{"lr": 1.8790585913795754e-05, "data_time": 0.030896997451782225, "loss": 0.021638503507710995, "time": 8.336081528663636, "iter": 280, "memory": 25144, "step": 280}
-{"lr": 1.8051840195789513e-05, "data_time": 0.029442191123962402, "loss": 0.048532775009516624, "time": 8.265917181968689, "iter": 290, "memory": 25135, "step": 290}
-{"lr": 1.7305285652449754e-05, "data_time": 0.032686376571655275, "loss": 0.03968424997292459, "time": 8.249550986289979, "iter": 300, "memory": 25158, "step": 300}
-{"lr": 1.6552832512902796e-05, "data_time": 0.0281200647354126, "loss": 0.020129067089874296, "time": 8.248900055885315, "iter": 310, "memory": 25154, "step": 310}
-{"lr": 1.579640609917124e-05, "data_time": 0.03012106418609619, "loss": 0.0355167729256209, "time": 8.344560074806214, "iter": 320, "memory": 25135, "step": 320}
-{"lr": 1.5037941899800858e-05, "data_time": 1.001177978515625, "loss": 0.003393273818073794, "time": 9.311607456207275, "iter": 330, "memory": 25116, "step": 330}
-{"lr": 1.4279380617474167e-05, "data_time": 0.029903650283813477, "loss": 0.004092969404882752, "time": 8.308899617195129, "iter": 340, "memory": 25121, "step": 340}
-{"lr": 1.3522663203282473e-05, "data_time": 0.029579424858093263, "loss": 0.003685022075660527, "time": 8.364891529083252, "iter": 350, "memory": 25140, "step": 350}
-{"lr": 1.2769725890362214e-05, "data_time": 0.028301119804382324, "loss": 0.022200413083191962, "time": 8.36471996307373, "iter": 360, "memory": 25158, "step": 360}
-{"lr": 1.2022495239603391e-05, "data_time": 0.05469930171966553, "loss": 0.012180326259112917, "time": 8.49920928478241, "iter": 370, "memory": 25140, "step": 370}
-{"lr": 1.1282883210106502e-05, "data_time": 0.03603675365447998, "loss": 0.011220535292522982, "time": 8.47714102268219, "iter": 380, "memory": 25140, "step": 380}
-{"lr": 1.0552782267001564e-05, "data_time": 0.713586688041687, "loss": 0.001590725060668774, "time": 9.058996796607971, "iter": 390, "memory": 25140, "step": 390}
-{"lr": 9.834060539146829e-06, "data_time": 0.029806923866271973, "loss": 0.0017313689793809317, "time": 8.444029283523559, "iter": 400, "memory": 25144, "step": 400}
-{"lr": 9.128557039097413e-06, "data_time": 0.02931809425354004, "loss": 0.0007458233783836476, "time": 8.399807620048524, "iter": 410, "memory": 25140, "step": 410}
-{"lr": 8.438076957574515e-06, "data_time": 0.028744888305664063, "loss": 0.00037622548115905373, "time": 8.259072613716125, "iter": 420, "memory": 25135, "step": 420}
-{"lr": 7.764387044475588e-06, "data_time": 0.027129387855529784, "loss": 0.011610788863617928, "time": 8.371538615226745, "iter": 430, "memory": 25158, "step": 430}
-{"lr": 7.10921108824393e-06, "data_time": 0.031836867332458496, "loss": 0.0004490063933189958, "time": 8.256002140045165, "iter": 440, "memory": 25158, "step": 440}
-{"lr": 6.474225505165039e-06, "data_time": 0.8085074424743652, "loss": 0.0010431828413857147, "time": 9.073117136955261, "iter": 450, "memory": 25158, "step": 450}
-{"lr": 5.8610550498752785e-06, "data_time": 0.027729082107543945, "loss": 0.0005872567562619224, "time": 8.278436923027039, "iter": 460, "memory": 25140, "step": 460}
-{"lr": 5.271268658058654e-06, "data_time": 0.03227052688598633, "loss": 0.0003798419769736938, "time": 8.24707763195038, "iter": 470, "memory": 25144, "step": 470}
-{"lr": 4.7063754319689895e-06, "data_time": 0.029940247535705566, "loss": 0.00033261780481552705, "time": 8.335689330101014, "iter": 480, "memory": 25144, "step": 480}
-{"lr": 4.167820779049542e-06, "data_time": 0.04668259620666504, "loss": 0.0003239909165131394, "time": 8.391029930114746, "iter": 490, "memory": 25154, "step": 490}
-{"lr": 3.6569827135302208e-06, "data_time": 0.030267667770385743, "loss": 0.00845504235912813, "time": 8.211200761795045, "iter": 500, "memory": 25102, "step": 500}
-{"lr": 3.175168330465622e-06, "data_time": 0.028434014320373534, "loss": 0.00045800769439665603, "time": 8.308828830718994, "iter": 510, "memory": 25140, "step": 510}
-{"lr": 2.7236104612358904e-06, "data_time": 0.7829745054244995, "loss": 0.00024433504731860014, "time": 9.145132136344909, "iter": 520, "memory": 25154, "step": 520}
-{"lr": 2.303464519067985e-06, "data_time": 0.02796189785003662, "loss": 0.00031818639108678327, "time": 8.131602883338928, "iter": 530, "memory": 25135, "step": 530}
-{"lr": 1.9158055426488924e-06, "data_time": 0.02785019874572754, "loss": 0.0003762481253943406, "time": 8.03377182483673, "iter": 540, "memory": 25158, "step": 540}
-{"lr": 1.5616254453953114e-06, "data_time": 0.027363348007202148, "loss": 0.000291722928523086, "time": 8.167411088943481, "iter": 550, "memory": 25149, "step": 550}
-{"lr": 1.2418304774182065e-06, "data_time": 0.02551448345184326, "loss": 0.00028513450088212265, "time": 8.342180752754212, "iter": 560, "memory": 25140, "step": 560}
-{"lr": 9.572389066763321e-07, "data_time": 0.03239858150482178, "loss": 0.00023931484029162675, "time": 8.44539041519165, "iter": 570, "memory": 25144, "step": 570}
-{"lr": 7.085789252520916e-07, "data_time": 0.7712726831436157, "loss": 0.00028982786534470506, "time": 9.253246903419495, "iter": 580, "memory": 25130, "step": 580}
-{"lr": 4.964867861069083e-07, "data_time": 0.029996538162231447, "loss": 0.00023493821936426684, "time": 8.585424041748047, "iter": 590, "memory": 25135, "step": 590}
-{"lr": 3.2150517508373746e-07, "data_time": 0.02820746898651123, "loss": 0.0003231823196983896, "time": 8.54775574207306, "iter": 600, "memory": 25140, "step": 600}
-{"lr": 1.8408182232222553e-07, "data_time": 0.029062747955322266, "loss": 0.00035379168984945865, "time": 8.482127594947816, "iter": 610, "memory": 25135, "step": 610}
-{"lr": 8.456835663962096e-08, "data_time": 0.02910158634185791, "loss": 0.0002700063669180963, "time": 8.424311804771424, "iter": 620, "memory": 25149, "step": 620}
-{"lr": 2.3219405808672077e-08, "data_time": 0.027964448928833006, "loss": 0.00025728107284521683, "time": 8.240233373641967, "iter": 630, "memory": 25144, "step": 630}
-{"lr": 1.9194503473318726e-10, "data_time": 0.029975509643554686, "loss": 0.0002424279176921118, "time": 8.14018223285675, "iter": 640, "memory": 25154, "step": 640}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/config.py DELETED Viewed

@@ -1,139 +0,0 @@
-accumulative_counts = 2
-batch_size = 4
-betas = (
-    0.9,
-    0.999,
-)
-custom_hooks = [
-    dict(
-        tokenizer=dict(
-            pretrained_model_name_or_path=
-            '/root/share/new_models/OpenGVLab/InternVL2-2B',
-            trust_remote_code=True,
-            type='transformers.AutoTokenizer.from_pretrained'),
-        type='xtuner.engine.hooks.DatasetInfoHook'),
-]
-data_path = '/root/share/datasets/FoodieQA/sivqa_llava.json'
-data_root = '/root/share/datasets/FoodieQA/'
-dataloader_num_workers = 4
-default_hooks = dict(
-    checkpoint=dict(
-        by_epoch=False,
-        interval=64,
-        max_keep_ckpts=-1,
-        save_optimizer=False,
-        type='mmengine.hooks.CheckpointHook'),
-    logger=dict(
-        interval=10,
-        log_metric_by_epoch=False,
-        type='mmengine.hooks.LoggerHook'),
-    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
-    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
-    timer=dict(type='mmengine.hooks.IterTimerHook'))
-env_cfg = dict(
-    cudnn_benchmark=False,
-    dist_cfg=dict(backend='nccl'),
-    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
-image_folder = '/root/share/datasets/FoodieQA/'
-launcher = 'none'
-llava_dataset = dict(
-    data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-    image_folders='/root/share/datasets/FoodieQA/',
-    max_length=8192,
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-    type='xtuner.dataset.InternVL_V1_5_Dataset')
-load_from = None
-log_level = 'INFO'
-log_processor = dict(by_epoch=False)
-lr = 3e-05
-max_epochs = 10
-max_length = 8192
-max_norm = 1
-model = dict(
-    freeze_llm=True,
-    freeze_visual_encoder=True,
-    llm_lora=dict(
-        lora_alpha=256,
-        lora_dropout=0.05,
-        r=128,
-        target_modules=None,
-        task_type='CAUSAL_LM',
-        type='peft.LoraConfig'),
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    type='xtuner.model.InternVL_V1_5')
-optim_type = 'torch.optim.AdamW'
-optim_wrapper = dict(
-    optimizer=dict(
-        betas=(
-            0.9,
-            0.999,
-        ),
-        lr=3e-05,
-        type='torch.optim.AdamW',
-        weight_decay=0.05),
-    type='DeepSpeedOptimWrapper')
-param_scheduler = [
-    dict(
-        begin=0,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=0.3,
-        start_factor=1e-05,
-        type='mmengine.optim.LinearLR'),
-    dict(
-        begin=0.3,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=10,
-        eta_min=0.0,
-        type='mmengine.optim.CosineAnnealingLR'),
-]
-path = '/root/share/new_models/OpenGVLab/InternVL2-2B'
-prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat'
-randomness = dict(deterministic=False, seed=None)
-resume = False
-runner_type = 'FlexibleRunner'
-save_steps = 64
-save_total_limit = -1
-strategy = dict(
-    config=dict(
-        bf16=dict(enabled=True),
-        fp16=dict(enabled=False, initial_scale_power=16),
-        gradient_accumulation_steps='auto',
-        gradient_clipping='auto',
-        train_micro_batch_size_per_gpu='auto',
-        zero_allow_untested_optimizer=True,
-        zero_force_ds_cpu_optimizer=False,
-        zero_optimization=dict(overlap_comm=True, stage=2)),
-    exclude_frozen_parameters=True,
-    gradient_accumulation_steps=2,
-    gradient_clipping=1,
-    sequence_parallel_size=1,
-    train_micro_batch_size_per_gpu=4,
-    type='xtuner.engine.DeepSpeedStrategy')
-tokenizer = dict(
-    pretrained_model_name_or_path=
-    '/root/share/new_models/OpenGVLab/InternVL2-2B',
-    trust_remote_code=True,
-    type='transformers.AutoTokenizer.from_pretrained')
-train_cfg = dict(max_epochs=10, type='xtuner.engine.runner.TrainLoop')
-train_dataloader = dict(
-    batch_size=4,
-    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
-    dataset=dict(
-        data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-        image_folders='/root/share/datasets/FoodieQA/',
-        max_length=8192,
-        model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-        template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-        type='xtuner.dataset.InternVL_V1_5_Dataset'),
-    num_workers=4,
-    sampler=dict(
-        length_property='modality_length',
-        per_device_batch_size=8,
-        type='xtuner.dataset.samplers.LengthGroupedSampler'))
-visualizer = None
-warmup_ratio = 0.03
-weight_decay = 0.05
-work_dir = './work_dirs/internvl_v2_internlm2_2b_lora_finetune_food'

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/20241230_145825/vis_data/scalars.json DELETED Viewed

@@ -1,64 +0,0 @@
-{"lr": 1.500015e-05, "data_time": 0.020780611038208007, "loss": 5.205293273925781, "time": 10.055773782730103, "iter": 10, "memory": 25140, "step": 10}
-{"lr": 3e-05, "data_time": 0.03608603477478027, "loss": 2.9355292201042174, "time": 8.311998891830445, "iter": 20, "memory": 25130, "step": 20}
-{"lr": 2.99808095489134e-05, "data_time": 0.027062010765075684, "loss": 1.6511537075042724, "time": 8.220871329307556, "iter": 30, "memory": 25135, "step": 30}
-{"lr": 2.9923287298775314e-05, "data_time": 0.02596433162689209, "loss": 1.2374991178512573, "time": 8.251240038871765, "iter": 40, "memory": 25158, "step": 40}
-{"lr": 2.9827580433309446e-05, "data_time": 0.027050209045410157, "loss": 0.8767685353755951, "time": 8.38843400478363, "iter": 50, "memory": 25135, "step": 50}
-{"lr": 2.9693933840238504e-05, "data_time": 0.02903637886047363, "loss": 0.9338608592748642, "time": 8.341249108314514, "iter": 60, "memory": 25144, "step": 60}
-{"lr": 2.952268948468346e-05, "data_time": 0.6511948108673096, "loss": 0.5485243916511535, "time": 9.00740897655487, "iter": 70, "memory": 25149, "step": 70}
-{"lr": 2.9314285534168186e-05, "data_time": 0.024419164657592772, "loss": 0.42309151738882067, "time": 8.19894368648529, "iter": 80, "memory": 25144, "step": 80}
-{"lr": 2.90692552374685e-05, "data_time": 0.026521754264831544, "loss": 0.48458234667778016, "time": 8.221326875686646, "iter": 90, "memory": 25174, "step": 90}
-{"lr": 2.8788225560174216e-05, "data_time": 0.024730491638183593, "loss": 0.3923974633216858, "time": 7.991903710365295, "iter": 100, "memory": 25158, "step": 100}
-{"lr": 2.847191558045545e-05, "data_time": 0.026031899452209472, "loss": 0.398375840485096, "time": 8.351808667182922, "iter": 110, "memory": 25140, "step": 110}
-{"lr": 2.8121134649138086e-05, "data_time": 0.026431798934936523, "loss": 0.40561383217573166, "time": 8.331733465194702, "iter": 120, "memory": 25135, "step": 120}
-{"lr": 2.7736780318796056e-05, "data_time": 1.4211087226867676, "loss": 0.41130477637052537, "time": 9.81301429271698, "iter": 130, "memory": 25174, "step": 130}
-{"lr": 2.7319836047159543e-05, "data_time": 0.03063185214996338, "loss": 0.1909895658493042, "time": 8.292147588729858, "iter": 140, "memory": 25140, "step": 140}
-{"lr": 2.68713686807153e-05, "data_time": 0.03131427764892578, "loss": 0.13053411729633807, "time": 8.237112450599671, "iter": 150, "memory": 25135, "step": 150}
-{"lr": 2.639252572493797e-05, "data_time": 0.02983086109161377, "loss": 0.20683301314711572, "time": 8.285038375854493, "iter": 160, "memory": 25140, "step": 160}
-{"lr": 2.5884532408136998e-05, "data_time": 0.028959155082702637, "loss": 0.14075656346976756, "time": 8.280118131637574, "iter": 170, "memory": 25140, "step": 170}
-{"lr": 2.534868854643217e-05, "data_time": 0.03114640712738037, "loss": 0.16897681429982186, "time": 8.363134860992432, "iter": 180, "memory": 25125, "step": 180}
-{"lr": 2.4786365217879254e-05, "data_time": 0.029333734512329103, "loss": 0.1947656821459532, "time": 8.32378842830658, "iter": 190, "memory": 25144, "step": 190}
-{"lr": 2.419900125425576e-05, "data_time": 0.6446581840515136, "loss": 0.05592906894162297, "time": 8.864284300804139, "iter": 200, "memory": 25140, "step": 200}
-{"lr": 2.3588099559483543e-05, "data_time": 0.030133438110351563, "loss": 0.055754791107028724, "time": 8.271190738677978, "iter": 210, "memory": 25174, "step": 210}
-{"lr": 2.2955223264108254e-05, "data_time": 0.14820406436920167, "loss": 0.08434767709113658, "time": 8.665787267684937, "iter": 220, "memory": 25135, "step": 220}
-{"lr": 2.2301991725675243e-05, "data_time": 0.02662975788116455, "loss": 0.05806073579005897, "time": 8.390367531776429, "iter": 230, "memory": 25158, "step": 230}
-{"lr": 2.163007638523606e-05, "data_time": 0.030708694458007814, "loss": 0.05118440636433661, "time": 8.436671686172485, "iter": 240, "memory": 25140, "step": 240}
-{"lr": 2.094119649058736e-05, "data_time": 0.026027894020080565, "loss": 0.05493427044712007, "time": 8.333375477790833, "iter": 250, "memory": 25144, "step": 250}
-{"lr": 2.0237114697185536e-05, "data_time": 0.7423177719116211, "loss": 0.04782075872644782, "time": 9.081082081794738, "iter": 260, "memory": 25158, "step": 260}
-{"lr": 1.9519632557992884e-05, "data_time": 0.031020736694335936, "loss": 0.014392800262430682, "time": 8.344713401794433, "iter": 270, "memory": 25140, "step": 270}
-{"lr": 1.8790585913795754e-05, "data_time": 0.030896997451782225, "loss": 0.021638503507710995, "time": 8.336081528663636, "iter": 280, "memory": 25144, "step": 280}
-{"lr": 1.8051840195789513e-05, "data_time": 0.029442191123962402, "loss": 0.048532775009516624, "time": 8.265917181968689, "iter": 290, "memory": 25135, "step": 290}
-{"lr": 1.7305285652449754e-05, "data_time": 0.032686376571655275, "loss": 0.03968424997292459, "time": 8.249550986289979, "iter": 300, "memory": 25158, "step": 300}
-{"lr": 1.6552832512902796e-05, "data_time": 0.0281200647354126, "loss": 0.020129067089874296, "time": 8.248900055885315, "iter": 310, "memory": 25154, "step": 310}
-{"lr": 1.579640609917124e-05, "data_time": 0.03012106418609619, "loss": 0.0355167729256209, "time": 8.344560074806214, "iter": 320, "memory": 25135, "step": 320}
-{"lr": 1.5037941899800858e-05, "data_time": 1.001177978515625, "loss": 0.003393273818073794, "time": 9.311607456207275, "iter": 330, "memory": 25116, "step": 330}
-{"lr": 1.4279380617474167e-05, "data_time": 0.029903650283813477, "loss": 0.004092969404882752, "time": 8.308899617195129, "iter": 340, "memory": 25121, "step": 340}
-{"lr": 1.3522663203282473e-05, "data_time": 0.029579424858093263, "loss": 0.003685022075660527, "time": 8.364891529083252, "iter": 350, "memory": 25140, "step": 350}
-{"lr": 1.2769725890362214e-05, "data_time": 0.028301119804382324, "loss": 0.022200413083191962, "time": 8.36471996307373, "iter": 360, "memory": 25158, "step": 360}
-{"lr": 1.2022495239603391e-05, "data_time": 0.05469930171966553, "loss": 0.012180326259112917, "time": 8.49920928478241, "iter": 370, "memory": 25140, "step": 370}
-{"lr": 1.1282883210106502e-05, "data_time": 0.03603675365447998, "loss": 0.011220535292522982, "time": 8.47714102268219, "iter": 380, "memory": 25140, "step": 380}
-{"lr": 1.0552782267001564e-05, "data_time": 0.713586688041687, "loss": 0.001590725060668774, "time": 9.058996796607971, "iter": 390, "memory": 25140, "step": 390}
-{"lr": 9.834060539146829e-06, "data_time": 0.029806923866271973, "loss": 0.0017313689793809317, "time": 8.444029283523559, "iter": 400, "memory": 25144, "step": 400}
-{"lr": 9.128557039097413e-06, "data_time": 0.02931809425354004, "loss": 0.0007458233783836476, "time": 8.399807620048524, "iter": 410, "memory": 25140, "step": 410}
-{"lr": 8.438076957574515e-06, "data_time": 0.028744888305664063, "loss": 0.00037622548115905373, "time": 8.259072613716125, "iter": 420, "memory": 25135, "step": 420}
-{"lr": 7.764387044475588e-06, "data_time": 0.027129387855529784, "loss": 0.011610788863617928, "time": 8.371538615226745, "iter": 430, "memory": 25158, "step": 430}
-{"lr": 7.10921108824393e-06, "data_time": 0.031836867332458496, "loss": 0.0004490063933189958, "time": 8.256002140045165, "iter": 440, "memory": 25158, "step": 440}
-{"lr": 6.474225505165039e-06, "data_time": 0.8085074424743652, "loss": 0.0010431828413857147, "time": 9.073117136955261, "iter": 450, "memory": 25158, "step": 450}
-{"lr": 5.8610550498752785e-06, "data_time": 0.027729082107543945, "loss": 0.0005872567562619224, "time": 8.278436923027039, "iter": 460, "memory": 25140, "step": 460}
-{"lr": 5.271268658058654e-06, "data_time": 0.03227052688598633, "loss": 0.0003798419769736938, "time": 8.24707763195038, "iter": 470, "memory": 25144, "step": 470}
-{"lr": 4.7063754319689895e-06, "data_time": 0.029940247535705566, "loss": 0.00033261780481552705, "time": 8.335689330101014, "iter": 480, "memory": 25144, "step": 480}
-{"lr": 4.167820779049542e-06, "data_time": 0.04668259620666504, "loss": 0.0003239909165131394, "time": 8.391029930114746, "iter": 490, "memory": 25154, "step": 490}
-{"lr": 3.6569827135302208e-06, "data_time": 0.030267667770385743, "loss": 0.00845504235912813, "time": 8.211200761795045, "iter": 500, "memory": 25102, "step": 500}
-{"lr": 3.175168330465622e-06, "data_time": 0.028434014320373534, "loss": 0.00045800769439665603, "time": 8.308828830718994, "iter": 510, "memory": 25140, "step": 510}
-{"lr": 2.7236104612358904e-06, "data_time": 0.7829745054244995, "loss": 0.00024433504731860014, "time": 9.145132136344909, "iter": 520, "memory": 25154, "step": 520}
-{"lr": 2.303464519067985e-06, "data_time": 0.02796189785003662, "loss": 0.00031818639108678327, "time": 8.131602883338928, "iter": 530, "memory": 25135, "step": 530}
-{"lr": 1.9158055426488924e-06, "data_time": 0.02785019874572754, "loss": 0.0003762481253943406, "time": 8.03377182483673, "iter": 540, "memory": 25158, "step": 540}
-{"lr": 1.5616254453953114e-06, "data_time": 0.027363348007202148, "loss": 0.000291722928523086, "time": 8.167411088943481, "iter": 550, "memory": 25149, "step": 550}
-{"lr": 1.2418304774182065e-06, "data_time": 0.02551448345184326, "loss": 0.00028513450088212265, "time": 8.342180752754212, "iter": 560, "memory": 25140, "step": 560}
-{"lr": 9.572389066763321e-07, "data_time": 0.03239858150482178, "loss": 0.00023931484029162675, "time": 8.44539041519165, "iter": 570, "memory": 25144, "step": 570}
-{"lr": 7.085789252520916e-07, "data_time": 0.7712726831436157, "loss": 0.00028982786534470506, "time": 9.253246903419495, "iter": 580, "memory": 25130, "step": 580}
-{"lr": 4.964867861069083e-07, "data_time": 0.029996538162231447, "loss": 0.00023493821936426684, "time": 8.585424041748047, "iter": 590, "memory": 25135, "step": 590}
-{"lr": 3.2150517508373746e-07, "data_time": 0.02820746898651123, "loss": 0.0003231823196983896, "time": 8.54775574207306, "iter": 600, "memory": 25140, "step": 600}
-{"lr": 1.8408182232222553e-07, "data_time": 0.029062747955322266, "loss": 0.00035379168984945865, "time": 8.482127594947816, "iter": 610, "memory": 25135, "step": 610}
-{"lr": 8.456835663962096e-08, "data_time": 0.02910158634185791, "loss": 0.0002700063669180963, "time": 8.424311804771424, "iter": 620, "memory": 25149, "step": 620}
-{"lr": 2.3219405808672077e-08, "data_time": 0.027964448928833006, "loss": 0.00025728107284521683, "time": 8.240233373641967, "iter": 630, "memory": 25144, "step": 630}
-{"lr": 1.9194503473318726e-10, "data_time": 0.029975509643554686, "loss": 0.0002424279176921118, "time": 8.14018223285675, "iter": 640, "memory": 25154, "step": 640}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/internvl_v2_internlm2_2b_lora_finetune_food.py DELETED Viewed

@@ -1,139 +0,0 @@
-accumulative_counts = 2
-batch_size = 4
-betas = (
-    0.9,
-    0.999,
-)
-custom_hooks = [
-    dict(
-        tokenizer=dict(
-            pretrained_model_name_or_path=
-            '/root/share/new_models/OpenGVLab/InternVL2-2B',
-            trust_remote_code=True,
-            type='transformers.AutoTokenizer.from_pretrained'),
-        type='xtuner.engine.hooks.DatasetInfoHook'),
-]
-data_path = '/root/share/datasets/FoodieQA/sivqa_llava.json'
-data_root = '/root/share/datasets/FoodieQA/'
-dataloader_num_workers = 4
-default_hooks = dict(
-    checkpoint=dict(
-        by_epoch=False,
-        interval=64,
-        max_keep_ckpts=-1,
-        save_optimizer=False,
-        type='mmengine.hooks.CheckpointHook'),
-    logger=dict(
-        interval=10,
-        log_metric_by_epoch=False,
-        type='mmengine.hooks.LoggerHook'),
-    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
-    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
-    timer=dict(type='mmengine.hooks.IterTimerHook'))
-env_cfg = dict(
-    cudnn_benchmark=False,
-    dist_cfg=dict(backend='nccl'),
-    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
-image_folder = '/root/share/datasets/FoodieQA/'
-launcher = 'none'
-llava_dataset = dict(
-    data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-    image_folders='/root/share/datasets/FoodieQA/',
-    max_length=8192,
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-    type='xtuner.dataset.InternVL_V1_5_Dataset')
-load_from = None
-log_level = 'INFO'
-log_processor = dict(by_epoch=False)
-lr = 3e-05
-max_epochs = 10
-max_length = 8192
-max_norm = 1
-model = dict(
-    freeze_llm=True,
-    freeze_visual_encoder=True,
-    llm_lora=dict(
-        lora_alpha=256,
-        lora_dropout=0.05,
-        r=128,
-        target_modules=None,
-        task_type='CAUSAL_LM',
-        type='peft.LoraConfig'),
-    model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-    type='xtuner.model.InternVL_V1_5')
-optim_type = 'torch.optim.AdamW'
-optim_wrapper = dict(
-    optimizer=dict(
-        betas=(
-            0.9,
-            0.999,
-        ),
-        lr=3e-05,
-        type='torch.optim.AdamW',
-        weight_decay=0.05),
-    type='DeepSpeedOptimWrapper')
-param_scheduler = [
-    dict(
-        begin=0,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=0.3,
-        start_factor=1e-05,
-        type='mmengine.optim.LinearLR'),
-    dict(
-        begin=0.3,
-        by_epoch=True,
-        convert_to_iter_based=True,
-        end=10,
-        eta_min=0.0,
-        type='mmengine.optim.CosineAnnealingLR'),
-]
-path = '/root/share/new_models/OpenGVLab/InternVL2-2B'
-prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat'
-randomness = dict(deterministic=False, seed=None)
-resume = False
-runner_type = 'FlexibleRunner'
-save_steps = 64
-save_total_limit = -1
-strategy = dict(
-    config=dict(
-        bf16=dict(enabled=True),
-        fp16=dict(enabled=False, initial_scale_power=16),
-        gradient_accumulation_steps='auto',
-        gradient_clipping='auto',
-        train_micro_batch_size_per_gpu='auto',
-        zero_allow_untested_optimizer=True,
-        zero_force_ds_cpu_optimizer=False,
-        zero_optimization=dict(overlap_comm=True, stage=2)),
-    exclude_frozen_parameters=True,
-    gradient_accumulation_steps=2,
-    gradient_clipping=1,
-    sequence_parallel_size=1,
-    train_micro_batch_size_per_gpu=4,
-    type='xtuner.engine.DeepSpeedStrategy')
-tokenizer = dict(
-    pretrained_model_name_or_path=
-    '/root/share/new_models/OpenGVLab/InternVL2-2B',
-    trust_remote_code=True,
-    type='transformers.AutoTokenizer.from_pretrained')
-train_cfg = dict(max_epochs=10, type='xtuner.engine.runner.TrainLoop')
-train_dataloader = dict(
-    batch_size=4,
-    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
-    dataset=dict(
-        data_paths='/root/share/datasets/FoodieQA/sivqa_llava.json',
-        image_folders='/root/share/datasets/FoodieQA/',
-        max_length=8192,
-        model_path='/root/share/new_models/OpenGVLab/InternVL2-2B',
-        template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
-        type='xtuner.dataset.InternVL_V1_5_Dataset'),
-    num_workers=4,
-    sampler=dict(
-        length_property='modality_length',
-        per_device_batch_size=8,
-        type='xtuner.dataset.samplers.LengthGroupedSampler'))
-visualizer = None
-warmup_ratio = 0.03
-weight_decay = 0.05
-work_dir = './work_dirs/internvl_v2_internlm2_2b_lora_finetune_food'

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_128.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e53e56447e7524be69f42aca9b30162b0bb000e02ffec0cebaf6a5a59cf2c960
-size 301178818

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_192.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:89e9a96a15f6f75ef4fab42a7be13492a67c89af796449cd7cbd596a4607df1f
-size 301183554

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_256.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4f8e005f42c8f7723815ceec8b539d7940f307be7a2fe921c4795c9c7cc5be83
-size 301188290

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_320.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8d25fb677f1c77193bb00947d986ab451223bef2221cfd8ad7720e9e4200fb48
-size 301193026

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_384.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d9d3ccb385b403e9970ac4928bb391adc7b61e0f3993215599c82a12168cbeec
-size 301197698

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_448.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e24ae86b5ba6d12d835697ab12020af2c7867dd636c7861c044ca942d59f65d1
-size 301202306

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_512.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:98b32034ab80a99e8f8ebd5fc44e84d202e115e7ff70403d22632967c06c08ca
-size 301206978

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_576.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d940269d3b4e7fe59b41eab8013bb50607b04a8144bb1f802b0f51288c2ff7af
-size 301211650

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_64.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:90ca69f20113b248ae93f5ac21e68ecd680fc9bc2737bcf734762ddb74ac9074
-size 301174082

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_640.pth DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9fad9f5203dc97b5f61720c6141252a3e7444dfb9b796860e977088d54d21e62
-size 301216258

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/last_checkpoint DELETED Viewed

	@@ -1 +0,0 @@
1	- /root/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/iter_640.pth

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/added_tokens.json DELETED Viewed

@@ -1,11 +0,0 @@
-{
-  "</box>": 92552,
-  "</img>": 92545,
-  "</quad>": 92548,
-  "</ref>": 92550,
-  "<IMG_CONTEXT>": 92546,
-  "<box>": 92551,
-  "<img>": 92544,
-  "<quad>": 92547,
-  "<ref>": 92549
-}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/config.json DELETED Viewed

@@ -1,199 +0,0 @@
-{
-  "_commit_hash": null,
-  "_name_or_path": "/root/share/new_models/OpenGVLab/InternVL2-2B",
-  "architectures": [
-    "InternVLChatModel"
-  ],
-  "auto_map": {
-    "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
-    "AutoModel": "modeling_internvl_chat.InternVLChatModel",
-    "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
-  },
-  "downsample_ratio": 0.5,
-  "dynamic_image_size": true,
-  "force_image_size": 448,
-  "llm_config": {
-    "_name_or_path": "internlm/internlm2-chat-1_8b",
-    "add_cross_attention": false,
-    "architectures": [
-      "InternLM2ForCausalLM"
-    ],
-    "attn_implementation": "eager",
-    "auto_map": {
-      "AutoConfig": "configuration_internlm2.InternLM2Config",
-      "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
-      "AutoModelForCausalLM": "modeling_internlm2.InternLM2ForCausalLM"
-    },
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bias": false,
-    "bos_token_id": 1,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": 2,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "silu",
-    "hidden_size": 2048,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "initializer_range": 0.02,
-    "intermediate_size": 8192,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 32768,
-    "min_length": 0,
-    "model_type": "internlm2",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 24,
-    "num_key_value_heads": 8,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": 2,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "rms_norm_eps": 1e-05,
-    "rope_scaling": {
-      "factor": 2.0,
-      "type": "dynamic"
-    },
-    "rope_theta": 1000000,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": false,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "bfloat16",
-    "torchscript": false,
-    "transformers_version": "4.39.0",
-    "typical_p": 1.0,
-    "use_bfloat16": true,
-    "use_cache": true,
-    "vocab_size": 92553
-  },
-  "max_dynamic_patch": 12,
-  "min_dynamic_patch": 1,
-  "model_type": "internvl_chat",
-  "ps_version": "v2",
-  "select_layer": -1,
-  "template": "internlm2-chat",
-  "torch_dtype": "bfloat16",
-  "transformers_version": null,
-  "use_backbone_lora": 0,
-  "use_llm_lora": 0,
-  "use_thumbnail": true,
-  "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": [
-      "InternVisionModel"
-    ],
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "drop_path_rate": 0.0,
-    "dropout": 0.0,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_size": 1024,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 448,
-    "initializer_factor": 1.0,
-    "initializer_range": 0.02,
-    "intermediate_size": 4096,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-06,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
-    "model_type": "intern_vit_6b",
-    "no_repeat_ngram_size": 0,
-    "norm_type": "layer_norm",
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_channels": 3,
-    "num_hidden_layers": 24,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "qk_normalization": false,
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "bfloat16",
-    "torchscript": false,
-    "transformers_version": "4.39.0",
-    "typical_p": 1.0,
-    "use_bfloat16": true,
-    "use_flash_attn": true
-  }
-}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_intern_vit.py DELETED Viewed

@@ -1,119 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2024 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-import os
-from typing import Union
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-logger = logging.get_logger(__name__)
-class InternVisionConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to
-    instantiate a vision encoder according to the specified arguments, defining the model architecture.
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-    Args:
-        num_channels (`int`, *optional*, defaults to 3):
-            Number of color channels in the input images (e.g., 3 for RGB).
-        patch_size (`int`, *optional*, defaults to 14):
-            The size (resolution) of each patch.
-        image_size (`int`, *optional*, defaults to 224):
-            The size (resolution) of each image.
-        qkv_bias (`bool`, *optional*, defaults to `False`):
-            Whether to add a bias to the queries and values in the self-attention layers.
-        hidden_size (`int`, *optional*, defaults to 3200):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_attention_heads (`int`, *optional*, defaults to 25):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (`int`, *optional*, defaults to 12800):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        qk_normalization (`bool`, *optional*, defaults to `True`):
-            Whether to normalize the queries and keys in the self-attention layers.
-        num_hidden_layers (`int`, *optional*, defaults to 48):
-            Number of hidden layers in the Transformer encoder.
-        use_flash_attn (`bool`, *optional*, defaults to `True`):
-            Whether to use flash attention mechanism.
-        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
-            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
-            `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported.
-        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
-            The epsilon used by the layer normalization layers.
-        dropout (`float`, *optional*, defaults to 0.0):
-            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
-        drop_path_rate (`float`, *optional*, defaults to 0.0):
-            Dropout rate for stochastic depth.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        initializer_factor (`float`, *optional*, defaults to 0.1):
-            A factor for layer scale.
-    """
-    model_type = 'intern_vit_6b'
-    def __init__(
-            self,
-            num_channels=3,
-            patch_size=14,
-            image_size=224,
-            qkv_bias=False,
-            hidden_size=3200,
-            num_attention_heads=25,
-            intermediate_size=12800,
-            qk_normalization=True,
-            num_hidden_layers=48,
-            use_flash_attn=True,
-            hidden_act='gelu',
-            norm_type='rms_norm',
-            layer_norm_eps=1e-6,
-            dropout=0.0,
-            drop_path_rate=0.0,
-            attention_dropout=0.0,
-            initializer_range=0.02,
-            initializer_factor=0.1,
-            **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.dropout = dropout
-        self.drop_path_rate = drop_path_rate
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_channels = num_channels
-        self.patch_size = patch_size
-        self.image_size = image_size
-        self.initializer_range = initializer_range
-        self.initializer_factor = initializer_factor
-        self.attention_dropout = attention_dropout
-        self.layer_norm_eps = layer_norm_eps
-        self.hidden_act = hidden_act
-        self.norm_type = norm_type
-        self.qkv_bias = qkv_bias
-        self.qk_normalization = qk_normalization
-        self.use_flash_attn = use_flash_attn
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig':
-        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
-        if 'vision_config' in config_dict:
-            config_dict = config_dict['vision_config']
-        if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type:
-            logger.warning(
-                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
-                f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
-            )
-        return cls.from_dict(config_dict, **kwargs)

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_internlm2.py DELETED Viewed

@@ -1,150 +0,0 @@
-# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
-#
-# This code is based on transformers/src/transformers/models/llama/configuration_llama.py
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" InternLM2 model configuration"""
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-logger = logging.get_logger(__name__)
-INTERNLM2_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
-# Modified from transformers.model.llama.configuration_llama.LlamaConfig
-class InternLM2Config(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`InternLM2Model`]. It is used to instantiate
-    an InternLM2 model according to the specified arguments, defining the model architecture. Instantiating a
-    configuration with the defaults will yield a similar configuration to that of the InternLM2-7B.
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-    Args:
-        vocab_size (`int`, *optional*, defaults to 32000):
-            Vocabulary size of the InternLM2 model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`InternLM2Model`]
-        hidden_size (`int`, *optional*, defaults to 4096):
-            Dimension of the hidden representations.
-        intermediate_size (`int`, *optional*, defaults to 11008):
-            Dimension of the MLP representations.
-        num_hidden_layers (`int`, *optional*, defaults to 32):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (`int`, *optional*, defaults to 32):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        num_key_value_heads (`int`, *optional*):
-            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
-            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
-            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
-            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
-            by meanpooling all the original heads within that group. For more details checkout [this
-            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
-            `num_attention_heads`.
-        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
-            The non-linear activation function (function or string) in the decoder.
-        max_position_embeddings (`int`, *optional*, defaults to 2048):
-            The maximum sequence length that this model might ever be used with. Typically set this to something large
-            just in case (e.g., 512 or 1024 or 2048).
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        rms_norm_eps (`float`, *optional*, defaults to 1e-12):
-            The epsilon used by the rms normalization layers.
-        use_cache (`bool`, *optional*, defaults to `True`):
-            Whether or not the model should return the last key/values attentions (not used by all models). Only
-            relevant if `config.is_decoder=True`.
-        tie_word_embeddings(`bool`, *optional*, defaults to `False`):
-            Whether to tie weight embeddings
-        Example:
-    """
-    model_type = 'internlm2'
-    _auto_class = 'AutoConfig'
-    def __init__(  # pylint: disable=W0102
-        self,
-        vocab_size=103168,
-        hidden_size=4096,
-        intermediate_size=11008,
-        num_hidden_layers=32,
-        num_attention_heads=32,
-        num_key_value_heads=None,
-        hidden_act='silu',
-        max_position_embeddings=2048,
-        initializer_range=0.02,
-        rms_norm_eps=1e-6,
-        use_cache=True,
-        pad_token_id=0,
-        bos_token_id=1,
-        eos_token_id=2,
-        tie_word_embeddings=False,
-        bias=True,
-        rope_theta=10000,
-        rope_scaling=None,
-        attn_implementation='eager',
-        **kwargs,
-    ):
-        self.vocab_size = vocab_size
-        self.max_position_embeddings = max_position_embeddings
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.bias = bias
-        if num_key_value_heads is None:
-            num_key_value_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        self.hidden_act = hidden_act
-        self.initializer_range = initializer_range
-        self.rms_norm_eps = rms_norm_eps
-        self.use_cache = use_cache
-        self.rope_theta = rope_theta
-        self.rope_scaling = rope_scaling
-        self._rope_scaling_validation()
-        self.attn_implementation = attn_implementation
-        if self.attn_implementation is None:
-            self.attn_implementation = 'eager'
-        super().__init__(
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            tie_word_embeddings=tie_word_embeddings,
-            **kwargs,
-        )
-    def _rope_scaling_validation(self):
-        """
-        Validate the `rope_scaling` configuration.
-        """
-        if self.rope_scaling is None:
-            return
-        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
-            raise ValueError(
-                '`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, '
-                f'got {self.rope_scaling}'
-            )
-        rope_scaling_type = self.rope_scaling.get('type', None)
-        rope_scaling_factor = self.rope_scaling.get('factor', None)
-        if rope_scaling_type is None or rope_scaling_type not in ['linear', 'dynamic']:
-            raise ValueError(
-                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
-            )
-        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor < 1.0:
-            raise ValueError(f"`rope_scaling`'s factor field must be a float >= 1, got {rope_scaling_factor}")

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/configuration_internvl_chat.py DELETED Viewed

@@ -1,96 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2024 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-import copy
-from transformers import AutoConfig, LlamaConfig
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-from .configuration_intern_vit import InternVisionConfig
-from .configuration_internlm2 import InternLM2Config
-logger = logging.get_logger(__name__)
-class InternVLChatConfig(PretrainedConfig):
-    model_type = 'internvl_chat'
-    is_composition = True
-    def __init__(
-            self,
-            vision_config=None,
-            llm_config=None,
-            use_backbone_lora=0,
-            use_llm_lora=0,
-            select_layer=-1,
-            force_image_size=None,
-            downsample_ratio=0.5,
-            template=None,
-            dynamic_image_size=False,
-            use_thumbnail=False,
-            ps_version='v1',
-            min_dynamic_patch=1,
-            max_dynamic_patch=6,
-            **kwargs):
-        super().__init__(**kwargs)
-        if vision_config is None:
-            vision_config = {}
-            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')
-        if llm_config is None:
-            llm_config = {}
-            logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')
-        self.vision_config = InternVisionConfig(**vision_config)
-        if llm_config['architectures'][0] == 'LlamaForCausalLM':
-            self.llm_config = LlamaConfig(**llm_config)
-        elif llm_config['architectures'][0] == 'InternLM2ForCausalLM':
-            self.llm_config = InternLM2Config(**llm_config)
-        else:
-            raise ValueError('Unsupported architecture: {}'.format(llm_config['architectures'][0]))
-        self.use_backbone_lora = use_backbone_lora
-        self.use_llm_lora = use_llm_lora
-        self.select_layer = select_layer
-        self.force_image_size = force_image_size
-        self.downsample_ratio = downsample_ratio
-        self.template = template
-        self.dynamic_image_size = dynamic_image_size
-        self.use_thumbnail = use_thumbnail
-        self.ps_version = ps_version  # pixel shuffle version
-        self.min_dynamic_patch = min_dynamic_patch
-        self.max_dynamic_patch = max_dynamic_patch
-        logger.info(f'vision_select_layer: {self.select_layer}')
-        logger.info(f'ps_version: {self.ps_version}')
-        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
-        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
-    def to_dict(self):
-        """
-        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
-        Returns:
-            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
-        """
-        output = copy.deepcopy(self.__dict__)
-        output['vision_config'] = self.vision_config.to_dict()
-        output['llm_config'] = self.llm_config.to_dict()
-        output['model_type'] = self.__class__.model_type
-        output['use_backbone_lora'] = self.use_backbone_lora
-        output['use_llm_lora'] = self.use_llm_lora
-        output['select_layer'] = self.select_layer
-        output['force_image_size'] = self.force_image_size
-        output['downsample_ratio'] = self.downsample_ratio
-        output['template'] = self.template
-        output['dynamic_image_size'] = self.dynamic_image_size
-        output['use_thumbnail'] = self.use_thumbnail
-        output['ps_version'] = self.ps_version
-        output['min_dynamic_patch'] = self.min_dynamic_patch
-        output['max_dynamic_patch'] = self.max_dynamic_patch
-        return output

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/conversation.py DELETED Viewed

@@ -1,393 +0,0 @@
-"""
-Conversation prompt templates.
-We kindly request that you import fastchat instead of copying this file if you wish to use it.
-If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
-"""
-import dataclasses
-from enum import IntEnum, auto
-from typing import Any, Dict, List, Tuple, Union
-class SeparatorStyle(IntEnum):
-    """Separator styles."""
-    ADD_COLON_SINGLE = auto()
-    ADD_COLON_TWO = auto()
-    ADD_COLON_SPACE_SINGLE = auto()
-    NO_COLON_SINGLE = auto()
-    NO_COLON_TWO = auto()
-    ADD_NEW_LINE_SINGLE = auto()
-    LLAMA2 = auto()
-    CHATGLM = auto()
-    CHATML = auto()
-    CHATINTERN = auto()
-    DOLLY = auto()
-    RWKV = auto()
-    PHOENIX = auto()
-    ROBIN = auto()
-    FALCON_CHAT = auto()
-    CHATGLM3 = auto()
-    INTERNVL_ZH = auto()
-    MPT = auto()
-@dataclasses.dataclass
-class Conversation:
-    """A class that manages prompt templates and keeps all conversation history."""
-    # The name of this template
-    name: str
-    # The template of the system prompt
-    system_template: str = '{system_message}'
-    # The system message
-    system_message: str = ''
-    # The names of two roles
-    roles: Tuple[str] = ('USER', 'ASSISTANT')
-    # All messages. Each item is (role, message).
-    messages: List[List[str]] = ()
-    # The number of few shot examples
-    offset: int = 0
-    # The separator style and configurations
-    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
-    sep: str = '\n'
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: Union[str, List[str]] = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-    def get_prompt(self) -> str:
-        """Get the prompt for generation."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ': '  # must be end with a space
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
-            ret = '' if system_prompt == '' else system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + message + seps[i % 2]
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.RWKV:
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += (
-                        role
-                        + ': '
-                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
-                    )
-                    ret += '\n\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.LLAMA2:
-            seps = [self.sep, self.sep2]
-            if self.system_message:
-                ret = system_prompt
-            else:
-                ret = '[INST] '
-            for i, (role, message) in enumerate(self.messages):
-                tag = self.roles[i % 2]
-                if message:
-                    if i == 0:
-                        ret += message + ' '
-                    else:
-                        ret += tag + ' ' + message + seps[i % 2]
-                else:
-                    ret += tag
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM:
-            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
-            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
-            round_add_n = 1 if self.name == 'chatglm2' else 0
-            if system_prompt:
-                ret = system_prompt + self.sep
-            else:
-                ret = ''
-            for i, (role, message) in enumerate(self.messages):
-                if i % 2 == 0:
-                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
-                if message:
-                    ret += f'{role}：{message}{self.sep}'
-                else:
-                    ret += f'{role}：'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATML:
-            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep + '\n'
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM3:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + ' ' + message
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATINTERN:
-            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                # if i % 2 == 0:
-                #     ret += "<s>"
-                if message:
-                    ret += role + ':' + message + seps[i % 2] + '\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.DOLLY:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ':\n' + message + seps[i % 2]
-                    if i % 2 == 1:
-                        ret += '\n\n'
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.PHOENIX:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + '<s>' + message + '</s>'
-                else:
-                    ret += role + ': ' + '<s>'
-            return ret
-        elif self.sep_style == SeparatorStyle.ROBIN:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ':\n' + message + self.sep
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
-            seps = [self.sep, self.sep2]
-            ret = self.system_message + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.MPT:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        else:
-            raise ValueError(f'Invalid style: {self.sep_style}')
-    def set_system_message(self, system_message: str):
-        """Set the system message."""
-        self.system_message = system_message
-    def append_message(self, role: str, message: str):
-        """Append a new message."""
-        self.messages.append([role, message])
-    def update_last_message(self, message: str):
-        """Update the last output.
-        The last message is typically set to be None when constructing the prompt,
-        so we need to update it in-place after getting the response from a model.
-        """
-        self.messages[-1][1] = message
-    def to_gradio_chatbot(self):
-        """Convert the conversation to gradio chatbot format."""
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-    def to_openai_api_messages(self):
-        """Convert the conversation to OpenAI chat completion format."""
-        ret = [{'role': 'system', 'content': self.system_message}]
-        for i, (_, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append({'role': 'user', 'content': msg})
-            else:
-                if msg is not None:
-                    ret.append({'role': 'assistant', 'content': msg})
-        return ret
-    def copy(self):
-        return Conversation(
-            name=self.name,
-            system_template=self.system_template,
-            system_message=self.system_message,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-        )
-    def dict(self):
-        return {
-            'template_name': self.name,
-            'system_message': self.system_message,
-            'roles': self.roles,
-            'messages': self.messages,
-            'offset': self.offset,
-        }
-# A global registry for all conversation templates
-conv_templates: Dict[str, Conversation] = {}
-def register_conv_template(template: Conversation, override: bool = False):
-    """Register a new conversation template."""
-    if not override:
-        assert (
-            template.name not in conv_templates
-        ), f'{template.name} has been registered.'
-    conv_templates[template.name] = template
-def get_conv_template(name: str) -> Conversation:
-    """Get a conversation template."""
-    return conv_templates[name].copy()
-# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
-# is that during training, the preprocessing function for the Hermes-2 template doesn't add
-# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
-# Therefore, they are completely equivalent during inference.
-register_conv_template(
-    Conversation(
-        name='Hermes-2',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            6,
-            7,
-            8,
-        ],
-        stop_str='<|endoftext|>',
-    )
-)
-register_conv_template(
-    Conversation(
-        name='internlm2-chat',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            92543,
-            92542
-        ]
-    )
-)
-register_conv_template(
-    Conversation(
-        name='phi3-chat',
-        system_template='<|system|>\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
-        system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
-        roles=('<|user|>\n', '<|assistant|>\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|end|>',
-        stop_token_ids=[
-            2,
-            32000,
-            32007
-        ]
-    )
-)

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/generation_config.json DELETED Viewed

@@ -1,4 +0,0 @@
-{
-  "_from_model_config": true,
-  "transformers_version": "4.39.0"
-}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/model.safetensors DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:406ffc350bcee757ec9106d862d40fed29a47723513359bb7c0cf61ba1be99a1
-size 4411571040

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_intern_vit.py DELETED Viewed

@@ -1,435 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2024 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-from typing import Optional, Tuple, Union
-import torch
-import torch.nn.functional as F
-import torch.utils.checkpoint
-from einops import rearrange
-from timm.models.layers import DropPath
-from torch import nn
-from transformers.activations import ACT2FN
-from transformers.modeling_outputs import (BaseModelOutput,
-                                           BaseModelOutputWithPooling)
-from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import logging
-from .configuration_intern_vit import InternVisionConfig
-try:
-    try:  # v1
-        from flash_attn.flash_attn_interface import \
-            flash_attn_unpadded_qkvpacked_func
-    except:  # v2
-        from flash_attn.flash_attn_interface import \
-            flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func
-    from flash_attn.bert_padding import pad_input, unpad_input
-    has_flash_attn = True
-except:
-    print('FlashAttention is not installed.')
-    has_flash_attn = False
-logger = logging.get_logger(__name__)
-class FlashAttention(nn.Module):
-    """Implement the scaled dot product attention with softmax.
-    Arguments
-    ---------
-        softmax_scale: The temperature to use for the softmax attention.
-                      (default: 1/sqrt(d_keys) where d_keys is computed at
-                      runtime)
-        attention_dropout: The dropout rate to apply to the attention
-                           (default: 0.0)
-    """
-    def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
-        super().__init__()
-        self.softmax_scale = softmax_scale
-        self.dropout_p = attention_dropout
-    def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
-                max_s=None, need_weights=False):
-        """Implements the multihead softmax attention.
-        Arguments
-        ---------
-            qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
-                if unpadded: (nnz, 3, h, d)
-            key_padding_mask: a bool tensor of shape (B, S)
-        """
-        assert not need_weights
-        assert qkv.dtype in [torch.float16, torch.bfloat16]
-        assert qkv.is_cuda
-        if cu_seqlens is None:
-            batch_size = qkv.shape[0]
-            seqlen = qkv.shape[1]
-            if key_padding_mask is None:
-                qkv = rearrange(qkv, 'b s ... -> (b s) ...')
-                max_s = seqlen
-                cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
-                                          device=qkv.device)
-                output = flash_attn_unpadded_qkvpacked_func(
-                    qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
-            else:
-                nheads = qkv.shape[-2]
-                x = rearrange(qkv, 'b s three h d -> b s (three h d)')
-                x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
-                x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
-                output_unpad = flash_attn_unpadded_qkvpacked_func(
-                    x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
-                                             indices, batch_size, seqlen),
-                                   'b s (h d) -> b s h d', h=nheads)
-        else:
-            assert max_s is not None
-            output = flash_attn_unpadded_qkvpacked_func(
-                qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                softmax_scale=self.softmax_scale, causal=causal
-            )
-        return output, None
-class InternRMSNorm(nn.Module):
-    def __init__(self, hidden_size, eps=1e-6):
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-try:
-    from apex.normalization import FusedRMSNorm
-    InternRMSNorm = FusedRMSNorm  # noqa
-    logger.info('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm')
-except ImportError:
-    # using the normal InternRMSNorm
-    pass
-except Exception:
-    logger.warning('discovered apex but it failed to load, falling back to InternRMSNorm')
-    pass
-NORM2FN = {
-    'rms_norm': InternRMSNorm,
-    'layer_norm': nn.LayerNorm,
-}
-class InternVisionEmbeddings(nn.Module):
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.embed_dim = config.hidden_size
-        self.image_size = config.image_size
-        self.patch_size = config.patch_size
-        self.class_embedding = nn.Parameter(
-            torch.randn(1, 1, self.embed_dim),
-        )
-        self.patch_embedding = nn.Conv2d(
-            in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size
-        )
-        self.num_patches = (self.image_size // self.patch_size) ** 2
-        self.num_positions = self.num_patches + 1
-        self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim))
-    def _get_pos_embed(self, pos_embed, H, W):
-        target_dtype = pos_embed.dtype
-        pos_embed = pos_embed.float().reshape(
-            1, self.image_size // self.patch_size, self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
-        pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False). \
-            reshape(1, -1, H * W).permute(0, 2, 1).to(target_dtype)
-        return pos_embed
-    def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
-        target_dtype = self.patch_embedding.weight.dtype
-        patch_embeds = self.patch_embedding(pixel_values)  # shape = [*, channel, width, height]
-        batch_size, _, height, width = patch_embeds.shape
-        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
-        class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype)
-        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
-        position_embedding = torch.cat([
-            self.position_embedding[:, :1, :],
-            self._get_pos_embed(self.position_embedding[:, 1:, :], height, width)
-        ], dim=1)
-        embeddings = embeddings + position_embedding.to(target_dtype)
-        return embeddings
-class InternAttention(nn.Module):
-    """Multi-headed attention from 'Attention Is All You Need' paper"""
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.embed_dim = config.hidden_size
-        self.num_heads = config.num_attention_heads
-        self.use_flash_attn = config.use_flash_attn and has_flash_attn
-        if config.use_flash_attn and not has_flash_attn:
-            print('Warning: Flash Attention is not available, use_flash_attn is set to False.')
-        self.head_dim = self.embed_dim // self.num_heads
-        if self.head_dim * self.num_heads != self.embed_dim:
-            raise ValueError(
-                f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:'
-                f' {self.num_heads}).'
-            )
-        self.scale = self.head_dim ** -0.5
-        self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=config.qkv_bias)
-        self.attn_drop = nn.Dropout(config.attention_dropout)
-        self.proj_drop = nn.Dropout(config.dropout)
-        self.qk_normalization = config.qk_normalization
-        if self.qk_normalization:
-            self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
-            self.k_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
-        if self.use_flash_attn:
-            self.inner_attn = FlashAttention(attention_dropout=config.attention_dropout)
-        self.proj = nn.Linear(self.embed_dim, self.embed_dim)
-    def _naive_attn(self, x):
-        B, N, C = x.shape
-        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
-        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
-        if self.qk_normalization:
-            B_, H_, N_, D_ = q.shape
-            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-        attn = ((q * self.scale) @ k.transpose(-2, -1))
-        attn = attn.softmax(dim=-1)
-        attn = self.attn_drop(attn)
-        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
-        qkv = self.qkv(x)
-        qkv = rearrange(qkv, 'b s (three h d) -> b s three h d', three=3, h=self.num_heads)
-        if self.qk_normalization:
-            q, k, v = qkv.unbind(2)
-            q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
-            k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
-            qkv = torch.stack([q, k, v], dim=2)
-        context, _ = self.inner_attn(
-            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=False
-        )
-        outs = self.proj(rearrange(context, 'b s h d -> b s (h d)'))
-        outs = self.proj_drop(outs)
-        return outs
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        x = self._naive_attn(hidden_states) if not self.use_flash_attn else self._flash_attn(hidden_states)
-        return x
-class InternMLP(nn.Module):
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.act = ACT2FN[config.hidden_act]
-        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
-        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        hidden_states = self.fc1(hidden_states)
-        hidden_states = self.act(hidden_states)
-        hidden_states = self.fc2(hidden_states)
-        return hidden_states
-class InternVisionEncoderLayer(nn.Module):
-    def __init__(self, config: InternVisionConfig, drop_path_rate: float):
-        super().__init__()
-        self.embed_dim = config.hidden_size
-        self.intermediate_size = config.intermediate_size
-        self.norm_type = config.norm_type
-        self.attn = InternAttention(config)
-        self.mlp = InternMLP(config)
-        self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
-        self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
-        self.ls1 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
-        self.ls2 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
-        self.drop_path1 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
-        self.drop_path2 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
-    def forward(
-            self,
-            hidden_states: torch.Tensor,
-    ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor], Optional[Tuple[torch.FloatTensor]]]:
-        """
-        Args:
-            hidden_states (`Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]`): input to the layer of shape `(batch, seq_len, embed_dim)`
-        """
-        hidden_states = hidden_states + self.drop_path1(self.attn(self.norm1(hidden_states)) * self.ls1)
-        hidden_states = hidden_states + self.drop_path2(self.mlp(self.norm2(hidden_states)) * self.ls2)
-        return hidden_states
-class InternVisionEncoder(nn.Module):
-    """
-    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
-    [`InternEncoderLayer`].
-    Args:
-        config (`InternConfig`):
-            The corresponding vision configuration for the `InternEncoder`.
-    """
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        # stochastic depth decay rule
-        dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)]
-        self.layers = nn.ModuleList([
-            InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
-        self.gradient_checkpointing = True
-    def forward(
-            self,
-            inputs_embeds,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, BaseModelOutput]:
-        r"""
-        Args:
-            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
-                Embedded representation of the inputs. Should be float, not int tokens.
-            output_hidden_states (`bool`, *optional*):
-                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
-                for more detail.
-            return_dict (`bool`, *optional*):
-                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-        """
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        encoder_states = () if output_hidden_states else None
-        hidden_states = inputs_embeds
-        for idx, encoder_layer in enumerate(self.layers):
-            if output_hidden_states:
-                encoder_states = encoder_states + (hidden_states,)
-            if self.gradient_checkpointing and self.training:
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    encoder_layer,
-                    hidden_states)
-            else:
-                layer_outputs = encoder_layer(
-                    hidden_states,
-                )
-            hidden_states = layer_outputs
-        if output_hidden_states:
-            encoder_states = encoder_states + (hidden_states,)
-        if not return_dict:
-            return tuple(v for v in [hidden_states, encoder_states] if v is not None)
-        return BaseModelOutput(
-            last_hidden_state=hidden_states, hidden_states=encoder_states
-        )
-class InternVisionModel(PreTrainedModel):
-    main_input_name = 'pixel_values'
-    _supports_flash_attn_2 = True
-    config_class = InternVisionConfig
-    _no_split_modules = ['InternVisionEncoderLayer']
-    def __init__(self, config: InternVisionConfig):
-        super().__init__(config)
-        self.config = config
-        self.embeddings = InternVisionEmbeddings(config)
-        self.encoder = InternVisionEncoder(config)
-    def resize_pos_embeddings(self, old_size, new_size, patch_size):
-        pos_emb = self.embeddings.position_embedding
-        _, num_positions, embed_dim = pos_emb.shape
-        cls_emb = pos_emb[:, :1, :]
-        pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size, old_size // patch_size, -1).permute(0, 3, 1, 2)
-        pos_emb = F.interpolate(pos_emb.float(), size=new_size // patch_size, mode='bicubic', align_corners=False)
-        pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim, -1).permute(0, 2, 1)
-        pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
-        self.embeddings.position_embedding = nn.Parameter(pos_emb)
-        self.embeddings.image_size = new_size
-        logger.info('Resized position embeddings from {} to {}'.format(old_size, new_size))
-    def get_input_embeddings(self):
-        return self.embeddings
-    def forward(
-            self,
-            pixel_values: Optional[torch.FloatTensor] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-            pixel_embeds: Optional[torch.FloatTensor] = None,
-    ) -> Union[Tuple, BaseModelOutputWithPooling]:
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        if pixel_values is None and pixel_embeds is None:
-            raise ValueError('You have to specify pixel_values or pixel_embeds')
-        if pixel_embeds is not None:
-            hidden_states = pixel_embeds
-        else:
-            if len(pixel_values.shape) == 4:
-                hidden_states = self.embeddings(pixel_values)
-            else:
-                raise ValueError(f'wrong pixel_values size: {pixel_values.shape}')
-        encoder_outputs = self.encoder(
-            inputs_embeds=hidden_states,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        last_hidden_state = encoder_outputs.last_hidden_state
-        pooled_output = last_hidden_state[:, 0, :]
-        if not return_dict:
-            return (last_hidden_state, pooled_output) + encoder_outputs[1:]
-        return BaseModelOutputWithPooling(
-            last_hidden_state=last_hidden_state,
-            pooler_output=pooled_output,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-        )

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_internlm2.py DELETED Viewed

@@ -1,1415 +0,0 @@
-# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
-#
-# This code is based on transformers/src/transformers/models/llama/modeling_llama.py
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" PyTorch InternLM2 model."""
-import math
-import queue
-import threading
-import warnings
-from typing import List, Optional, Tuple, Union
-import torch
-import torch.nn.functional as F
-import torch.utils.checkpoint
-from einops import rearrange
-from torch import nn
-from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
-from transformers.activations import ACT2FN
-from transformers.modeling_outputs import (BaseModelOutputWithPast,
-                                           CausalLMOutputWithPast,
-                                           SequenceClassifierOutputWithPast)
-from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import (add_start_docstrings,
-                                add_start_docstrings_to_model_forward, logging,
-                                replace_return_docstrings)
-try:
-    from transformers.generation.streamers import BaseStreamer
-except:  # noqa # pylint: disable=bare-except
-    BaseStreamer = None
-from .configuration_internlm2 import InternLM2Config
-logger = logging.get_logger(__name__)
-_CONFIG_FOR_DOC = 'InternLM2Config'
-flash_attn_func, flash_attn_varlen_func = None, None
-pad_input, index_first_axis, unpad_input = None, None, None
-try:
-    from flash_attn import flash_attn_func as _flash_attn_func
-    from flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
-    from flash_attn.bert_padding import index_first_axis as _index_first_axis
-    from flash_attn.bert_padding import pad_input as _pad_input
-    from flash_attn.bert_padding import unpad_input as _unpad_input
-    flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func
-    pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input
-    has_flash_attn = True
-except:
-    has_flash_attn = False
-def _import_flash_attn():
-    global flash_attn_func, flash_attn_varlen_func
-    global pad_input, index_first_axis, unpad_input
-    try:
-        from flash_attn import flash_attn_func as _flash_attn_func
-        from flash_attn import \
-            flash_attn_varlen_func as _flash_attn_varlen_func
-        from flash_attn.bert_padding import \
-            index_first_axis as _index_first_axis
-        from flash_attn.bert_padding import pad_input as _pad_input
-        from flash_attn.bert_padding import unpad_input as _unpad_input
-        flash_attn_func, flash_attn_varlen_func = _flash_attn_func, _flash_attn_varlen_func
-        pad_input, index_first_axis, unpad_input = _pad_input, _index_first_axis, _unpad_input
-    except ImportError:
-        raise ImportError('flash_attn is not installed.')
-# Copied from transformers.models.llama.modeling_llama._get_unpad_data
-def _get_unpad_data(attention_mask):
-    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
-    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
-    max_seqlen_in_batch = seqlens_in_batch.max().item()
-    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
-    return (
-        indices,
-        cu_seqlens,
-        max_seqlen_in_batch,
-    )
-# Copied from transformers.models.bart.modeling_bart._make_causal_mask
-def _make_causal_mask(
-    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
-):
-    """
-    Make causal mask used for bi-directional self-attention.
-    """
-    bsz, tgt_len = input_ids_shape
-    mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)
-    mask_cond = torch.arange(mask.size(-1), device=device)
-    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
-    mask = mask.to(dtype)
-    if past_key_values_length > 0:
-        mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
-    return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
-# Copied from transformers.models.bart.modeling_bart._expand_mask
-def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
-    """
-    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
-    """
-    bsz, src_len = mask.size()
-    tgt_len = tgt_len if tgt_len is not None else src_len
-    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
-    inverted_mask = 1.0 - expanded_mask
-    return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
-# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->InternLM2
-class InternLM2RMSNorm(nn.Module):
-    def __init__(self, hidden_size, eps=1e-6):
-        """
-        InternLM2RMSNorm is equivalent to T5LayerNorm
-        """
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-# Copied from transformers.model.llama.modeling_llama.LlamaRotaryEmbedding with Llama->InternLM2
-class InternLM2RotaryEmbedding(nn.Module):
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
-        super().__init__()
-        self.dim = dim
-        self.max_position_embeddings = max_position_embeddings
-        self.base = base
-        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
-        self.register_buffer('inv_freq', inv_freq, persistent=False)
-        # Build here to make `torch.jit.trace` work.
-        self._set_cos_sin_cache(
-            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
-        )
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)
-        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)
-        self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)
-    def forward(self, x, seq_len=None):
-        # x: [bs, num_attention_heads, seq_len, head_size]
-        if seq_len > self.max_seq_len_cached:
-            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=torch.float32)
-        return (
-            self.cos_cached[:seq_len].to(dtype=x.dtype),
-            self.sin_cached[:seq_len].to(dtype=x.dtype),
-        )
-# Copied from transformers.model.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->InternLM2
-class InternLM2LinearScalingRotaryEmbedding(InternLM2RotaryEmbedding):
-    """InternLM2RotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
-        self.scaling_factor = scaling_factor
-        super().__init__(dim, max_position_embeddings, base, device)
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)
-        t = t / self.scaling_factor
-        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)
-        self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)
-# Copied from transformers.model.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->InternLM2
-class InternLM2DynamicNTKScalingRotaryEmbedding(InternLM2RotaryEmbedding):
-    """InternLM2RotaryEmbedding extended with Dynamic NTK scaling.
-    Credits to the Reddit users /u/bloc97 and /u/emozilla.
-    """
-    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
-        self.scaling_factor = scaling_factor
-        super().__init__(dim, max_position_embeddings, base, device)
-    def _set_cos_sin_cache(self, seq_len, device, dtype):
-        self.max_seq_len_cached = seq_len
-        if seq_len > self.max_position_embeddings:
-            base = self.base * (
-                (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
-            ) ** (self.dim / (self.dim - 2))
-            inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
-            self.register_buffer('inv_freq', inv_freq, persistent=False)
-        t = torch.arange(self.max_seq_len_cached, device=device).to(dtype=self.inv_freq.dtype)
-        freqs = torch.einsum('i,j->ij', t, self.inv_freq)
-        # Different from paper, but it uses a different permutation in order to obtain the same calculation
-        emb = torch.cat((freqs, freqs), dim=-1)
-        self.register_buffer('cos_cached', emb.cos().to(dtype), persistent=False)
-        self.register_buffer('sin_cached', emb.sin().to(dtype), persistent=False)
-# Copied from transformers.model.llama.modeling_llama.rotate_half
-def rotate_half(x):
-    """Rotates half the hidden dims of the input."""
-    x1 = x[..., : x.shape[-1] // 2]
-    x2 = x[..., x.shape[-1] // 2 :]
-    return torch.cat((-x2, x1), dim=-1)
-# Copied from transformers.model.llama.modeling_llama.apply_rotary_pos_emb
-def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
-    """Applies Rotary Position Embedding to the query and key tensors."""
-    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
-    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
-    q_embed = (q * cos) + (rotate_half(q) * sin)
-    k_embed = (k * cos) + (rotate_half(k) * sin)
-    return q_embed, k_embed
-class InternLM2MLP(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-        self.hidden_size = config.hidden_size
-        self.intermediate_size = config.intermediate_size
-        self.w1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
-        self.w3 = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
-        self.w2 = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
-        self.act_fn = ACT2FN[config.hidden_act]
-    def forward(self, x):
-        down_proj = self.w2(self.act_fn(self.w1(x)) * self.w3(x))
-        return down_proj
-# Copied from transformers.model.llama.modeling_llama.repeat_kv
-def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
-    """
-    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
-    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
-    """
-    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
-    if n_rep == 1:
-        return hidden_states
-    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
-    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
-# Modified from transformers.model.llama.modeling_llama.LlamaAttention
-class InternLM2Attention(nn.Module):
-    """Multi-headed attention from 'Attention Is All You Need' paper"""
-    def __init__(self, config: InternLM2Config):
-        super().__init__()
-        self.config = config
-        self.hidden_size = config.hidden_size
-        self.num_heads = config.num_attention_heads
-        self.head_dim = self.hidden_size // self.num_heads
-        self.num_key_value_heads = config.num_key_value_heads
-        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
-        self.max_position_embeddings = config.max_position_embeddings
-        self.is_causal = True
-        if (self.head_dim * self.num_heads) != self.hidden_size:
-            raise ValueError(
-                f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}'
-                f' and `num_heads`: {self.num_heads}).'
-            )
-        self.wqkv = nn.Linear(
-            self.hidden_size,
-            (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,
-            bias=config.bias,
-        )
-        self.wo = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)
-        self._init_rope()
-    def _init_rope(self):
-        if self.config.rope_scaling is None:
-            self.rotary_emb = InternLM2RotaryEmbedding(
-                self.head_dim,
-                max_position_embeddings=self.max_position_embeddings,
-                base=self.config.rope_theta,
-            )
-        else:
-            scaling_type = self.config.rope_scaling['type']
-            scaling_factor = self.config.rope_scaling['factor']
-            if scaling_type == 'dynamic':
-                self.rotary_emb = InternLM2DynamicNTKScalingRotaryEmbedding(
-                    self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
-                    base=self.config.rope_theta,
-                    scaling_factor=scaling_factor,
-                )
-            elif scaling_type == 'linear':
-                self.rotary_emb = InternLM2LinearScalingRotaryEmbedding(
-                    self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
-                    base=self.config.rope_theta,
-                    scaling_factor=scaling_factor,
-                )
-            else:
-                raise ValueError("Currently we only support rotary embedding's type being 'dynamic' or 'linear'.")
-        return self.rotary_emb
-    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
-        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_value: Optional[Tuple[torch.Tensor]] = None,
-        output_attentions: bool = False,
-        use_cache: bool = False,
-        **kwargs,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        if 'padding_mask' in kwargs:
-            warnings.warn(
-                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
-                'Please make sure use `attention_mask` instead.`'
-            )
-        bsz, q_len, _ = hidden_states.size()
-        qkv_states = self.wqkv(hidden_states)
-        qkv_states = rearrange(
-            qkv_states,
-            'b q (h gs d) -> b q h gs d',
-            gs=2 + self.num_key_value_groups,
-            d=self.head_dim,
-        )
-        query_states = qkv_states[..., : self.num_key_value_groups, :]
-        query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')
-        key_states = qkv_states[..., -2, :]
-        value_states = qkv_states[..., -1, :]
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        kv_seq_len = key_states.shape[-2]
-        if past_key_value is not None:
-            kv_seq_len += past_key_value[0].shape[-2]
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
-        if past_key_value is not None:
-            # reuse k, v, self_attention
-            key_states = torch.cat([past_key_value[0], key_states], dim=2)
-            value_states = torch.cat([past_key_value[1], value_states], dim=2)
-        past_key_value = (key_states, value_states) if use_cache else None
-        key_states = repeat_kv(key_states, self.num_key_value_groups)
-        value_states = repeat_kv(value_states, self.num_key_value_groups)
-        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
-        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
-            raise ValueError(
-                f'Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is'
-                f' {attn_weights.size()}'
-            )
-        if attention_mask is not None:
-            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
-                raise ValueError(
-                    f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}'
-                )
-            attn_weights = attn_weights + attention_mask
-        # upcast attention to fp32
-        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
-        attn_output = torch.matmul(attn_weights, value_states)
-        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
-            raise ValueError(
-                f'`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is'
-                f' {attn_output.size()}'
-            )
-        attn_output = attn_output.transpose(1, 2).contiguous()
-        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
-        attn_output = self.wo(attn_output)
-        if not output_attentions:
-            attn_weights = None
-        return attn_output, attn_weights, past_key_value
-# Modified from transformers.model.llama.modeling_llama.InternLM2FlashAttention2
-class InternLM2FlashAttention2(InternLM2Attention):
-    """
-    InternLM2 flash attention module. This module inherits from `InternLM2Attention` as the weights of the module stays
-    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
-    flash attention and deal with padding tokens in case the input contains any of them.
-    """
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        attention_mask: Optional[torch.LongTensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_value: Optional[Tuple[torch.Tensor]] = None,
-        output_attentions: bool = False,
-        use_cache: bool = False,
-        **kwargs,
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        # InternLM2FlashAttention2 attention does not support output_attentions
-        if 'padding_mask' in kwargs:
-            warnings.warn(
-                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
-                'Please make sure use `attention_mask` instead.`'
-            )
-            # overwrite attention_mask with padding_mask
-            attention_mask = kwargs.pop('padding_mask')
-        output_attentions = False
-        bsz, q_len, _ = hidden_states.size()
-        qkv_states = self.wqkv(hidden_states)
-        qkv_states = rearrange(
-            qkv_states,
-            'b q (h gs d) -> b q h gs d',
-            gs=2 + self.num_key_value_groups,
-            d=self.head_dim,
-        )
-        query_states = qkv_states[..., : self.num_key_value_groups, :]
-        query_states = rearrange(query_states, 'b q h gs d -> b q (h gs) d')
-        key_states = qkv_states[..., -2, :]
-        value_states = qkv_states[..., -1, :]
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        kv_seq_len = key_states.shape[-2]
-        if past_key_value is not None:
-            kv_seq_len += past_key_value[0].shape[-2]
-        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
-        if past_key_value is not None:
-            # reuse k, v, self_attention
-            key_states = torch.cat([past_key_value[0], key_states], dim=2)
-            value_states = torch.cat([past_key_value[1], value_states], dim=2)
-        past_key_value = (key_states, value_states) if use_cache else None
-        query_states = query_states.transpose(1, 2)
-        key_states = key_states.transpose(1, 2)
-        value_states = value_states.transpose(1, 2)
-        attn_output = self._flash_attention_forward(
-            query_states, key_states, value_states, attention_mask, q_len
-        )
-        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
-        attn_output = self.wo(attn_output)
-        if not output_attentions:
-            attn_weights = None
-        return attn_output, attn_weights, past_key_value
-    def _flash_attention_forward(
-        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
-    ):
-        """
-        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
-        first unpad the input, then computes the attention scores and pad the final attention scores.
-        Args:
-            query_states (`torch.Tensor`):
-                Input query states to be passed to Flash Attention API
-            key_states (`torch.Tensor`):
-                Input key states to be passed to Flash Attention API
-            value_states (`torch.Tensor`):
-                Input value states to be passed to Flash Attention API
-            attention_mask (`torch.Tensor`):
-                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
-                position of padding tokens and 1 for the position of non-padding tokens.
-            dropout (`int`, *optional*):
-                Attention dropout
-            softmax_scale (`float`, *optional*):
-                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
-        """
-        # Contains at least one padding token in the sequence
-        causal = self.is_causal and query_length != 1
-        if attention_mask is not None:
-            batch_size = query_states.shape[0]
-            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._unpad_input(
-                query_states, key_states, value_states, attention_mask, query_length
-            )
-            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
-            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
-            attn_output_unpad = flash_attn_varlen_func(
-                query_states,
-                key_states,
-                value_states,
-                cu_seqlens_q=cu_seqlens_q,
-                cu_seqlens_k=cu_seqlens_k,
-                max_seqlen_q=max_seqlen_in_batch_q,
-                max_seqlen_k=max_seqlen_in_batch_k,
-                dropout_p=dropout,
-                softmax_scale=softmax_scale,
-                causal=causal,
-            )
-            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
-        else:
-            attn_output = flash_attn_func(
-                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
-            )
-        return attn_output
-    def _unpad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
-        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
-        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
-        key_layer = index_first_axis(
-            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
-        )
-        value_layer = index_first_axis(
-            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
-        )
-        if query_length == kv_seq_len:
-            query_layer = index_first_axis(
-                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
-            )
-            cu_seqlens_q = cu_seqlens_k
-            max_seqlen_in_batch_q = max_seqlen_in_batch_k
-            indices_q = indices_k
-        elif query_length == 1:
-            max_seqlen_in_batch_q = 1
-            cu_seqlens_q = torch.arange(
-                batch_size + 1, dtype=torch.int32, device=query_layer.device
-            )  # There is a memcpy here, that is very bad.
-            indices_q = cu_seqlens_q[:-1]
-            query_layer = query_layer.squeeze(1)
-        else:
-            # The -q_len: slice assumes left padding.
-            attention_mask = attention_mask[:, -query_length:]
-            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
-        return (
-            query_layer,
-            key_layer,
-            value_layer,
-            indices_q.to(torch.int64),
-            (cu_seqlens_q, cu_seqlens_k),
-            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
-        )
-INTERNLM2_ATTENTION_CLASSES = {
-    'eager': InternLM2Attention,
-    'flash_attention_2': InternLM2FlashAttention2,
-}
-# Modified from transformers.model.llama.modeling_llama.LlamaDecoderLayer
-class InternLM2DecoderLayer(nn.Module):
-    def __init__(self, config: InternLM2Config):
-        super().__init__()
-        self.hidden_size = config.hidden_size
-        self.attention = INTERNLM2_ATTENTION_CLASSES[config.attn_implementation](config=config)
-        self.feed_forward = InternLM2MLP(config)
-        self.attention_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.ffn_norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_value: Optional[Tuple[torch.Tensor]] = None,
-        output_attentions: Optional[bool] = False,
-        use_cache: Optional[bool] = False,
-        **kwargs,
-    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
-        """
-        Args:
-            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
-            attention_mask (`torch.FloatTensor`, *optional*):
-                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
-                query_sequence_length, key_sequence_length)` if default attention is used.
-            output_attentions (`bool`, *optional*):
-                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
-                returned tensors for more detail.
-            use_cache (`bool`, *optional*):
-                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
-                (see `past_key_values`).
-            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
-        """
-        if 'padding_mask' in kwargs:
-            warnings.warn(
-                'Passing `padding_mask` is deprecated and will be removed in v4.37. '
-                'Please make sure use `attention_mask` instead.`'
-            )
-        residual = hidden_states
-        hidden_states = self.attention_norm(hidden_states)
-        # Self Attention
-        hidden_states, self_attn_weights, present_key_value = self.attention(
-            hidden_states=hidden_states,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_value=past_key_value,
-            output_attentions=output_attentions,
-            use_cache=use_cache,
-            **kwargs,
-        )
-        hidden_states = residual + hidden_states
-        # Fully Connected
-        residual = hidden_states
-        hidden_states = self.ffn_norm(hidden_states)
-        hidden_states = self.feed_forward(hidden_states)
-        hidden_states = residual + hidden_states
-        outputs = (hidden_states,)
-        if output_attentions:
-            outputs += (self_attn_weights,)
-        if use_cache:
-            outputs += (present_key_value,)
-        return outputs
-InternLM2_START_DOCSTRING = r"""
-    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
-    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
-    etc.)
-    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
-    and behavior.
-    Parameters:
-        config ([`InternLM2Config`]):
-            Model configuration class with all the parameters of the model. Initializing with a config file does not
-            load the weights associated with the model, only the configuration. Check out the
-            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
-"""
-# Copied from transformers.models.llama.modeling_llama.LlamaPreTrainedModel with Llama->InternLM2
-@add_start_docstrings(
-    'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',
-    InternLM2_START_DOCSTRING,
-)
-class InternLM2PreTrainedModel(PreTrainedModel):
-    config_class = InternLM2Config
-    base_model_prefix = 'model'
-    supports_gradient_checkpointing = True
-    _no_split_modules = ['InternLM2DecoderLayer']
-    _skip_keys_device_placement = 'past_key_values'
-    _supports_flash_attn_2 = True
-    def _init_weights(self, module):
-        std = self.config.initializer_range
-        if isinstance(module, nn.Linear):
-            module.weight.data.normal_(mean=0.0, std=std)
-            if module.bias is not None:
-                module.bias.data.zero_()
-        elif isinstance(module, nn.Embedding):
-            module.weight.data.normal_(mean=0.0, std=std)
-            if module.padding_idx is not None:
-                module.weight.data[module.padding_idx].zero_()
-InternLM2_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
-            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
-            it.
-            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
-            [`PreTrainedTokenizer.__call__`] for details.
-            [What are input IDs?](../glossary#input-ids)
-        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
-            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-            [What are attention masks?](../glossary#attention-mask)
-            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
-            [`PreTrainedTokenizer.__call__`] for details.
-            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
-            `past_key_values`).
-            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
-            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
-            information on the default strategy.
-            - 1 indicates the head is **not masked**,
-            - 0 indicates the head is **masked**.
-        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
-            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
-            config.n_positions - 1]`.
-            [What are position IDs?](../glossary#position-ids)
-        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or
-            when `config.use_cache=True`):
-            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
-            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
-            `(batch_size, num_heads, decoder_sequence_length, embed_size_per_head)`.
-            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
-            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
-            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
-            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
-            of shape `(batch_size, sequence_length)`.
-        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
-            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
-            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
-            model's internal embedding lookup matrix.
-        use_cache (`bool`, *optional*):
-            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
-            `past_key_values`).
-        output_attentions (`bool`, *optional*):
-            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
-            tensors for more detail.
-        output_hidden_states (`bool`, *optional*):
-            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
-            more detail.
-        return_dict (`bool`, *optional*):
-            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-"""
-# Modified from transformers.model.llama.modeling_llama.LlamaModel
-@add_start_docstrings(
-    'The bare InternLM2 Model outputting raw hidden-states without any specific head on top.',
-    InternLM2_START_DOCSTRING,
-)
-class InternLM2Model(InternLM2PreTrainedModel):
-    """
-    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLM2DecoderLayer`]
-    Args:
-        config: InternLM2Config
-    """
-    _auto_class = 'AutoModel'
-    def __init__(self, config: InternLM2Config):
-        super().__init__(config)
-        self.padding_idx = config.pad_token_id
-        self.vocab_size = config.vocab_size
-        self.config = config
-        if not has_flash_attn:
-            self.config.attn_implementation = 'eager'
-            print('Warning: Flash attention is not available, using eager attention instead.')
-        self.tok_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
-        self.layers = nn.ModuleList([InternLM2DecoderLayer(config) for _ in range(config.num_hidden_layers)])
-        self.norm = InternLM2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
-        self.gradient_checkpointing = False
-        # Initialize weights and apply final processing
-        self.post_init()
-    def get_input_embeddings(self):
-        return self.tok_embeddings
-    def set_input_embeddings(self, value):
-        self.tok_embeddings = value
-    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
-        # create causal mask
-        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
-        combined_attention_mask = None
-        if input_shape[-1] > 1:
-            combined_attention_mask = _make_causal_mask(
-                input_shape,
-                inputs_embeds.dtype,
-                device=inputs_embeds.device,
-                past_key_values_length=past_key_values_length,
-            )
-        if attention_mask is not None:
-            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
-            expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
-                inputs_embeds.device
-            )
-            combined_attention_mask = (
-                expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
-            )
-        return combined_attention_mask
-    @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)
-    def forward(
-        self,
-        input_ids: torch.LongTensor = None,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_values: Optional[List[torch.FloatTensor]] = None,
-        inputs_embeds: Optional[torch.FloatTensor] = None,
-        use_cache: Optional[bool] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, BaseModelOutputWithPast]:
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        use_cache = use_cache if use_cache is not None else self.config.use_cache
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        if self.config.attn_implementation == 'flash_attention_2':
-            _import_flash_attn()
-        # retrieve input_ids and inputs_embeds
-        if input_ids is not None and inputs_embeds is not None:
-            raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
-        elif input_ids is not None:
-            batch_size, seq_length = input_ids.shape[:2]
-        elif inputs_embeds is not None:
-            batch_size, seq_length = inputs_embeds.shape[:2]
-        else:
-            raise ValueError('You have to specify either input_ids or inputs_embeds')
-        seq_length_with_past = seq_length
-        past_key_values_length = 0
-        if past_key_values is not None:
-            past_key_values_length = past_key_values[0][0].shape[2]
-            seq_length_with_past = seq_length_with_past + past_key_values_length
-        if position_ids is None:
-            device = input_ids.device if input_ids is not None else inputs_embeds.device
-            position_ids = torch.arange(
-                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
-            )
-            position_ids = position_ids.unsqueeze(0)
-        if inputs_embeds is None:
-            inputs_embeds = self.tok_embeddings(input_ids)
-        if self.config.attn_implementation == 'flash_attention_2':
-            # 2d mask is passed through the layers
-            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
-        else:
-            if attention_mask is None:
-                attention_mask = torch.ones(
-                    (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device
-                )
-            attention_mask = self._prepare_decoder_attention_mask(
-                attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
-            )
-        # embed positions
-        hidden_states = inputs_embeds
-        if self.gradient_checkpointing and self.training:
-            if use_cache:
-                logger.warning_once(
-                    '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...'
-                )
-                use_cache = False
-        # decoder layers
-        all_hidden_states = () if output_hidden_states else None
-        all_self_attns = () if output_attentions else None
-        next_decoder_cache = () if use_cache else None
-        for idx, decoder_layer in enumerate(self.layers):
-            if output_hidden_states:
-                all_hidden_states += (hidden_states,)
-            past_key_value = past_key_values[idx] if past_key_values is not None else None
-            if self.gradient_checkpointing and self.training:
-                def create_custom_forward(module):
-                    def custom_forward(*inputs):
-                        # None for past_key_value
-                        return module(*inputs, output_attentions, None)
-                    return custom_forward
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    create_custom_forward(decoder_layer),
-                    hidden_states,
-                    attention_mask,
-                    position_ids,
-                    None,
-                )
-            else:
-                layer_outputs = decoder_layer(
-                    hidden_states,
-                    attention_mask=attention_mask,
-                    position_ids=position_ids,
-                    past_key_value=past_key_value,
-                    output_attentions=output_attentions,
-                    use_cache=use_cache,
-                )
-            hidden_states = layer_outputs[0]
-            if use_cache:
-                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
-            if output_attentions:
-                all_self_attns += (layer_outputs[1],)
-        hidden_states = self.norm(hidden_states)
-        # add hidden states from the last decoder layer
-        if output_hidden_states:
-            all_hidden_states += (hidden_states,)
-        next_cache = next_decoder_cache if use_cache else None
-        if not return_dict:
-            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
-        return BaseModelOutputWithPast(
-            last_hidden_state=hidden_states,
-            past_key_values=next_cache,
-            hidden_states=all_hidden_states,
-            attentions=all_self_attns,
-        )
-# Modified from transformers.model.llama.modeling_llama.LlamaForCausalLM
-class InternLM2ForCausalLM(InternLM2PreTrainedModel):
-    _auto_class = 'AutoModelForCausalLM'
-    _tied_weights_keys = ['output.weight']
-    def __init__(self, config):
-        super().__init__(config)
-        self.model = InternLM2Model(config)
-        self.vocab_size = config.vocab_size
-        self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-        # Initialize weights and apply final processing
-        self.post_init()
-    def get_input_embeddings(self):
-        return self.model.tok_embeddings
-    def set_input_embeddings(self, value):
-        self.model.tok_embeddings = value
-    def get_output_embeddings(self):
-        return self.output
-    def set_output_embeddings(self, new_embeddings):
-        self.output = new_embeddings
-    def set_decoder(self, decoder):
-        self.model = decoder
-    def get_decoder(self):
-        return self.model
-    @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)
-    @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
-    def forward(
-        self,
-        input_ids: torch.LongTensor = None,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_values: Optional[List[torch.FloatTensor]] = None,
-        inputs_embeds: Optional[torch.FloatTensor] = None,
-        labels: Optional[torch.LongTensor] = None,
-        use_cache: Optional[bool] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, CausalLMOutputWithPast]:
-        r"""
-        Args:
-            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
-                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
-                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
-                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
-        Returns:
-        Example:
-        ```python
-        >>> from transformers import AutoTokenizer, InternLM2ForCausalLM
-        >>> model = InternLM2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
-        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
-        >>> prompt = "Hey, are you conscious? Can you talk to me?"
-        >>> inputs = tokenizer(prompt, return_tensors="pt")
-        >>> # Generate
-        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
-        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
-        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
-        ```"""
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
-        outputs = self.model(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_values=past_key_values,
-            inputs_embeds=inputs_embeds,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        hidden_states = outputs[0]
-        logits = self.output(hidden_states)
-        logits = logits.float()
-        loss = None
-        if labels is not None:
-            # Shift so that tokens < n predict n
-            shift_logits = logits[..., :-1, :].contiguous()
-            shift_labels = labels[..., 1:].contiguous()
-            # Flatten the tokens
-            loss_fct = CrossEntropyLoss()
-            shift_logits = shift_logits.view(-1, self.config.vocab_size)
-            shift_labels = shift_labels.view(-1)
-            # Enable model parallelism
-            shift_labels = shift_labels.to(shift_logits.device)
-            loss = loss_fct(shift_logits, shift_labels)
-        if not return_dict:
-            output = (logits,) + outputs[1:]
-            return (loss,) + output if loss is not None else output
-        device = input_ids.device if input_ids is not None else inputs_embeds.device
-        output = CausalLMOutputWithPast(
-            loss=loss,
-            logits=logits,
-            past_key_values=outputs.past_key_values,
-            hidden_states=outputs.hidden_states,
-            attentions=outputs.attentions,
-        )
-        output['logits'] = output['logits'].to(device)
-        return output
-    def prepare_inputs_for_generation(
-        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
-    ):
-        if past_key_values is not None:
-            past_length = past_key_values[0][0].shape[2]
-            # Some generation methods already pass only the last input ID
-            if input_ids.shape[1] > past_length:
-                remove_prefix_length = past_length
-            else:
-                # Default to old behavior: keep only final ID
-                remove_prefix_length = input_ids.shape[1] - 1
-            input_ids = input_ids[:, remove_prefix_length:]
-        position_ids = kwargs.get('position_ids', None)
-        if attention_mask is not None and position_ids is None:
-            # create position_ids on the fly for batch generation
-            position_ids = attention_mask.long().cumsum(-1) - 1
-            position_ids.masked_fill_(attention_mask == 0, 1)
-            if past_key_values:
-                position_ids = position_ids[:, -input_ids.shape[1] :]
-        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
-        if inputs_embeds is not None and past_key_values is None:
-            model_inputs = {'inputs_embeds': inputs_embeds}
-        else:
-            model_inputs = {'input_ids': input_ids}
-        model_inputs.update(
-            {
-                'position_ids': position_ids,
-                'past_key_values': past_key_values,
-                'use_cache': kwargs.get('use_cache'),
-                'attention_mask': attention_mask,
-            }
-        )
-        return model_inputs
-    @staticmethod
-    def _reorder_cache(past_key_values, beam_idx):
-        reordered_past = ()
-        for layer_past in past_key_values:
-            reordered_past += (
-                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
-            )
-        return reordered_past
-    def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = [], meta_instruction=''):
-        if tokenizer.add_bos_token:
-            prompt = ''
-        else:
-            prompt = tokenizer.bos_token
-        if meta_instruction:
-            prompt += f"""<|im_start|>system\n{meta_instruction}<|im_end|>\n"""
-        for record in history:
-            prompt += f"""<|im_start|>user\n{record[0]}<|im_end|>\n<|im_start|>assistant\n{record[1]}<|im_end|>\n"""
-        prompt += f"""<|im_start|>user\n{query}<|im_end|>\n<|im_start|>assistant\n"""
-        return tokenizer([prompt], return_tensors='pt')
-    @torch.no_grad()
-    def chat(
-        self,
-        tokenizer,
-        query: str,
-        history: List[Tuple[str, str]] = [],
-        streamer: Optional[BaseStreamer] = None,
-        max_new_tokens: int = 1024,
-        do_sample: bool = True,
-        temperature: float = 0.8,
-        top_p: float = 0.8,
-        meta_instruction: str = 'You are an AI assistant whose name is InternLM (书生·浦语).\n'
-        '- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n'
-        '- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.',
-        **kwargs,
-    ):
-        inputs = self.build_inputs(tokenizer, query, history, meta_instruction)
-        inputs = {k: v.to(self.device) for k, v in inputs.items() if torch.is_tensor(v)}
-        # also add end-of-assistant token in eos token id to avoid unnecessary generation
-        eos_token_id = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(['<|im_end|>'])[0]]
-        outputs = self.generate(
-            **inputs,
-            streamer=streamer,
-            max_new_tokens=max_new_tokens,
-            do_sample=do_sample,
-            temperature=temperature,
-            top_p=top_p,
-            eos_token_id=eos_token_id,
-            **kwargs,
-        )
-        outputs = outputs[0].cpu().tolist()[len(inputs['input_ids'][0]) :]
-        response = tokenizer.decode(outputs, skip_special_tokens=True)
-        response = response.split('<|im_end|>')[0]
-        history = history + [(query, response)]
-        return response, history
-    @torch.no_grad()
-    def stream_chat(
-        self,
-        tokenizer,
-        query: str,
-        history: List[Tuple[str, str]] = [],
-        max_new_tokens: int = 1024,
-        do_sample: bool = True,
-        temperature: float = 0.8,
-        top_p: float = 0.8,
-        **kwargs,
-    ):
-        """
-        Return a generator in format: (response, history)
-        Eg.
-        ('你好，有什么可以帮助您的吗', [('你好', '你好，有什么可以帮助您的吗')])
-        ('你好，有什么可以帮助您的吗？', [('你好', '你好，有什么可以帮助您的吗？')])
-        """
-        if BaseStreamer is None:
-            raise ModuleNotFoundError(
-                'The version of `transformers` is too low. Please make sure '
-                'that you have installed `transformers>=4.28.0`.'
-            )
-        response_queue = queue.Queue(maxsize=20)
-        class ChatStreamer(BaseStreamer):
-            def __init__(self, tokenizer) -> None:
-                super().__init__()
-                self.tokenizer = tokenizer
-                self.queue = response_queue
-                self.query = query
-                self.history = history
-                self.response = ''
-                self.cache = []
-                self.received_inputs = False
-                self.queue.put((self.response, history + [(self.query, self.response)]))
-            def put(self, value):
-                if len(value.shape) > 1 and value.shape[0] > 1:
-                    raise ValueError('ChatStreamer only supports batch size 1')
-                elif len(value.shape) > 1:
-                    value = value[0]
-                if not self.received_inputs:
-                    # The first received value is input_ids, ignore here
-                    self.received_inputs = True
-                    return
-                self.cache.extend(value.tolist())
-                token = self.tokenizer.decode(self.cache, skip_special_tokens=True)
-                if token.strip() != '<|im_end|>':
-                    self.response = self.response + token
-                    history = self.history + [(self.query, self.response)]
-                    self.queue.put((self.response, history))
-                    self.cache = []
-                else:
-                    self.end()
-            def end(self):
-                self.queue.put(None)
-        def stream_producer():
-            return self.chat(
-                tokenizer=tokenizer,
-                query=query,
-                streamer=ChatStreamer(tokenizer=tokenizer),
-                history=history,
-                max_new_tokens=max_new_tokens,
-                do_sample=do_sample,
-                temperature=temperature,
-                top_p=top_p,
-                **kwargs,
-            )
-        def consumer():
-            producer = threading.Thread(target=stream_producer)
-            producer.start()
-            while True:
-                res = response_queue.get()
-                if res is None:
-                    return
-                yield res
-        return consumer()
-# Copied from transformers.model.llama.modeling_llama.LlamaForSequenceClassification with Llama->InternLM2
-@add_start_docstrings(
-    """
-    The InternLM2 Model transformer with a sequence classification head on top (linear layer).
-    [`InternLM2ForSequenceClassification`] uses the last token in order to do the classification,
-    as other causal models (e.g. GPT-2) do.
-    Since it does classification on the last token, it requires to know the position of the last token. If a
-    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
-    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
-    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
-    each row of the batch).
-    """,
-    InternLM2_START_DOCSTRING,
-)
-class InternLM2ForSequenceClassification(InternLM2PreTrainedModel):
-    def __init__(self, config):
-        super().__init__(config)
-        self.num_labels = config.num_labels
-        self.model = InternLM2Model(config)
-        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
-        # Initialize weights and apply final processing
-        self.post_init()
-    def get_input_embeddings(self):
-        return self.model.tok_embeddings
-    def set_input_embeddings(self, value):
-        self.model.tok_embeddings = value
-    @add_start_docstrings_to_model_forward(InternLM2_INPUTS_DOCSTRING)
-    def forward(
-        self,
-        input_ids: torch.LongTensor = None,
-        attention_mask: Optional[torch.Tensor] = None,
-        position_ids: Optional[torch.LongTensor] = None,
-        past_key_values: Optional[List[torch.FloatTensor]] = None,
-        inputs_embeds: Optional[torch.FloatTensor] = None,
-        labels: Optional[torch.LongTensor] = None,
-        use_cache: Optional[bool] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
-        r"""
-        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
-            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
-            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
-            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        transformer_outputs = self.model(
-            input_ids,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_values=past_key_values,
-            inputs_embeds=inputs_embeds,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        hidden_states = transformer_outputs[0]
-        logits = self.score(hidden_states)
-        if input_ids is not None:
-            batch_size = input_ids.shape[0]
-        else:
-            batch_size = inputs_embeds.shape[0]
-        if self.config.pad_token_id is None and batch_size != 1:
-            raise ValueError('Cannot handle batch sizes > 1 if no padding token is defined.')
-        if self.config.pad_token_id is None:
-            sequence_lengths = -1
-        else:
-            if input_ids is not None:
-                sequence_lengths = (torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1).to(
-                    logits.device
-                )
-            else:
-                sequence_lengths = -1
-        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
-        loss = None
-        if labels is not None:
-            labels = labels.to(logits.device)
-            if self.config.problem_type is None:
-                if self.num_labels == 1:
-                    self.config.problem_type = 'regression'
-                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
-                    self.config.problem_type = 'single_label_classification'
-                else:
-                    self.config.problem_type = 'multi_label_classification'
-            if self.config.problem_type == 'regression':
-                loss_fct = MSELoss()
-                if self.num_labels == 1:
-                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
-                else:
-                    loss = loss_fct(pooled_logits, labels)
-            elif self.config.problem_type == 'single_label_classification':
-                loss_fct = CrossEntropyLoss()
-                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
-            elif self.config.problem_type == 'multi_label_classification':
-                loss_fct = BCEWithLogitsLoss()
-                loss = loss_fct(pooled_logits, labels)
-        if not return_dict:
-            output = (pooled_logits,) + transformer_outputs[1:]
-            return ((loss,) + output) if loss is not None else output
-        return SequenceClassifierOutputWithPast(
-            loss=loss,
-            logits=pooled_logits,
-            past_key_values=transformer_outputs.past_key_values,
-            hidden_states=transformer_outputs.hidden_states,
-            attentions=transformer_outputs.attentions,
-        )

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/modeling_internvl_chat.py DELETED Viewed

@@ -1,345 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2024 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-import warnings
-from typing import Any, List, Optional, Tuple, Union
-import torch.utils.checkpoint
-import transformers
-from torch import nn
-from torch.nn import CrossEntropyLoss
-from transformers import (AutoModel, GenerationConfig, LlamaForCausalLM,
-                          LlamaTokenizer)
-from transformers.modeling_outputs import CausalLMOutputWithPast
-from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import ModelOutput, logging
-from .configuration_internvl_chat import InternVLChatConfig
-from .conversation import get_conv_template
-from .modeling_intern_vit import InternVisionModel
-from .modeling_internlm2 import InternLM2ForCausalLM
-logger = logging.get_logger(__name__)
-def version_cmp(v1, v2, op='eq'):
-    import operator
-    from packaging import version
-    op_func = getattr(operator, op)
-    return op_func(version.parse(v1), version.parse(v2))
-class InternVLChatModel(PreTrainedModel):
-    config_class = InternVLChatConfig
-    main_input_name = 'pixel_values'
-    _supports_flash_attn_2 = True
-    _no_split_modules = ['InternVisionModel', 'LlamaDecoderLayer', 'InternLM2DecoderLayer']
-    def __init__(self, config: InternVLChatConfig, vision_model=None, language_model=None):
-        super().__init__(config)
-        assert version_cmp(transformers.__version__, '4.36.2', 'ge')
-        image_size = config.force_image_size or config.vision_config.image_size
-        patch_size = config.vision_config.patch_size
-        self.patch_size = patch_size
-        self.select_layer = config.select_layer
-        self.template = config.template
-        self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
-        self.downsample_ratio = config.downsample_ratio
-        self.ps_version = config.ps_version
-        logger.info(f'num_image_token: {self.num_image_token}')
-        logger.info(f'ps_version: {self.ps_version}')
-        if vision_model is not None:
-            self.vision_model = vision_model
-        else:
-            self.vision_model = InternVisionModel(config.vision_config)
-        if language_model is not None:
-            self.language_model = language_model
-        else:
-            if config.llm_config.architectures[0] == 'LlamaForCausalLM':
-                self.language_model = LlamaForCausalLM(config.llm_config)
-            elif config.llm_config.architectures[0] == 'InternLM2ForCausalLM':
-                self.language_model = InternLM2ForCausalLM(config.llm_config)
-            else:
-                raise NotImplementedError(f'{config.llm_config.architectures[0]} is not implemented.')
-        vit_hidden_size = config.vision_config.hidden_size
-        llm_hidden_size = config.llm_config.hidden_size
-        self.mlp1 = nn.Sequential(
-            nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
-            nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size),
-            nn.GELU(),
-            nn.Linear(llm_hidden_size, llm_hidden_size)
-        )
-        self.img_context_token_id = None
-        self.conv_template = get_conv_template(self.template)
-        self.system_message = self.conv_template.system_message
-    def forward(
-            self,
-            pixel_values: torch.FloatTensor,
-            input_ids: torch.LongTensor = None,
-            attention_mask: Optional[torch.Tensor] = None,
-            position_ids: Optional[torch.LongTensor] = None,
-            image_flags: Optional[torch.LongTensor] = None,
-            past_key_values: Optional[List[torch.FloatTensor]] = None,
-            labels: Optional[torch.LongTensor] = None,
-            use_cache: Optional[bool] = None,
-            output_attentions: Optional[bool] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, CausalLMOutputWithPast]:
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        image_flags = image_flags.squeeze(-1)
-        input_embeds = self.language_model.get_input_embeddings()(input_ids)
-        vit_embeds = self.extract_feature(pixel_values)
-        vit_embeds = vit_embeds[image_flags == 1]
-        vit_batch_size = pixel_values.shape[0]
-        B, N, C = input_embeds.shape
-        input_embeds = input_embeds.reshape(B * N, C)
-        if torch.distributed.get_rank() == 0:
-            print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}')
-        input_ids = input_ids.reshape(B * N)
-        selected = (input_ids == self.img_context_token_id)
-        try:
-            input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C)
-        except Exception as e:
-            vit_embeds = vit_embeds.reshape(-1, C)
-            print(f'warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, '
-                  f'vit_embeds.shape={vit_embeds.shape}')
-            n_token = selected.sum()
-            input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds[:n_token]
-        input_embeds = input_embeds.reshape(B, N, C)
-        outputs = self.language_model(
-            inputs_embeds=input_embeds,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        logits = outputs.logits
-        loss = None
-        if labels is not None:
-            # Shift so that tokens < n predict n
-            shift_logits = logits[..., :-1, :].contiguous()
-            shift_labels = labels[..., 1:].contiguous()
-            # Flatten the tokens
-            loss_fct = CrossEntropyLoss()
-            shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size)
-            shift_labels = shift_labels.view(-1)
-            # Enable model parallelism
-            shift_labels = shift_labels.to(shift_logits.device)
-            loss = loss_fct(shift_logits, shift_labels)
-        if not return_dict:
-            output = (logits,) + outputs[1:]
-            return (loss,) + output if loss is not None else output
-        return CausalLMOutputWithPast(
-            loss=loss,
-            logits=logits,
-            past_key_values=outputs.past_key_values,
-            hidden_states=outputs.hidden_states,
-            attentions=outputs.attentions,
-        )
-    def pixel_shuffle(self, x, scale_factor=0.5):
-        n, w, h, c = x.size()
-        # N, W, H, C --> N, W, H * scale, C // scale
-        x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
-        # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
-        x = x.permute(0, 2, 1, 3).contiguous()
-        # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
-        x = x.view(n, int(h * scale_factor), int(w * scale_factor),
-                   int(c / (scale_factor * scale_factor)))
-        if self.ps_version == 'v1':
-            warnings.warn("In ps_version 'v1', the height and width have not been swapped back, "
-                          'which results in a transposed image.')
-        else:
-            x = x.permute(0, 2, 1, 3).contiguous()
-        return x
-    def extract_feature(self, pixel_values):
-        if self.select_layer == -1:
-            vit_embeds = self.vision_model(
-                pixel_values=pixel_values,
-                output_hidden_states=False,
-                return_dict=True).last_hidden_state
-        else:
-            vit_embeds = self.vision_model(
-                pixel_values=pixel_values,
-                output_hidden_states=True,
-                return_dict=True).hidden_states[self.select_layer]
-        vit_embeds = vit_embeds[:, 1:, :]
-        h = w = int(vit_embeds.shape[1] ** 0.5)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
-        vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
-        vit_embeds = self.mlp1(vit_embeds)
-        return vit_embeds
-    def batch_chat(self, tokenizer, pixel_values, questions, generation_config, num_patches_list=None,
-                   history=None, return_history=False, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>',
-                   IMG_CONTEXT_TOKEN='<IMG_CONTEXT>', verbose=False, image_counts=None):
-        if history is not None or return_history:
-            print('Now multi-turn chat is not supported in batch_chat.')
-            raise NotImplementedError
-        if image_counts is not None:
-            num_patches_list = image_counts
-            print('Warning: `image_counts` is deprecated. Please use `num_patches_list` instead.')
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-        queries = []
-        for idx, num_patches in enumerate(num_patches_list):
-            question = questions[idx]
-            if pixel_values is not None and '<image>' not in question:
-                question = '<image>\n' + question
-            template = get_conv_template(self.template)
-            template.append_message(template.roles[0], question)
-            template.append_message(template.roles[1], None)
-            query = template.get_prompt()
-            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-            queries.append(query)
-        tokenizer.padding_side = 'left'
-        model_inputs = tokenizer(queries, return_tensors='pt', padding=True)
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
-        responses = [response.split(template.sep)[0].strip() for response in responses]
-        return responses
-    def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
-             num_patches_list=None, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>', IMG_CONTEXT_TOKEN='<IMG_CONTEXT>',
-             verbose=False):
-        if history is None and pixel_values is not None and '<image>' not in question:
-            question = '<image>\n' + question
-        if num_patches_list is None:
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-        template = get_conv_template(self.template)
-        template.system_message = self.system_message
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-        history = [] if history is None else history
-        for (old_question, old_answer) in history:
-            template.append_message(template.roles[0], old_question)
-            template.append_message(template.roles[1], old_answer)
-        template.append_message(template.roles[0], question)
-        template.append_message(template.roles[1], None)
-        query = template.get_prompt()
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-        for num_patches in num_patches_list:
-            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-        model_inputs = tokenizer(query, return_tensors='pt')
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
-        response = response.split(template.sep)[0].strip()
-        history.append((question, response))
-        if return_history:
-            return response, history
-        else:
-            query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-            query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-            if verbose:
-                print(query_to_print, response)
-            return response
-    @torch.no_grad()
-    def generate(
-            self,
-            pixel_values: Optional[torch.FloatTensor] = None,
-            input_ids: Optional[torch.FloatTensor] = None,
-            attention_mask: Optional[torch.LongTensor] = None,
-            visual_features: Optional[torch.FloatTensor] = None,
-            generation_config: Optional[GenerationConfig] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-            **generate_kwargs,
-    ) -> torch.LongTensor:
-        assert self.img_context_token_id is not None
-        if pixel_values is not None:
-            if visual_features is not None:
-                vit_embeds = visual_features
-            else:
-                vit_embeds = self.extract_feature(pixel_values)
-            input_embeds = self.language_model.get_input_embeddings()(input_ids)
-            B, N, C = input_embeds.shape
-            input_embeds = input_embeds.reshape(B * N, C)
-            input_ids = input_ids.reshape(B * N)
-            selected = (input_ids == self.img_context_token_id)
-            assert selected.sum() != 0
-            input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
-            input_embeds = input_embeds.reshape(B, N, C)
-        else:
-            input_embeds = self.language_model.get_input_embeddings()(input_ids)
-        outputs = self.language_model.generate(
-            inputs_embeds=input_embeds,
-            attention_mask=attention_mask,
-            generation_config=generation_config,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            use_cache=True,
-            **generate_kwargs,
-        )
-        return outputs

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/special_tokens_map.json DELETED Viewed

@@ -1,47 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|action_start|>",
-    "<|action_end|>",
-    "<|interpreter|>",
-    "<|plugin|>",
-    "<img>",
-    "</img>",
-    "<IMG_CONTEXT>",
-    "<quad>",
-    "</quad>",
-    "<ref>",
-    "</ref>",
-    "<box>",
-    "</box>"
-  ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenization_internlm2.py DELETED Viewed

@@ -1,235 +0,0 @@
-# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
-#
-# This code is based on transformers/src/transformers/models/llama/tokenization_llama.py
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tokenization classes for InternLM."""
-import os
-from shutil import copyfile
-from typing import Any, Dict, List, Optional, Tuple
-import sentencepiece as spm
-from transformers.tokenization_utils import PreTrainedTokenizer
-from transformers.utils import logging
-logger = logging.get_logger(__name__)
-VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'}
-PRETRAINED_VOCAB_FILES_MAP = {}
-# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer
-class InternLM2Tokenizer(PreTrainedTokenizer):
-    """
-    Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding.
-    Args:
-        vocab_file (`str`):
-            Path to the vocabulary file.
-    """
-    vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
-    model_input_names = ['input_ids', 'attention_mask']
-    _auto_class = 'AutoTokenizer'
-    def __init__(
-        self,
-        vocab_file,
-        unk_token='<unk>',
-        bos_token='<s>',
-        eos_token='</s>',
-        pad_token='</s>',
-        sp_model_kwargs: Optional[Dict[str, Any]] = None,
-        add_bos_token=True,
-        add_eos_token=False,
-        decode_with_prefix_space=False,
-        clean_up_tokenization_spaces=False,
-        **kwargs,
-    ):
-        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
-        self.vocab_file = vocab_file
-        self.add_bos_token = add_bos_token
-        self.add_eos_token = add_eos_token
-        self.decode_with_prefix_space = decode_with_prefix_space
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
-        self._no_prefix_space_tokens = None
-        super().__init__(
-            bos_token=bos_token,
-            eos_token=eos_token,
-            unk_token=unk_token,
-            pad_token=pad_token,
-            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-            **kwargs,
-        )
-    @property
-    def no_prefix_space_tokens(self):
-        if self._no_prefix_space_tokens is None:
-            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
-            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith('▁')}
-        return self._no_prefix_space_tokens
-    @property
-    def vocab_size(self):
-        """Returns vocab size"""
-        return self.sp_model.get_piece_size()
-    @property
-    def bos_token_id(self) -> Optional[int]:
-        return self.sp_model.bos_id()
-    @property
-    def eos_token_id(self) -> Optional[int]:
-        return self.sp_model.eos_id()
-    def get_vocab(self):
-        """Returns vocab as a dict"""
-        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
-        vocab.update(self.added_tokens_encoder)
-        return vocab
-    def _tokenize(self, text):
-        """Returns a tokenized string."""
-        return self.sp_model.encode(text, out_type=str)
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.sp_model.piece_to_id(token)
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        token = self.sp_model.IdToPiece(index)
-        return token
-    def _maybe_add_prefix_space(self, tokens, decoded):
-        if tokens and tokens[0] not in self.no_prefix_space_tokens:
-            return ' ' + decoded
-        else:
-            return decoded
-    def convert_tokens_to_string(self, tokens):
-        """Converts a sequence of tokens (string) in a single string."""
-        current_sub_tokens = []
-        out_string = ''
-        prev_is_special = False
-        for token in tokens:
-            # make sure that special tokens are not decoded using sentencepiece model
-            if token in self.all_special_tokens:
-                if not prev_is_special:
-                    out_string += ' '
-                out_string += self.sp_model.decode(current_sub_tokens) + token
-                prev_is_special = True
-                current_sub_tokens = []
-            else:
-                current_sub_tokens.append(token)
-                prev_is_special = False
-        out_string += self.sp_model.decode(current_sub_tokens)
-        out_string = self.clean_up_tokenization(out_string)
-        out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
-        return out_string[1:]
-    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
-        """
-        Save the vocabulary and special tokens file to a directory.
-        Args:
-            save_directory (`str`):
-                The directory in which to save the vocabulary.
-        Returns:
-            `Tuple(str)`: Paths to the files saved.
-        """
-        if not os.path.isdir(save_directory):
-            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
-            return
-        out_vocab_file = os.path.join(
-            save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']
-        )
-        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
-            copyfile(self.vocab_file, out_vocab_file)
-        elif not os.path.isfile(self.vocab_file):
-            with open(out_vocab_file, 'wb') as fi:
-                content_spiece_model = self.sp_model.serialized_model_proto()
-                fi.write(content_spiece_model)
-        return (out_vocab_file,)
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        if self.add_bos_token:
-            bos_token_ids = [self.bos_token_id]
-        else:
-            bos_token_ids = []
-        output = bos_token_ids + token_ids_0
-        if token_ids_1 is not None:
-            output = output + token_ids_1
-        if self.add_eos_token:
-            output = output + [self.eos_token_id]
-        return output
-    def get_special_tokens_mask(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
-    ) -> List[int]:
-        """
-        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer `prepare_for_model` method.
-        Args:
-            token_ids_0 (`List[int]`):
-                List of IDs.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
-            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-        Returns:
-            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
-        """
-        if already_has_special_tokens:
-            return super().get_special_tokens_mask(
-                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
-            )
-        if token_ids_1 is None:
-            return [1] + ([0] * len(token_ids_0)) + [1]
-        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
-    def create_token_type_ids_from_sequences(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
-    ) -> List[int]:
-        """
-        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
-        use of token type ids, therefore a list of zeros is returned.
-        Args:
-            token_ids_0 (`List[int]`):
-                List of IDs.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
-        Returns:
-            `List[int]`: List of zeros.
-        """
-        eos = [self.eos_token_id]
-        if token_ids_1 is None:
-            return len(token_ids_0 + eos) * [0]
-        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenizer.model DELETED Viewed

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f868398fc4e05ee1e8aeba95ddf18ddcc45b8bce55d5093bead5bbf80429b48b
-size 1477754

InternVL101/work_dirs/internvl_v2_internlm2_2b_lora_finetune_food/lr35_ep10/tokenizer_config.json DELETED Viewed

@@ -1,179 +0,0 @@
-{
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92538": {
-      "content": "<|plugin|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92539": {
-      "content": "<|interpreter|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92540": {
-      "content": "<|action_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92541": {
-      "content": "<|action_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92542": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92543": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92544": {
-      "content": "<img>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92545": {
-      "content": "</img>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92546": {
-      "content": "<IMG_CONTEXT>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92547": {
-      "content": "<quad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92548": {
-      "content": "</quad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92549": {
-      "content": "<ref>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92550": {
-      "content": "</ref>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92551": {
-      "content": "<box>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "92552": {
-      "content": "</box>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|action_start|>",
-    "<|action_end|>",
-    "<|interpreter|>",
-    "<|plugin|>",
-    "<img>",
-    "</img>",
-    "<IMG_CONTEXT>",
-    "<quad>",
-    "</quad>",
-    "<ref>",
-    "</ref>",
-    "<box>",
-    "</box>"
-  ],
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenization_internlm2.InternLM2Tokenizer",
-      null
-    ]
-  },
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "model_max_length": 8192,
-  "pad_token": "</s>",
-  "tokenizer_class": "InternLM2Tokenizer",
-  "unk_token": "<unk>"
-}