Upload folder 20250922_053837 to stage_1/token_merging/

stage_1/token_merging/vis_data/config.py

@@ -0,0 +1,224 @@
+SYSTEM = ''
+accumulative_counts = 400
+batch_size = 1
+betas = (
+    0.9,
+    0.999,
+)
+bnb = dict(
+    bnb_4bit_compute_dtype='torch.bfloat16',
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_use_double_quant=True,
+    llm_int8_has_fp16_weight=False,
+    llm_int8_threshold=6.0,
+    load_in_4bit=True,
+    load_in_8bit=False,
+    type='transformers.BitsAndBytesConfig')
+custom_hooks = [
+    dict(
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.DatasetInfoHook'),
+    dict(
+        evaluation_images=
+        '/mnt/bn/xudong-va/meilong/datasets/Token_Compression/skcm_224x224_b20_t15/h5_files/TCGA-EB-A5UN-06Z-00-DX1.h5',
+        evaluation_inputs=[
+            'Are the tumor cells organized in a lobulated pattern within the slide?',
+        ],
+        every_n_iters=512,
+        prompt_template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+        system='',
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.EvaluateChatHook'),
+    dict(type='xtuner.engine.hooks.ThroughputHook'),
+]
+data_path = '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json'
+dataloader_num_workers = 5
+default_hooks = dict(
+    checkpoint=dict(
+        by_epoch=False,
+        interval=5120,
+        max_keep_ckpts=8,
+        type='mmengine.hooks.CheckpointHook'),
+    logger=dict(
+        interval=10,
+        log_metric_by_epoch=False,
+        type='mmengine.hooks.LoggerHook'),
+    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
+    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
+    timer=dict(type='mmengine.hooks.IterTimerHook'))
+env_cfg = dict(
+    cudnn_benchmark=False,
+    dist_cfg=dict(backend='nccl'),
+    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
+evaluation_freq = 512
+evaluation_images = '/mnt/bn/xudong-va/meilong/datasets/Token_Compression/skcm_224x224_b20_t15/h5_files/TCGA-EB-A5UN-06Z-00-DX1.h5'
+evaluation_inputs = [
+    'Are the tumor cells organized in a lobulated pattern within the slide?',
+]
+image_path_list = None
+launcher = 'pytorch'
+llava_dataset = dict(
+    data_path=
+    '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json',
+    dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+    identifier='_224x224_b20_t15',
+    image_feature_prefix='/mnt/bn/xudong-va/meilong/datasets/Token_Compression',
+    image_feature_suffix='.h5',
+    image_folder='',
+    image_path_list=None,
+    max_length=15836,
+    pad_image_to_square=False,
+    per_image_length=10240,
+    sample_num=10240,
+    sample_strategy='linspace',
+    template_map_fn=dict(
+        template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+        type='xtuner.dataset.map_fns.template_map_fn_factory'),
+    tokenizer=dict(
+        padding_side='right',
+        pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+        trust_remote_code=True,
+        type='transformers.AutoTokenizer.from_pretrained'),
+    type='xtuner.dataset.LLaVADataset',
+    unwanted_prefix_csv=
+    '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/missing_slides3.csv'
+)
+llm_name_or_path = 'Qwen/Qwen2.5-7B-Instruct'
+load_from = None
+log_level = 'INFO'
+log_processor = dict(
+    by_epoch=False,
+    mean_pattern='.*(loss|time|data_time|grad_norm|tflops).*',
+    window_size=1)
+lr = 0.001
+max_epochs = 2
+max_length = 15836
+max_norm = 1
+model = dict(
+    enable_token_merge=True,
+    freeze_llm=True,
+    llm=dict(
+        attn_implementation='flash_attention_2',
+        pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+        quantization_config=dict(
+            bnb_4bit_compute_dtype='torch.bfloat16',
+            bnb_4bit_quant_type='nf4',
+            bnb_4bit_use_double_quant=True,
+            llm_int8_has_fp16_weight=False,
+            llm_int8_threshold=6.0,
+            load_in_4bit=True,
+            load_in_8bit=False,
+            type='transformers.BitsAndBytesConfig'),
+        torch_dtype='torch.bfloat16',
+        trust_remote_code=True,
+        type='transformers.AutoModelForCausalLM.from_pretrained'),
+    max_position_embeddings=None,
+    train_stage='1',
+    type='xtuner.model.llava_no_longnet.LLaVAModel',
+    use_perceiver_resampler=False)
+optim_type = 'torch.optim.AdamW'
+optim_wrapper = dict(
+    optimizer=dict(
+        betas=(
+            0.9,
+            0.999,
+        ),
+        lr=0.001,
+        type='torch.optim.AdamW',
+        weight_decay=0.0),
+    paramwise_cfg=dict(bias_decay_mult=0.0, norm_decay_mult=0.0),
+    type='DeepSpeedOptimWrapper')
+param_scheduler = [
+    dict(
+        begin=0,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=0.1,
+        start_factor=0.01,
+        type='mmengine.optim.LinearLR'),
+    dict(
+        begin=0.1,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=2,
+        eta_min=0.0,
+        type='mmengine.optim.CosineAnnealingLR'),
+]
+per_image_length = 10240
+prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.qwen_chat'
+randomness = dict(deterministic=False, seed=None)
+resume = False
+runner_type = 'FlexibleRunner'
+sample_type = 'wsi'
+save_steps = 5120
+save_total_limit = 8
+seed = 2025
+strategy = dict(
+    config=dict(
+        bf16=dict(enabled=True),
+        fp16=dict(enabled=False, initial_scale_power=16),
+        gradient_accumulation_steps='auto',
+        gradient_clipping='auto',
+        train_micro_batch_size_per_gpu='auto',
+        zero_allow_untested_optimizer=True,
+        zero_force_ds_cpu_optimizer=False,
+        zero_optimization=dict(overlap_comm=False, stage=2)),
+    exclude_frozen_parameters=True,
+    gradient_accumulation_steps=400,
+    gradient_clipping=1,
+    sequence_parallel_size=1,
+    train_micro_batch_size_per_gpu=1,
+    type='xtuner.engine.DeepSpeedStrategy')
+tokenizer = dict(
+    padding_side='right',
+    pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+    trust_remote_code=True,
+    type='transformers.AutoTokenizer.from_pretrained')
+train_cfg = dict(max_epochs=2, type='xtuner.engine.runner.TrainLoop')
+train_dataloader = dict(
+    batch_size=1,
+    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
+    dataset=dict(
+        data_path=
+        '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json',
+        dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+        identifier='_224x224_b20_t15',
+        image_feature_prefix=
+        '/mnt/bn/xudong-va/meilong/datasets/Token_Compression',
+        image_feature_suffix='.h5',
+        image_folder='',
+        image_path_list=None,
+        max_length=15836,
+        pad_image_to_square=False,
+        per_image_length=10240,
+        sample_num=10240,
+        sample_strategy='linspace',
+        template_map_fn=dict(
+            template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+            type='xtuner.dataset.map_fns.template_map_fn_factory'),
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.dataset.LLaVADataset',
+        unwanted_prefix_csv=
+        '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/missing_slides3.csv'
+    ),
+    num_workers=5,
+    persistent_workers=True,
+    pin_memory=True,
+    prefetch_factor=4,
+    sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler'))
+visualizer = None
+warmup_ratio = 0.05
+weight_decay = 0.0
+work_dir = '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/checkpoints/stage_1/token_merging/'

stage_1/token_merging/vis_data/eval_outputs_iter_10239.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern within the slide. The lobules are well-defined and separated by fibrous stroma.<|im_end|>
+
+

stage_1/token_merging/vis_data/eval_outputs_iter_15359.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern. The lobules are well-defined and separated by fibrous stroma, creating a distinct and organized structure.<|im_end|>
+
+

stage_1/token_merging/vis_data/eval_outputs_iter_20479.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern. The lobules are well-defined and separated by fibrous stroma, creating a distinct and organized structure.<|im_end|>
+
+

stage_1/token_merging/vis_data/eval_outputs_iter_23095.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern. The lobules are well-defined and separated by fibrous stroma, creating a distinct and organized structure.<|im_end|>
+
+

stage_1/token_merging/vis_data/eval_outputs_iter_5119.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are indeed organized in a lobulated pattern. This arrangement is characterized by the presence of multiple lobes or areas of tumor tissue, each with its own distinct boundaries. The lobules are typically well-defined and separated by fibrous stroma, which contributes to the overall architecture of the lesion.<|im_end|>
+
+