Upload folder 20250924_184838 to stage_1/token_merge_plus_resampler/20250924_184838

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/config.py

@@ -0,0 +1,229 @@
+SYSTEM = ''
+accumulative_counts = 256
+batch_size = 1
+betas = (
+    0.9,
+    0.999,
+)
+bnb = dict(
+    bnb_4bit_compute_dtype='torch.bfloat16',
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_use_double_quant=True,
+    llm_int8_has_fp16_weight=False,
+    llm_int8_threshold=6.0,
+    load_in_4bit=True,
+    load_in_8bit=False,
+    type='transformers.BitsAndBytesConfig')
+custom_hooks = [
+    dict(
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.DatasetInfoHook'),
+    dict(
+        evaluation_images=
+        '/mnt/bn/xudong-va/meilong/datasets/Token_Compression/skcm_224x224_b20_t15/h5_files/TCGA-EB-A5UN-06Z-00-DX1.h5',
+        evaluation_inputs=[
+            'Are the tumor cells organized in a lobulated pattern within the slide?',
+        ],
+        every_n_iters=512,
+        prompt_template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+        system='',
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.EvaluateChatHookResampler'),
+    dict(type='xtuner.engine.hooks.ThroughputHook'),
+]
+data_path = '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json'
+dataloader_num_workers = 10
+default_hooks = dict(
+    checkpoint=dict(
+        by_epoch=False,
+        interval=5120,
+        max_keep_ckpts=8,
+        type='mmengine.hooks.CheckpointHook'),
+    logger=dict(
+        interval=10,
+        log_metric_by_epoch=False,
+        type='mmengine.hooks.LoggerHook'),
+    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
+    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
+    timer=dict(type='mmengine.hooks.IterTimerHook'))
+env_cfg = dict(
+    cudnn_benchmark=False,
+    dist_cfg=dict(backend='nccl'),
+    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
+evaluation_freq = 512
+evaluation_images = '/mnt/bn/xudong-va/meilong/datasets/Token_Compression/skcm_224x224_b20_t15/h5_files/TCGA-EB-A5UN-06Z-00-DX1.h5'
+evaluation_inputs = [
+    'Are the tumor cells organized in a lobulated pattern within the slide?',
+]
+image_path_list = None
+launcher = 'pytorch'
+llava_dataset = dict(
+    data_path=
+    '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json',
+    dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+    identifier='_224x224_b20_t15',
+    image_feature_prefix='/mnt/bn/xudong-va/meilong/datasets/Token_Compression',
+    image_feature_suffix='.h5',
+    image_folder='',
+    image_path_list=None,
+    max_length=15836,
+    pad_image_to_square=False,
+    per_image_length=10240,
+    sample_num=10240,
+    sample_strategy='linspace',
+    template_map_fn=dict(
+        template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+        type='xtuner.dataset.map_fns.template_map_fn_factory'),
+    tokenizer=dict(
+        padding_side='right',
+        pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+        trust_remote_code=True,
+        type='transformers.AutoTokenizer.from_pretrained'),
+    type='xtuner.dataset.LLaVADataset',
+    unwanted_prefix_csv=
+    '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/missing_slides3.csv'
+)
+llm_name_or_path = 'Qwen/Qwen2.5-7B-Instruct'
+load_from = None
+log_level = 'INFO'
+log_processor = dict(
+    by_epoch=False,
+    mean_pattern='.*(loss|time|data_time|grad_norm|tflops).*',
+    window_size=1)
+lr = 0.001
+max_epochs = 2
+max_length = 15836
+max_norm = 1
+model = dict(
+    enable_token_merge=True,
+    freeze_llm=True,
+    llm=dict(
+        attn_implementation='flash_attention_2',
+        pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+        quantization_config=dict(
+            bnb_4bit_compute_dtype='torch.bfloat16',
+            bnb_4bit_quant_type='nf4',
+            bnb_4bit_use_double_quant=True,
+            llm_int8_has_fp16_weight=False,
+            llm_int8_threshold=6.0,
+            load_in_4bit=True,
+            load_in_8bit=False,
+            type='transformers.BitsAndBytesConfig'),
+        torch_dtype='torch.bfloat16',
+        trust_remote_code=True,
+        type='transformers.AutoModelForCausalLM.from_pretrained'),
+    max_position_embeddings=None,
+    resampler_num_latents=100,
+    train_stage='1',
+    type='xtuner.model.llava_no_longnet_simple_sampler.LLaVAModel',
+    use_resampler=True)
+optim_type = 'torch.optim.AdamW'
+optim_wrapper = dict(
+    optimizer=dict(
+        betas=(
+            0.9,
+            0.999,
+        ),
+        lr=0.001,
+        type='torch.optim.AdamW',
+        weight_decay=0.0),
+    paramwise_cfg=dict(
+        bias_decay_mult=0.0,
+        norm_decay_mult=0.0,
+        paramwise_cfg=dict(
+            custom_keys=dict({'^projector\.': dict(lr_mult=1.0)}))),
+    type='DeepSpeedOptimWrapper')
+param_scheduler = [
+    dict(
+        begin=0,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=0.16,
+        start_factor=0.01,
+        type='mmengine.optim.LinearLR'),
+    dict(
+        begin=0.16,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=2,
+        eta_min=0.0,
+        type='mmengine.optim.CosineAnnealingLR'),
+]
+per_image_length = 10240
+prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.qwen_chat'
+randomness = dict(deterministic=False, seed=42)
+resume = False
+runner_type = 'FlexibleRunner'
+sample_type = 'wsi'
+save_steps = 5120
+save_total_limit = 8
+seed = 42
+strategy = dict(
+    config=dict(
+        bf16=dict(enabled=True),
+        fp16=dict(enabled=False, initial_scale_power=16),
+        gradient_accumulation_steps='auto',
+        gradient_clipping='auto',
+        train_micro_batch_size_per_gpu='auto',
+        zero_allow_untested_optimizer=True,
+        zero_force_ds_cpu_optimizer=False,
+        zero_optimization=dict(overlap_comm=False, stage=2)),
+    exclude_frozen_parameters=True,
+    gradient_accumulation_steps=256,
+    gradient_clipping=1,
+    sequence_parallel_size=1,
+    train_micro_batch_size_per_gpu=1,
+    type='xtuner.engine.DeepSpeedStrategy')
+tokenizer = dict(
+    padding_side='right',
+    pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+    trust_remote_code=True,
+    type='transformers.AutoTokenizer.from_pretrained')
+train_cfg = dict(max_epochs=2, type='xtuner.engine.runner.TrainLoop')
+train_dataloader = dict(
+    batch_size=1,
+    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
+    dataset=dict(
+        data_path=
+        '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/stage1_morph2.json',
+        dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+        identifier='_224x224_b20_t15',
+        image_feature_prefix=
+        '/mnt/bn/xudong-va/meilong/datasets/Token_Compression',
+        image_feature_suffix='.h5',
+        image_folder='',
+        image_path_list=None,
+        max_length=15836,
+        pad_image_to_square=False,
+        per_image_length=10240,
+        sample_num=10240,
+        sample_strategy='linspace',
+        template_map_fn=dict(
+            template='xtuner.utils.PROMPT_TEMPLATE.qwen_chat',
+            type='xtuner.dataset.map_fns.template_map_fn_factory'),
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='Qwen/Qwen2.5-7B-Instruct',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.dataset.LLaVADataset',
+        unwanted_prefix_csv=
+        '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/merged_dataset/missing_slides3.csv'
+    ),
+    num_workers=10,
+    persistent_workers=True,
+    pin_memory=True,
+    prefetch_factor=4,
+    sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler'))
+visualizer = None
+warmup_ratio = 0.08
+weight_decay = 0.0
+work_dir = '/mnt/bn/yuxuanwang/meilong/code/projects/efficient_foundation_wsi_llava/checkpoints/stage_1/token_merge_plus_resampler/'

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/eval_outputs_iter_10239.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern within the slide.<|im_end|>
+
+

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/eval_outputs_iter_15359.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern within the slide.<|im_end|>
+
+

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/eval_outputs_iter_20479.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern within the slide.<|im_end|>
+
+

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/eval_outputs_iter_23095.txt

@@ -0,0 +1,8 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Yes, the tumor cells are organized in a lobulated pattern within the slide.<|im_end|>
+
+

stage_1/token_merge_plus_resampler/20250924_184838/vis_data/eval_outputs_iter_5119.txt

@@ -0,0 +1,34 @@
+Eval output 1:
+<|im_start|>user
+<image>
+Are the tumor cells organized in a lobulated pattern within the slide?<|im_end|>
+<|im_start|>assistant
+Based on the information provided, the correct answer is:
+
+A: The tumor cells are organized in a lobulated pattern within the slide.
+
+This answer is correct because:
+
+1. The lobulated pattern is a characteristic feature of invasive ductal carcinoma (IDC) of the breast. In this pattern, the tumor cells form irregular, rounded masses or nodules that are separated by normal breast tissue.
+
+2. Lobulation refers to the formation of rounded, irregularly shaped masses that are separated by intervening normal tissue. This is a hallmark of invasive carcinoma, distinguishing it from in situ carcinoma (DCIS) which typically shows a cribriform or microlobulated pattern.
+
+3. The lobulated pattern is often seen in invasive ductal carcinoma because the cancer cells invade the breast tissue, forming these irregular masses that are surrounded by normal breast tissue.
+
+The other options can be eliminated:
+- A lobulated pattern is not characteristic of DCIS, which typically shows a cribriform or microlobulated pattern.
+- A solid pattern would be more indicative of a poorly differentiated carcinoma or a sarcoma.
+- A cribriform pattern is seen in DCIS, not invasive carcinoma.
+- A microlobulated pattern is also seen in DCIS, not invasive carcinoma.
+
+Therefore, the correct answer is A. The tumor cells are organized in a lobulated pattern within the slide. 
+
+The correct answer is A. The tumor cells are organized in a lobulated pattern within the slide. 
+
+This answer is correct because the lobulated pattern is a characteristic feature of invasive ductal carcinoma (IDC) of the breast, where the tumor cells form irregular, rounded masses or nodules that are separated by normal breast tissue. This pattern distinguishes invasive carcinoma from in situ carcinoma (DCIS), which typically shows a cribriform or microlobulated pattern. The other options describe patterns more commonly associated with other types of breast lesions or carcinomas. 
+
+The final answer is A: The tumor cells are organized in a lobulated pattern within the slide. 
+
+\boxed{A} \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated pattern within the slide.\] \[A\] \[The tumor cells are organized in a lobulated
+
+