| { |
| "architectures": [ |
| "KeuralVisionModel" |
| ], |
| "default_token_budget": 512, |
| "depth": 24, |
| "dtype": "float32", |
| "embed_dim": 768, |
| "keural_config": { |
| "data": { |
| "captions_path": "/home/work/keural-model/vision-encoder-dataset/indexes/datacomp_train.tsv", |
| "dataset_name": "cc3m", |
| "extra_sources": [], |
| "image_buckets": [ |
| [ |
| 384, |
| 384 |
| ], |
| [ |
| 448, |
| 448 |
| ] |
| ], |
| "images_root": "/home/work/keural-model/vision-encoder-dataset/datacomp_medium", |
| "max_caption_length": 77, |
| "num_workers": 6, |
| "pixel_mean": [ |
| 0.485, |
| 0.456, |
| 0.406 |
| ], |
| "pixel_std": [ |
| 0.229, |
| 0.224, |
| 0.225 |
| ], |
| "text_encoder_name": "openai/clip-vit-large-patch14" |
| }, |
| "distillation": { |
| "enabled": true, |
| "internvit_embed_dim": 1024, |
| "internvit_model": "OpenGVLab/InternViT-300M-448px", |
| "lambda_internvit": 0.5, |
| "lambda_siglip": 0.5, |
| "siglip_embed_dim": 1152, |
| "siglip_model": "google/siglip-so400m-patch14-384" |
| }, |
| "loss": { |
| "lambda_distill": 0.5, |
| "lambda_hct": 1.0, |
| "lambda_sal": 0.5, |
| "learnable_bias_init": -10.0, |
| "learnable_temperature_init": 2.3, |
| "primary": "sigmoid_siglip", |
| "saliency_beta": 0.005, |
| "saliency_lambda_smooth": 4.0 |
| }, |
| "name": "keural-mid", |
| "saliency": { |
| "hidden_channels": 192 |
| }, |
| "spatial": { |
| "depth": 24, |
| "embed_dim": 768, |
| "head_dim": 64, |
| "mlp_ratio": 4, |
| "norm_eps": 1e-06, |
| "num_heads": 12, |
| "rope_base": 10000.0 |
| }, |
| "stem": { |
| "activation": "gelu", |
| "channels": [ |
| 128, |
| 256, |
| 512, |
| 768 |
| ], |
| "kernel_size": 3 |
| }, |
| "tokenizer": { |
| "budget_floors": { |
| "detail": 0, |
| "global": 8, |
| "region": 16 |
| }, |
| "budget_split": { |
| "detail": 0.7, |
| "global": 0.05, |
| "region": 0.25 |
| }, |
| "default_token_budget": 512, |
| "gumbel_temperature_end": 0.05, |
| "gumbel_temperature_start": 1.0, |
| "max_token_budget": 2048, |
| "nms_radius": { |
| "detail": 1, |
| "region": 3 |
| }, |
| "saliency_smooth_kernel": { |
| "detail": 3, |
| "region": 7 |
| }, |
| "temperature_anneal_steps": 10000, |
| "warmup_steps_fixed_grid": 4000 |
| }, |
| "training": { |
| "batch_size_per_device": 768, |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "gradient_accumulation": 2, |
| "gradient_checkpointing": true, |
| "log_every": 100, |
| "lr": 0.0005, |
| "lr_schedule": "cosine_with_warmup", |
| "max_steps": 18000, |
| "optimizer": "adamw", |
| "precision": "bfloat16", |
| "save_every": 1000, |
| "seed": 42, |
| "warmup_steps": 2000, |
| "weight_decay": 0.05 |
| } |
| }, |
| "max_token_budget": 2048, |
| "model_type": "keural_vision", |
| "num_heads": 12, |
| "transformers_version": "4.57.6" |
| } |
|
|
