| { |
| "model_name": "llava-hf/llava-1.5-7b-hf", |
| "dtype": "bfloat16", |
| "relation": "bathroom_toilet", |
| "object_keywords": [], |
| "layers": { |
| "mode": "text_only", |
| "specific_layer_patterns": [], |
| "top_k_layers": 0 |
| }, |
| "prompts": { |
| "prompts": [ |
| "Describe this image." |
| ], |
| "sampling": "uniform" |
| }, |
| "loss": { |
| "alpha": 0.0001, |
| "beta": 0.0, |
| "kl_clip": 10.0, |
| "kl_mode": "per_token", |
| "suppress_weight": 10.0, |
| "suppress_margin": 10.0, |
| "n_sigma": 0.5, |
| "epsilon": 0.05, |
| "lambda_mix": 0.1, |
| "smooth_tau": 0.01, |
| "adaptive_lambda": true, |
| "nuclear_beta": 0.0, |
| "nuclear_every_k": 10, |
| "depth_weighted_nuclear": false |
| }, |
| "lora": { |
| "lora_r": 16, |
| "lora_alpha": 32, |
| "lora_dropout": 0.05, |
| "lora_target_modules": "all-linear" |
| }, |
| "adv": { |
| "sae_checkpoint": "/mnt/raid10/erwin/testing/multilayer-sae/last.ckpt", |
| "probe_checkpoint": "", |
| "probe_layers": [], |
| "lambda_adv": 20.0, |
| "lambda_ret": 1.0, |
| "probe_lr": 1e-4, |
| "probe_steps": 5, |
| "probe_label_mode": "object_only", |
| "exclude_suppress_from_task_kl": true, |
| "probe_warmup_epochs": 3, |
| "freeze_probe_after_warmup": false, |
| "probe_label_smoothing": 0.05, |
| "probe_weight_decay": 0.1, |
| "probe_spectral_norm": false, |
| "probe_warmup_exclude_scene_only": true, |
| "lambda_present": 3.0, |
| "lambda_act_ret": 10.0, |
| "lambda_group_lasso": 0.0 |
| }, |
| "lr": 1e-5, |
| "num_epochs": 20, |
| "batch_size": 8, |
| "grad_accumulation_steps": 8, |
| "num_workers": 4, |
| "dataset_id": "", |
| "csv_path": null, |
| "image_dir": null, |
| "max_train_samples": null, |
| "general_image_dir": "/home/users/ntu/cong045/scratch/testing/hallucination/CC3M-Dataset/cc3m_images/train", |
| "num_general_samples": 0, |
| "general_image_seed": 42, |
| "general_image_recursive": false, |
| "upsample_non_scene_object": 1, |
| "output_dir": "./adv_outputs", |
| "save_every_n_steps": 100 |
| } |
|
|
