ablation_checkpoint.json

{
  "A8": [
    {
      "strategy": "Early fusion",
      "position": "After patch projection (layer 0/12)",
      "advantage": "Text guides visual processing from start",
      "limitation": "Corrupts pretrained ViT token space; requires ViT surgery"
    },
    {
      "strategy": "Mid fusion",
      "position": "After ViT block 6/12",
      "advantage": "Partial cross-modal adaptation",
      "limitation": "ViT features not fully formed; requires ViT surgery"
    },
    {
      "strategy": "Late fusion [CapsViT]",
      "position": "After all ViT blocks (CLS token)",
      "advantage": "Stable; used by CLIP, BLIP-2, LLaVA",
      "limitation": "No cross-modal interaction during ViT processing"
    },
    {
      "strategy": "Multi-scale",
      "position": "Layers {3,6,9,12} + aggregation",
      "advantage": "Hierarchical cross-modal grounding",
      "limitation": "High memory; requires ViT surgery; future work"
    }
  ]
}