config.json

{
  "model_type": "ar-cxr",
  "architectures": ["ArCXR"],
  "auto_map": { "AutoModel": "modeling_ar_cxr.ArCXR" },
  "_comment": "Composite vision-language model with TWO independent visual connectors that share the base models. Generation uses a BLIP-2 Q-Former; grounding uses an MLP connector + vision-LoRA. Base weights are NOT shipped; the loader downloads them from the repos below and applies the adapters in weights/.",

  "vision": {
    "repo": "microsoft/rad-dino-maira-2",
    "hidden_size": 768,
    "image_size": 518,
    "patch_grid": [37, 37],
    "_note": "vision-LoRA below is used ONLY by the grounding head; report generation uses the frozen base encoder.",
    "lora": { "r": 16, "alpha": 32, "dropout": 0.0,
              "target_modules": ["query", "key", "value", "dense", "fc1", "fc2"] }
  },

  "decoder": {
    "repo": "tiiuae/Falcon-H1-7B-Instruct",
    "hidden_size": 3072,
    "trust_remote_code": true,
    "_note": "decoder-LoRA is used ONLY by report generation.",
    "lora": { "r": 64, "alpha": 128, "dropout": 0.05,
              "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj",
                                 "gate_proj", "up_proj", "down_proj"] }
  },

  "generation_connector": {
    "type": "qformer",
    "qformer": { "impl": "Blip2QFormerModel", "num_hidden_layers": 12,
                 "num_attention_heads": 12, "hidden_size": 768,
                 "intermediate_size": 3072, "cross_attention_frequency": 2 },
    "num_queries": 64,
    "proj": "Linear(768->3072, bias=False)",
    "prefix_layernorm": true,
    "instruction_ar": "اكتب تقرير أشعة صدر باللغة العربية بناءً على الصورة:\n",
    "decoding": { "max_new_tokens": 200, "do_sample": false, "num_beams": 1,
                  "repetition_penalty": 1.3, "no_repeat_ngram_size": 3 }
  },

  "grounding_connector": {
    "type": "mlp",
    "pool": "adaptive_avg_pool2d",
    "vis_pool_tokens": 256,
    "keep_cls_token": true,
    "prefix_tokens": 257,
    "mlp": [768, 3072, 3072],
    "activation": "gelu",
    "prefix_layernorm": true
  },

  "aux_head": {
    "in_features": 3072,
    "num_labels": 11,
    "pooling": "mean_over_prefix",
    "labels": ["atelectasis", "cardiomegaly", "consolidation", "edema",
               "effusion", "fracture", "no_finding", "opacity",
               "pneumonia", "pneumothorax", "support_dev"]
  },

  "torch_dtype": "bfloat16"
}