Upload 16 files
Browse files- .gitattributes +2 -0
- LIS1-9B-LoraAdapter/adapter_config.json +57 -0
- LIS1-9B-LoraAdapter/adapter_model.safetensors +3 -0
- LIS1-9B-forLora.pt +3 -0
- LIS1-9B-forLora.safetensors +3 -0
- LIS1-9B-noLora.pt +3 -0
- LIS1-9B-noLora.safetensors +3 -0
- LIS1ModelCode.py +1152 -0
- README.md +233 -0
- benchmark_reasonseg.py +791 -0
- dataset.py +329 -0
- example.py +67 -0
- image1.jpg +3 -0
- lis1_infer.py +1461 -0
- logo.png +0 -0
- result_0_boxes_and_masks.png +3 -0
- train.py +224 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
image1.jpg filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
result_0_boxes_and_masks.png filter=lfs diff=lfs merge=lfs -text
|
LIS1-9B-LoraAdapter/adapter_config.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alora_invocation_tokens": null,
|
| 3 |
+
"alpha_pattern": {},
|
| 4 |
+
"arrow_config": null,
|
| 5 |
+
"auto_mapping": {
|
| 6 |
+
"base_model_class": "Qwen3_5TextModel",
|
| 7 |
+
"parent_library": "transformers.models.qwen3_5.modeling_qwen3_5"
|
| 8 |
+
},
|
| 9 |
+
"base_model_name_or_path": null,
|
| 10 |
+
"bias": "none",
|
| 11 |
+
"corda_config": null,
|
| 12 |
+
"ensure_weight_tying": false,
|
| 13 |
+
"eva_config": null,
|
| 14 |
+
"exclude_modules": null,
|
| 15 |
+
"fan_in_fan_out": false,
|
| 16 |
+
"inference_mode": true,
|
| 17 |
+
"init_lora_weights": true,
|
| 18 |
+
"kasa_config": null,
|
| 19 |
+
"layer_replication": null,
|
| 20 |
+
"layers_pattern": null,
|
| 21 |
+
"layers_to_transform": null,
|
| 22 |
+
"loftq_config": {},
|
| 23 |
+
"lora_alpha": 16,
|
| 24 |
+
"lora_bias": false,
|
| 25 |
+
"lora_dropout": 0.05,
|
| 26 |
+
"lora_ga_config": null,
|
| 27 |
+
"megatron_config": null,
|
| 28 |
+
"megatron_core": "megatron.core",
|
| 29 |
+
"modules_to_save": null,
|
| 30 |
+
"monteclora_config": null,
|
| 31 |
+
"peft_type": "LORA",
|
| 32 |
+
"peft_version": "0.21.2",
|
| 33 |
+
"qalora_group_size": 16,
|
| 34 |
+
"r": 8,
|
| 35 |
+
"rank_pattern": {},
|
| 36 |
+
"revision": null,
|
| 37 |
+
"target_modules": [
|
| 38 |
+
"layers.31.self_attn.v_proj",
|
| 39 |
+
"layers.31.self_attn.o_proj",
|
| 40 |
+
"layers.28.linear_attn.out_proj",
|
| 41 |
+
"layers.31.self_attn.k_proj",
|
| 42 |
+
"layers.28.linear_attn.in_proj_qkv",
|
| 43 |
+
"layers.31.self_attn.q_proj",
|
| 44 |
+
"layers.29.linear_attn.out_proj",
|
| 45 |
+
"layers.29.linear_attn.in_proj_qkv",
|
| 46 |
+
"layers.30.linear_attn.in_proj_qkv",
|
| 47 |
+
"layers.30.linear_attn.out_proj"
|
| 48 |
+
],
|
| 49 |
+
"target_parameters": null,
|
| 50 |
+
"task_type": null,
|
| 51 |
+
"trainable_token_indices": null,
|
| 52 |
+
"use_bdlora": null,
|
| 53 |
+
"use_dora": false,
|
| 54 |
+
"use_qalora": false,
|
| 55 |
+
"use_rslora": false,
|
| 56 |
+
"velora_config": null
|
| 57 |
+
}
|
LIS1-9B-LoraAdapter/adapter_model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e113ecf5c50b23446537f6ae9b06ba16a3900ba21554c0863b3245056690074e
|
| 3 |
+
size 2951736
|
LIS1-9B-forLora.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:57b16aa7c92b7f40f7293ed5c93928f7c4d0a6bf4f24c112af1065f771a8b6fc
|
| 3 |
+
size 27216895
|
LIS1-9B-forLora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:33ac1e881ceebfacba209c3e96cec1edc17f581f380b167a137d19581b8aaf7a
|
| 3 |
+
size 27200852
|
LIS1-9B-noLora.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ad62dddaeaf26a8c588fcf320d5aca23772b631353490bc0ebee85319352d158
|
| 3 |
+
size 27219743
|
LIS1-9B-noLora.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:365d2e214c8023d0066518d705aa68c9d4f6c3d9c4a82b0cd9f9c096bbb79969
|
| 3 |
+
size 27200812
|
LIS1ModelCode.py
ADDED
|
@@ -0,0 +1,1152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib.metadata as _im
|
| 2 |
+
import inspect
|
| 3 |
+
import json
|
| 4 |
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import cv2
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
|
| 12 |
+
# Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality
|
| 13 |
+
_orig_version = _im.version
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _safe_version(pkg: str) -> str:
|
| 17 |
+
if pkg == "huggingface-hub":
|
| 18 |
+
try:
|
| 19 |
+
ver = _orig_version(pkg)
|
| 20 |
+
if ver.startswith("2."):
|
| 21 |
+
return "1.99.0"
|
| 22 |
+
return ver
|
| 23 |
+
except Exception:
|
| 24 |
+
return "1.99.0"
|
| 25 |
+
return _orig_version(pkg)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
_im.version = _safe_version
|
| 29 |
+
|
| 30 |
+
from transformers import AutoModel, AutoProcessor
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
OBJECT_START_TOKEN = "<|object_ref_start|>"
|
| 34 |
+
OBJECT_QUERY_TOKEN = "<|object_ref_end|>"
|
| 35 |
+
DEFAULT_SYSTEM_PROMPT = (
|
| 36 |
+
"You are a fast single-pass visual reasoning and segmentation model. "
|
| 37 |
+
"For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely."
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def normalize_input_json(
|
| 42 |
+
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
|
| 43 |
+
) -> List[Dict[str, str]]:
|
| 44 |
+
"""
|
| 45 |
+
Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts)
|
| 46 |
+
into a canonical list of [{'id': str, 'description': str}, ...].
|
| 47 |
+
"""
|
| 48 |
+
if isinstance(query_input, str):
|
| 49 |
+
parsed = json.loads(query_input)
|
| 50 |
+
else:
|
| 51 |
+
parsed = query_input
|
| 52 |
+
|
| 53 |
+
if isinstance(parsed, dict):
|
| 54 |
+
raw_objects = parsed.get("objects", parsed.get("queries", [parsed]))
|
| 55 |
+
elif isinstance(parsed, (list, tuple)):
|
| 56 |
+
raw_objects = list(parsed)
|
| 57 |
+
else:
|
| 58 |
+
raise ValueError(f"Unsupported query_input format: {type(query_input)}")
|
| 59 |
+
|
| 60 |
+
normalized: List[Dict[str, str]] = []
|
| 61 |
+
for idx, item in enumerate(raw_objects):
|
| 62 |
+
if isinstance(item, str):
|
| 63 |
+
normalized.append({"id": str(idx), "description": item.strip()})
|
| 64 |
+
elif isinstance(item, dict):
|
| 65 |
+
obj_id = str(item.get("id", idx))
|
| 66 |
+
desc = str(
|
| 67 |
+
item.get("description")
|
| 68 |
+
or item.get("prompt")
|
| 69 |
+
or item.get("text")
|
| 70 |
+
or item.get("sent")
|
| 71 |
+
or ""
|
| 72 |
+
).strip()
|
| 73 |
+
if not desc:
|
| 74 |
+
raise ValueError(f"Object at index {idx} has an empty description: {item}")
|
| 75 |
+
normalized.append({"id": obj_id, "description": desc})
|
| 76 |
+
else:
|
| 77 |
+
raise ValueError(f"Unsupported object entry at index {idx}: {item}")
|
| 78 |
+
|
| 79 |
+
if not normalized:
|
| 80 |
+
raise ValueError("Input JSON must contain at least one object description.")
|
| 81 |
+
return normalized
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def build_segmentation_prompt(
|
| 85 |
+
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
|
| 86 |
+
query_token: str = OBJECT_QUERY_TOKEN,
|
| 87 |
+
include_query_token: bool = False,
|
| 88 |
+
) -> Tuple[str, List[Dict[str, str]]]:
|
| 89 |
+
"""
|
| 90 |
+
Builds the user-side JSON prompt describing all objects/riddles to segment.
|
| 91 |
+
"""
|
| 92 |
+
objects = normalize_input_json(query_input)
|
| 93 |
+
entries = []
|
| 94 |
+
for obj in objects:
|
| 95 |
+
entry_json = json.dumps(
|
| 96 |
+
{"id": obj["id"], "description": obj["description"]},
|
| 97 |
+
ensure_ascii=False,
|
| 98 |
+
)
|
| 99 |
+
suffix = query_token if include_query_token else ""
|
| 100 |
+
entries.append(f" {entry_json}{suffix}")
|
| 101 |
+
json_block = "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
|
| 102 |
+
prompt = (
|
| 103 |
+
"Reason about each description or riddle in the following JSON "
|
| 104 |
+
"and segment the corresponding object in the image:\n"
|
| 105 |
+
f"{json_block}"
|
| 106 |
+
)
|
| 107 |
+
return prompt, objects
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def build_assistant_grounding_prefill(
|
| 111 |
+
objects: Sequence[Dict[str, str]],
|
| 112 |
+
start_token: str = OBJECT_START_TOKEN,
|
| 113 |
+
query_token: str = OBJECT_QUERY_TOKEN,
|
| 114 |
+
) -> str:
|
| 115 |
+
"""
|
| 116 |
+
Builds the prefilled assistant response containing `<|object_ref_start|>{desc}<|object_ref_end|>`
|
| 117 |
+
for each object. Because this is passed as part of the single forward pass:
|
| 118 |
+
1. The LLM has already read the user's riddle AND processed all `<|image_pad|>` tokens.
|
| 119 |
+
2. Crossing `<|im_start|>assistant` switches Qwen into answering/grounding mode.
|
| 120 |
+
3. `<|object_ref_start|>...<|object_ref_end|>` activates Qwen's native localization circuit.
|
| 121 |
+
"""
|
| 122 |
+
entries = []
|
| 123 |
+
for obj in objects:
|
| 124 |
+
target_str = f"{start_token}{obj['description']}{query_token}"
|
| 125 |
+
entry_json = json.dumps(
|
| 126 |
+
{"id": obj["id"], "target": target_str},
|
| 127 |
+
ensure_ascii=False,
|
| 128 |
+
)
|
| 129 |
+
entries.append(f" {entry_json}")
|
| 130 |
+
return "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def build_segmentation_messages(
|
| 134 |
+
image: Any,
|
| 135 |
+
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
|
| 136 |
+
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
|
| 137 |
+
query_token: str = OBJECT_QUERY_TOKEN,
|
| 138 |
+
) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
|
| 139 |
+
"""
|
| 140 |
+
Constructs the single-pass multimodal chat messages in causal reasoning order:
|
| 141 |
+
[System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>]
|
| 142 |
+
Putting the user JSON before the image allows every `<|image_pad|>` token in the LLM
|
| 143 |
+
to attend to the riddle across all 36 layers before `<|object_ref_end|>` reads them out.
|
| 144 |
+
"""
|
| 145 |
+
user_prompt, objects = build_segmentation_prompt(
|
| 146 |
+
query_input, query_token=query_token, include_query_token=False
|
| 147 |
+
)
|
| 148 |
+
assistant_prefill = build_assistant_grounding_prefill(objects, query_token=query_token)
|
| 149 |
+
messages = [
|
| 150 |
+
{"role": "system", "content": system_prompt},
|
| 151 |
+
{
|
| 152 |
+
"role": "user",
|
| 153 |
+
"content": [
|
| 154 |
+
{"type": "text", "text": user_prompt + "\n"},
|
| 155 |
+
{"type": "image", "image": image},
|
| 156 |
+
],
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"role": "assistant",
|
| 160 |
+
"content": assistant_prefill,
|
| 161 |
+
},
|
| 162 |
+
]
|
| 163 |
+
return messages, objects
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def polygons_to_mask(
|
| 167 |
+
segmentation: Union[List[List[float]], Dict[str, Any]],
|
| 168 |
+
height: int,
|
| 169 |
+
width: int,
|
| 170 |
+
) -> np.ndarray:
|
| 171 |
+
"""
|
| 172 |
+
Rasterizes a COCO/RefCOCO annotation `segmentation` (polygon list or uncompressed RLE)
|
| 173 |
+
into a binary numpy mask of shape (height, width) with values in {0, 1}.
|
| 174 |
+
"""
|
| 175 |
+
mask = np.zeros((height, width), dtype=np.uint8)
|
| 176 |
+
if isinstance(segmentation, list):
|
| 177 |
+
for poly in segmentation:
|
| 178 |
+
if len(poly) < 6:
|
| 179 |
+
continue
|
| 180 |
+
pts = np.asarray(poly, dtype=np.float32).reshape(-1, 2)
|
| 181 |
+
pts = np.round(pts).astype(np.int32)
|
| 182 |
+
cv2.fillPoly(mask, [pts], 1)
|
| 183 |
+
elif isinstance(segmentation, dict) and "counts" in segmentation:
|
| 184 |
+
counts = segmentation["counts"]
|
| 185 |
+
if isinstance(counts, list):
|
| 186 |
+
h, w = segmentation.get("size", [height, width])
|
| 187 |
+
flat = np.zeros(h * w, dtype=np.uint8)
|
| 188 |
+
idx = 0
|
| 189 |
+
val = 0
|
| 190 |
+
for c in counts:
|
| 191 |
+
flat[idx : idx + c] = val
|
| 192 |
+
idx += c
|
| 193 |
+
val = 1 - val
|
| 194 |
+
mask = flat.reshape((w, h)).T
|
| 195 |
+
if mask.shape != (height, width):
|
| 196 |
+
mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_NEAREST)
|
| 197 |
+
else:
|
| 198 |
+
raise ValueError("Compressed string RLE requires pycocotools; RefCOCO uses polygon lists.")
|
| 199 |
+
else:
|
| 200 |
+
raise ValueError(f"Unsupported segmentation format: {type(segmentation)}")
|
| 201 |
+
return mask
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def mask_to_polygons(
|
| 205 |
+
binary_mask: Union[np.ndarray, torch.Tensor],
|
| 206 |
+
epsilon_ratio: float = 0.0015,
|
| 207 |
+
min_area: float = 4.0,
|
| 208 |
+
) -> Tuple[List[List[float]], List[float], float]:
|
| 209 |
+
"""
|
| 210 |
+
Converts a 2D binary mask (H, W) into RefCOCO/COCO dataset format:
|
| 211 |
+
- segmentation: List[List[float]] of polygon coordinates [x1, y1, x2, y2, ...]
|
| 212 |
+
- bbox: [x, y, width, height]
|
| 213 |
+
- area: float pixel area
|
| 214 |
+
"""
|
| 215 |
+
if isinstance(binary_mask, torch.Tensor):
|
| 216 |
+
mask_np = binary_mask.detach().cpu().numpy().astype(np.uint8)
|
| 217 |
+
else:
|
| 218 |
+
mask_np = np.asarray(binary_mask, dtype=np.uint8)
|
| 219 |
+
|
| 220 |
+
if mask_np.ndim > 2:
|
| 221 |
+
mask_np = mask_np.squeeze()
|
| 222 |
+
|
| 223 |
+
area = float(mask_np.sum())
|
| 224 |
+
if area < min_area:
|
| 225 |
+
return [], [0.0, 0.0, 0.0, 0.0], 0.0
|
| 226 |
+
|
| 227 |
+
ys, xs = np.where(mask_np > 0)
|
| 228 |
+
x_min, x_max = float(xs.min()), float(xs.max())
|
| 229 |
+
y_min, y_max = float(ys.min()), float(ys.max())
|
| 230 |
+
bbox = [
|
| 231 |
+
round(x_min, 2),
|
| 232 |
+
round(y_min, 2),
|
| 233 |
+
round(x_max - x_min + 1.0, 2),
|
| 234 |
+
round(y_max - y_min + 1.0, 2),
|
| 235 |
+
]
|
| 236 |
+
|
| 237 |
+
contours, _ = cv2.findContours(mask_np, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 238 |
+
polygons: List[List[float]] = []
|
| 239 |
+
for cnt in contours:
|
| 240 |
+
if cv2.contourArea(cnt) < min_area and len(contours) > 1:
|
| 241 |
+
continue
|
| 242 |
+
peri = cv2.arcLength(cnt, True)
|
| 243 |
+
approx = cv2.approxPolyDP(cnt, epsilon_ratio * peri, True) if epsilon_ratio > 0 else cnt
|
| 244 |
+
pts = approx.reshape(-1, 2)
|
| 245 |
+
if pts.shape[0] < 3:
|
| 246 |
+
pts = cnt.reshape(-1, 2)
|
| 247 |
+
if pts.shape[0] >= 3:
|
| 248 |
+
poly = [round(float(coord), 2) for coord in pts.flatten()]
|
| 249 |
+
if len(poly) >= 6:
|
| 250 |
+
polygons.append(poly)
|
| 251 |
+
|
| 252 |
+
if not polygons and area >= min_area:
|
| 253 |
+
# Fallback box polygon if contour is degenerate (e.g. 1-pixel thin line)
|
| 254 |
+
polygons.append(
|
| 255 |
+
[
|
| 256 |
+
round(x_min, 2),
|
| 257 |
+
round(y_min, 2),
|
| 258 |
+
round(x_max, 2),
|
| 259 |
+
round(y_min, 2),
|
| 260 |
+
round(x_max, 2),
|
| 261 |
+
round(y_max, 2),
|
| 262 |
+
round(x_min, 2),
|
| 263 |
+
round(y_max, 2),
|
| 264 |
+
]
|
| 265 |
+
)
|
| 266 |
+
|
| 267 |
+
return polygons, bbox, round(area, 2)
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
class ConvGNAct(nn.Module):
|
| 271 |
+
"""Residual-friendly 3x3 Conv + GroupNorm + GELU block."""
|
| 272 |
+
|
| 273 |
+
def __init__(self, in_ch: int, out_ch: int, kernel_size: int = 3, groups: int = 8):
|
| 274 |
+
super().__init__()
|
| 275 |
+
padding = kernel_size // 2
|
| 276 |
+
eff_groups = min(groups, out_ch)
|
| 277 |
+
while out_ch % eff_groups != 0 and eff_groups > 1:
|
| 278 |
+
eff_groups -= 1
|
| 279 |
+
self.block = nn.Sequential(
|
| 280 |
+
nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, padding=padding, bias=False),
|
| 281 |
+
nn.GroupNorm(eff_groups, out_ch),
|
| 282 |
+
nn.GELU(),
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 286 |
+
return self.block(x)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
class ResidualRefinementBlock(nn.Module):
|
| 290 |
+
"""Residual spatial refinement block for crisp mask boundaries."""
|
| 291 |
+
|
| 292 |
+
def __init__(self, channels: int):
|
| 293 |
+
super().__init__()
|
| 294 |
+
self.conv1 = ConvGNAct(channels, channels, kernel_size=3)
|
| 295 |
+
self.conv2 = ConvGNAct(channels, channels, kernel_size=3)
|
| 296 |
+
|
| 297 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 298 |
+
return x + self.conv2(self.conv1(x))
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
class MultiScaleMaskHead(nn.Module):
|
| 302 |
+
"""
|
| 303 |
+
Lightweight System-1 mask head that fuses:
|
| 304 |
+
1. LLM object decision vectors Q [K, llm_dim]
|
| 305 |
+
2. Coarse merged ViT features F_28 [C_out, H_m, W_m] (1/28 stride)
|
| 306 |
+
3. Fine unmerged ViT skip features F_14 [C_in, 2*H_m, 2*W_m] (1/14 stride)
|
| 307 |
+
and outputs full-resolution mask logits [K, orig_H, orig_W] + IoU confidence scores [K].
|
| 308 |
+
"""
|
| 309 |
+
|
| 310 |
+
def __init__(
|
| 311 |
+
self,
|
| 312 |
+
llm_hidden_size: int = 2048,
|
| 313 |
+
vision_merged_dim: int = 2048,
|
| 314 |
+
vision_unmerged_dim: int = 1280,
|
| 315 |
+
hidden_dim: int = 256,
|
| 316 |
+
fine_dim: int = 128,
|
| 317 |
+
num_heads: int = 4,
|
| 318 |
+
num_fused_layers: int = 4,
|
| 319 |
+
):
|
| 320 |
+
super().__init__()
|
| 321 |
+
self.hidden_dim = hidden_dim
|
| 322 |
+
self.fine_dim = fine_dim
|
| 323 |
+
self.num_fused_layers = num_fused_layers
|
| 324 |
+
|
| 325 |
+
# Learnable fusion across the last N LLM layers (preserves frozen LLM reasoning + mid-late visual grounding)
|
| 326 |
+
self.layer_weights = nn.Parameter(torch.zeros(num_fused_layers))
|
| 327 |
+
self.layer_norms = nn.ModuleList([nn.LayerNorm(llm_hidden_size) for _ in range(num_fused_layers)])
|
| 328 |
+
|
| 329 |
+
# Visual feature projections (computed once per image, shared across K object queries)
|
| 330 |
+
self.coarse_vis_proj = ConvGNAct(vision_merged_dim, hidden_dim, kernel_size=1)
|
| 331 |
+
self.llm_vis_proj = ConvGNAct(llm_hidden_size, hidden_dim, kernel_size=1)
|
| 332 |
+
self.llm_vis_gate = nn.Parameter(torch.tensor(0.5))
|
| 333 |
+
self.fine_vis_proj = ConvGNAct(vision_unmerged_dim, fine_dim, kernel_size=1)
|
| 334 |
+
|
| 335 |
+
# Query projection + cross-attention over coarse 2D visual tokens
|
| 336 |
+
self.query_in_proj = nn.Sequential(
|
| 337 |
+
nn.Linear(llm_hidden_size, hidden_dim),
|
| 338 |
+
nn.LayerNorm(hidden_dim),
|
| 339 |
+
nn.GELU(),
|
| 340 |
+
nn.Linear(hidden_dim, hidden_dim),
|
| 341 |
+
)
|
| 342 |
+
self.cross_attn = nn.MultiheadAttention(
|
| 343 |
+
embed_dim=hidden_dim,
|
| 344 |
+
num_heads=num_heads,
|
| 345 |
+
batch_first=True,
|
| 346 |
+
)
|
| 347 |
+
self.cross_norm = nn.LayerNorm(hidden_dim)
|
| 348 |
+
|
| 349 |
+
# Query-to-coarse and query-to-fine dynamic modulators
|
| 350 |
+
self.coarse_query_proj = nn.Linear(hidden_dim, hidden_dim)
|
| 351 |
+
self.llm_query_proj = nn.Linear(hidden_dim, hidden_dim)
|
| 352 |
+
self.coarse_gate_proj = nn.Linear(hidden_dim, hidden_dim)
|
| 353 |
+
self.fine_gate_proj = nn.Linear(hidden_dim, fine_dim)
|
| 354 |
+
|
| 355 |
+
# Coarse decoder at 1/28 scale (concatenates modulated features + dot-product similarity map)
|
| 356 |
+
self.coarse_fuse = ConvGNAct(hidden_dim + 1, hidden_dim, kernel_size=3)
|
| 357 |
+
self.coarse_res = ResidualRefinementBlock(hidden_dim)
|
| 358 |
+
|
| 359 |
+
# Fine decoder at 1/14 scale (fuses upsampled coarse features + fine 1/14 ViT skip features)
|
| 360 |
+
self.fine_fuse = ConvGNAct(hidden_dim + fine_dim, fine_dim, kernel_size=3)
|
| 361 |
+
self.fine_res1 = ResidualRefinementBlock(fine_dim)
|
| 362 |
+
self.fine_res2 = ResidualRefinementBlock(fine_dim)
|
| 363 |
+
|
| 364 |
+
# 2x learned upsampler to 1/7 stride before final bilinear interpolation
|
| 365 |
+
self.up_conv = nn.Sequential(
|
| 366 |
+
nn.ConvTranspose2d(fine_dim, fine_dim // 2, kernel_size=2, stride=2),
|
| 367 |
+
nn.GroupNorm(min(8, fine_dim // 2), fine_dim // 2),
|
| 368 |
+
nn.GELU(),
|
| 369 |
+
ConvGNAct(fine_dim // 2, fine_dim // 2, kernel_size=3),
|
| 370 |
+
)
|
| 371 |
+
self.mask_out = nn.Conv2d(fine_dim // 2, 1, kernel_size=1)
|
| 372 |
+
|
| 373 |
+
# Predicts calibrated mask confidence / IoU score per object query
|
| 374 |
+
self.score_head = nn.Sequential(
|
| 375 |
+
nn.Linear(hidden_dim, hidden_dim // 2),
|
| 376 |
+
nn.GELU(),
|
| 377 |
+
nn.Linear(hidden_dim // 2, 1),
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
def _build_2d_sincos_pos_embed(
|
| 381 |
+
self, h: int, w: int, dim: int, device: torch.device, dtype: torch.dtype
|
| 382 |
+
) -> torch.Tensor:
|
| 383 |
+
"""Generates parameter-free 2D sine-cosine positional embeddings [1, h*w, dim]."""
|
| 384 |
+
y = torch.linspace(-1.0, 1.0, steps=h, device=device, dtype=torch.float32)
|
| 385 |
+
x = torch.linspace(-1.0, 1.0, steps=w, device=device, dtype=torch.float32)
|
| 386 |
+
grid_y, grid_x = torch.meshgrid(y, x, indexing="ij")
|
| 387 |
+
half = dim // 4
|
| 388 |
+
if half == 0:
|
| 389 |
+
return torch.zeros((1, h * w, dim), device=device, dtype=dtype)
|
| 390 |
+
omega = torch.arange(half, device=device, dtype=torch.float32) / max(half, 1)
|
| 391 |
+
omega = 1.0 / (10000.0**omega)
|
| 392 |
+
out_y = grid_y.reshape(-1, 1) * omega.reshape(1, -1)
|
| 393 |
+
out_x = grid_x.reshape(-1, 1) * omega.reshape(1, -1)
|
| 394 |
+
pos = torch.cat([torch.sin(out_y), torch.cos(out_y), torch.sin(out_x), torch.cos(out_x)], dim=-1)
|
| 395 |
+
if pos.shape[-1] < dim:
|
| 396 |
+
pos = F.pad(pos, (0, dim - pos.shape[-1]))
|
| 397 |
+
return pos.unsqueeze(0).to(dtype=dtype)
|
| 398 |
+
|
| 399 |
+
def forward_single_image(
|
| 400 |
+
self,
|
| 401 |
+
queries: torch.Tensor,
|
| 402 |
+
f_coarse: torch.Tensor,
|
| 403 |
+
f_fine: torch.Tensor,
|
| 404 |
+
target_size: Tuple[int, int],
|
| 405 |
+
f_llm_coarse: Optional[torch.Tensor] = None,
|
| 406 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 407 |
+
"""
|
| 408 |
+
Args:
|
| 409 |
+
queries: [K, llm_hidden_size] or [K, L, llm_hidden_size] decision vectors for K objects
|
| 410 |
+
f_coarse: [1, vision_merged_dim, H_m, W_m] (1/28 resolution from ViT)
|
| 411 |
+
f_fine: [1, vision_unmerged_dim, H_f, W_f] (1/14 resolution from ViT)
|
| 412 |
+
target_size: (orig_H, orig_W)
|
| 413 |
+
f_llm_coarse: optional [1, llm_hidden_size, H_m, W_m] riddle-conditioned `<|image_pad|>` grid from LLM
|
| 414 |
+
|
| 415 |
+
Returns:
|
| 416 |
+
mask_logits: [K, orig_H, orig_W]
|
| 417 |
+
pred_scores: [K] (in [0, 1])
|
| 418 |
+
"""
|
| 419 |
+
num_queries = queries.shape[0]
|
| 420 |
+
_, _, h_m, w_m = f_coarse.shape
|
| 421 |
+
_, _, h_f, w_f = f_fine.shape
|
| 422 |
+
|
| 423 |
+
# Fuse multi-layer LLM hidden states [K, L, llm_hidden_size] -> [K, llm_hidden_size]
|
| 424 |
+
if queries.ndim == 3:
|
| 425 |
+
num_layers = queries.shape[1]
|
| 426 |
+
normed_layers = [
|
| 427 |
+
self.layer_norms[min(i, len(self.layer_norms) - 1)](queries[:, i, :])
|
| 428 |
+
for i in range(num_layers)
|
| 429 |
+
]
|
| 430 |
+
stacked = torch.stack(normed_layers, dim=1) # [K, L, llm_hidden_size]
|
| 431 |
+
weights = F.softmax(self.layer_weights[:num_layers], dim=0).view(1, num_layers, 1)
|
| 432 |
+
queries = (stacked * weights).sum(dim=1)
|
| 433 |
+
|
| 434 |
+
# Project visual features once for the image
|
| 435 |
+
vis_coarse = self.coarse_vis_proj(f_coarse) # [1, hidden_dim, H_m, W_m]
|
| 436 |
+
vis_llm = None
|
| 437 |
+
if f_llm_coarse is not None:
|
| 438 |
+
vis_llm = self.llm_vis_proj(f_llm_coarse) # [1, hidden_dim, H_m, W_m]
|
| 439 |
+
vis_coarse = vis_coarse + torch.tanh(self.llm_vis_gate) * vis_llm
|
| 440 |
+
vis_fine = self.fine_vis_proj(f_fine) # [1, fine_dim, H_f, W_f]
|
| 441 |
+
|
| 442 |
+
# Project queries & refine with cross-attention over 2D visual tokens
|
| 443 |
+
q = self.query_in_proj(queries).unsqueeze(0) # [1, K, hidden_dim]
|
| 444 |
+
vis_tokens = vis_coarse.flatten(2).transpose(1, 2) # [1, H_m*W_m, hidden_dim]
|
| 445 |
+
pos_2d = self._build_2d_sincos_pos_embed(
|
| 446 |
+
h_m, w_m, self.hidden_dim, vis_tokens.device, vis_tokens.dtype
|
| 447 |
+
)
|
| 448 |
+
attn_out, _ = self.cross_attn(
|
| 449 |
+
query=q,
|
| 450 |
+
key=vis_tokens + pos_2d,
|
| 451 |
+
value=vis_tokens,
|
| 452 |
+
)
|
| 453 |
+
q = self.cross_norm(q + attn_out).squeeze(0) # [K, hidden_dim]
|
| 454 |
+
|
| 455 |
+
# Predict confidence score per object query
|
| 456 |
+
pred_scores = torch.sigmoid(self.score_head(q).squeeze(-1)) # [K]
|
| 457 |
+
|
| 458 |
+
# 1. Coarse (1/28) dot-product similarity + channel gating
|
| 459 |
+
q_dot = F.normalize(self.coarse_query_proj(q), dim=-1) # [K, hidden_dim]
|
| 460 |
+
v_norm = F.normalize(vis_coarse.squeeze(0), dim=0) # [hidden_dim, H_m, W_m]
|
| 461 |
+
sim_map = torch.einsum("kd,dhw->khw", q_dot, v_norm).unsqueeze(1) # [K, 1, H_m, W_m]
|
| 462 |
+
if vis_llm is not None:
|
| 463 |
+
q_llm_dot = F.normalize(self.llm_query_proj(q), dim=-1)
|
| 464 |
+
v_llm_norm = F.normalize(vis_llm.squeeze(0), dim=0)
|
| 465 |
+
sim_map_llm = torch.einsum("kd,dhw->khw", q_llm_dot, v_llm_norm).unsqueeze(1)
|
| 466 |
+
sim_map = 0.5 * (sim_map + sim_map_llm)
|
| 467 |
+
|
| 468 |
+
coarse_gate = torch.sigmoid(self.coarse_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, hidden_dim, 1, 1]
|
| 469 |
+
vis_coarse_exp = vis_coarse.expand(num_queries, -1, -1, -1)
|
| 470 |
+
coarse_mod = vis_coarse_exp * coarse_gate # [K, hidden_dim, H_m, W_m]
|
| 471 |
+
|
| 472 |
+
x = torch.cat([coarse_mod, sim_map], dim=1) # [K, hidden_dim + 1, H_m, W_m]
|
| 473 |
+
x = self.coarse_res(self.coarse_fuse(x))
|
| 474 |
+
|
| 475 |
+
# 2. Upsample 2x to 1/14 resolution & fuse with unmerged ViT skip features
|
| 476 |
+
x = F.interpolate(x, size=(h_f, w_f), mode="bilinear", align_corners=False)
|
| 477 |
+
fine_gate = torch.sigmoid(self.fine_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, fine_dim, 1, 1]
|
| 478 |
+
vis_fine_exp = vis_fine.expand(num_queries, -1, -1, -1) * fine_gate
|
| 479 |
+
|
| 480 |
+
x = torch.cat([x, vis_fine_exp], dim=1) # [K, hidden_dim + fine_dim, H_f, W_f]
|
| 481 |
+
x = self.fine_fuse(x)
|
| 482 |
+
x = self.fine_res2(self.fine_res1(x))
|
| 483 |
+
|
| 484 |
+
# 3. Learned 2x upsample (to 1/7 stride) + bilinear resize to original image resolution
|
| 485 |
+
x = self.up_conv(x)
|
| 486 |
+
raw_logits = self.mask_out(x) # [K, 1, 2*H_f, 2*W_f]
|
| 487 |
+
upsampled_logits = F.interpolate(
|
| 488 |
+
raw_logits,
|
| 489 |
+
size=target_size,
|
| 490 |
+
mode="bilinear",
|
| 491 |
+
align_corners=False,
|
| 492 |
+
).squeeze(1) # [K, orig_H, orig_W]
|
| 493 |
+
|
| 494 |
+
return upsampled_logits, pred_scores
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
class Qwen2_5_VL_SinglePassSegmenter(nn.Module):
|
| 498 |
+
"""
|
| 499 |
+
System-1 Single-Pass Multi-Object Segmenter built on Qwen2.5-VL.
|
| 500 |
+
- Drops the autoregressive LM head completely.
|
| 501 |
+
- Runs the ViT encoder and LLM backbone once.
|
| 502 |
+
- Captures both 1/14 (unmerged) and 1/28 (merged) ViT spatial grids from the single pass.
|
| 503 |
+
- Extracts K decision vectors from `<|object_ref_end|>` tokens in the input JSON.
|
| 504 |
+
- Decodes all K object masks in parallel.
|
| 505 |
+
"""
|
| 506 |
+
|
| 507 |
+
def __init__(
|
| 508 |
+
self,
|
| 509 |
+
qwen_model_name: Optional[str] = "./Qwen3.5-4B",
|
| 510 |
+
backbone: Optional[nn.Module] = None,
|
| 511 |
+
processor: Optional[Any] = None,
|
| 512 |
+
torch_dtype: torch.dtype = torch.bfloat16,
|
| 513 |
+
device_map: Optional[Union[str, Dict[str, Any]]] = None,
|
| 514 |
+
quantization_config: Optional[Any] = None,
|
| 515 |
+
freeze_backbone: bool = True,
|
| 516 |
+
head_hidden_dim: int = 256,
|
| 517 |
+
head_fine_dim: int = 128,
|
| 518 |
+
num_fused_layers: int = 4,
|
| 519 |
+
):
|
| 520 |
+
super().__init__()
|
| 521 |
+
self.num_fused_layers = num_fused_layers
|
| 522 |
+
|
| 523 |
+
if backbone is not None:
|
| 524 |
+
self.qwen = backbone
|
| 525 |
+
self.processor = processor
|
| 526 |
+
else:
|
| 527 |
+
if qwen_model_name is None:
|
| 528 |
+
raise ValueError("Provide either `qwen_model_name` or a pre-built `backbone`.")
|
| 529 |
+
self.processor = processor or AutoProcessor.from_pretrained(qwen_model_name)
|
| 530 |
+
load_kwargs: Dict[str, Any] = {"torch_dtype": torch_dtype}
|
| 531 |
+
if device_map is not None:
|
| 532 |
+
load_kwargs["device_map"] = device_map
|
| 533 |
+
if quantization_config is not None:
|
| 534 |
+
load_kwargs["quantization_config"] = quantization_config
|
| 535 |
+
self.qwen = AutoModel.from_pretrained(qwen_model_name, **load_kwargs)
|
| 536 |
+
|
| 537 |
+
# Support both Qwen2_5_VLModel and Qwen2_5_VLForConditionalGeneration
|
| 538 |
+
if hasattr(self.qwen, "model") and hasattr(self.qwen.model, "visual"):
|
| 539 |
+
self.backbone = self.qwen.model
|
| 540 |
+
else:
|
| 541 |
+
self.backbone = self.qwen
|
| 542 |
+
|
| 543 |
+
self.freeze_backbone = freeze_backbone
|
| 544 |
+
if freeze_backbone:
|
| 545 |
+
for param in self.backbone.parameters():
|
| 546 |
+
param.requires_grad = False
|
| 547 |
+
|
| 548 |
+
# Read dimensions safely across transformers versions
|
| 549 |
+
cfg = self.backbone.config
|
| 550 |
+
text_cfg = getattr(cfg, "text_config", cfg)
|
| 551 |
+
vis_cfg = cfg.vision_config
|
| 552 |
+
|
| 553 |
+
self.llm_hidden_size = getattr(text_cfg, "hidden_size", 2560)
|
| 554 |
+
self.vision_merged_dim = getattr(vis_cfg, "out_hidden_size", self.llm_hidden_size)
|
| 555 |
+
self.vision_unmerged_dim = getattr(vis_cfg, "hidden_size", 1024)
|
| 556 |
+
self.spatial_merge_size = getattr(vis_cfg, "spatial_merge_size", 2)
|
| 557 |
+
self.patch_size = getattr(vis_cfg, "patch_size", 16)
|
| 558 |
+
|
| 559 |
+
if self.processor is not None and hasattr(self.processor, "tokenizer"):
|
| 560 |
+
self.query_token_id = self.processor.tokenizer.convert_tokens_to_ids(OBJECT_QUERY_TOKEN)
|
| 561 |
+
self.im_end_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
|
| 562 |
+
self.image_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|image_pad|>")
|
| 563 |
+
else:
|
| 564 |
+
self.query_token_id = getattr(cfg, "object_ref_end_token_id", 248048)
|
| 565 |
+
self.im_end_token_id = getattr(text_cfg, "eos_token_id", 248046)
|
| 566 |
+
self.image_token_id = getattr(cfg, "image_token_id", 248056)
|
| 567 |
+
|
| 568 |
+
self.mask_head = MultiScaleMaskHead(
|
| 569 |
+
llm_hidden_size=self.llm_hidden_size,
|
| 570 |
+
vision_merged_dim=self.vision_merged_dim,
|
| 571 |
+
vision_unmerged_dim=self.vision_unmerged_dim,
|
| 572 |
+
hidden_dim=head_hidden_dim,
|
| 573 |
+
fine_dim=head_fine_dim,
|
| 574 |
+
num_fused_layers=num_fused_layers,
|
| 575 |
+
)
|
| 576 |
+
|
| 577 |
+
# Hook buffers to capture single-pass visual outputs & last N LLM layers without extra memory
|
| 578 |
+
self._captured_visual: Dict[str, Any] = {}
|
| 579 |
+
self._captured_llm_layers: List[torch.Tensor] = []
|
| 580 |
+
self._visual_hook = self.backbone.visual.register_forward_hook(self._capture_visual_hook)
|
| 581 |
+
self._llm_hooks: List[Any] = []
|
| 582 |
+
self._register_llm_layer_hooks()
|
| 583 |
+
self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
|
| 584 |
+
|
| 585 |
+
def _check_needs_mm_token_type_ids(self) -> bool:
|
| 586 |
+
return "mm_token_type_ids" in inspect.signature(
|
| 587 |
+
self.backbone.forward
|
| 588 |
+
).parameters or (
|
| 589 |
+
hasattr(self.backbone, "compute_3d_position_ids")
|
| 590 |
+
and "mm_token_type_ids" in inspect.signature(self.backbone.compute_3d_position_ids).parameters
|
| 591 |
+
)
|
| 592 |
+
|
| 593 |
+
def _get_llm_decoder_layers(self) -> Optional[nn.ModuleList]:
|
| 594 |
+
lm = getattr(self.backbone, "language_model", None)
|
| 595 |
+
if lm is None:
|
| 596 |
+
return None
|
| 597 |
+
if hasattr(lm, "layers"):
|
| 598 |
+
return lm.layers
|
| 599 |
+
if hasattr(lm, "model") and hasattr(lm.model, "layers"):
|
| 600 |
+
return lm.model.layers
|
| 601 |
+
return None
|
| 602 |
+
|
| 603 |
+
def _register_llm_layer_hooks(self) -> None:
|
| 604 |
+
for h in self._llm_hooks:
|
| 605 |
+
h.remove()
|
| 606 |
+
self._llm_hooks.clear()
|
| 607 |
+
layers = self._get_llm_decoder_layers()
|
| 608 |
+
if layers is not None and len(layers) > 0:
|
| 609 |
+
k = min(self.num_fused_layers, len(layers))
|
| 610 |
+
for layer in list(layers)[-k:]:
|
| 611 |
+
self._llm_hooks.append(layer.register_forward_hook(self._capture_llm_layer_hook))
|
| 612 |
+
|
| 613 |
+
def _capture_llm_layer_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
|
| 614 |
+
hidden = output[0] if isinstance(output, tuple) else output
|
| 615 |
+
if isinstance(hidden, torch.Tensor):
|
| 616 |
+
self._captured_llm_layers.append(hidden)
|
| 617 |
+
|
| 618 |
+
def _capture_visual_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
|
| 619 |
+
if hasattr(output, "last_hidden_state") and hasattr(output, "pooler_output"):
|
| 620 |
+
self._captured_visual["unmerged"] = output.last_hidden_state
|
| 621 |
+
# Clone reference before get_image_features mutates output.pooler_output into a tuple
|
| 622 |
+
self._captured_visual["merged"] = output.pooler_output
|
| 623 |
+
elif isinstance(output, tuple) and len(output) >= 2:
|
| 624 |
+
self._captured_visual["unmerged"] = output[0]
|
| 625 |
+
self._captured_visual["merged"] = output[1]
|
| 626 |
+
else:
|
| 627 |
+
self._captured_visual["merged"] = output
|
| 628 |
+
self._captured_visual["unmerged"] = None
|
| 629 |
+
|
| 630 |
+
def enable_lora(
|
| 631 |
+
self,
|
| 632 |
+
r: int = 8,
|
| 633 |
+
lora_alpha: int = 16,
|
| 634 |
+
lora_dropout: float = 0.05,
|
| 635 |
+
top_k_layers: int = 4,
|
| 636 |
+
target_modules: Tuple[str, ...] = (
|
| 637 |
+
"q_proj",
|
| 638 |
+
"k_proj",
|
| 639 |
+
"v_proj",
|
| 640 |
+
"o_proj",
|
| 641 |
+
"in_proj_qkv",
|
| 642 |
+
"out_proj",
|
| 643 |
+
),
|
| 644 |
+
) -> None:
|
| 645 |
+
"""
|
| 646 |
+
Safe Stage 2 training helper:
|
| 647 |
+
Applies low-rank LoRA ONLY to the top `top_k_layers` (e.g., last 4 of 36 layers),
|
| 648 |
+
leaving the first ~32 layers 100% frozen so Qwen retains all world knowledge and
|
| 649 |
+
riddle-solving ability while the top layers adapt to spatial token compression.
|
| 650 |
+
"""
|
| 651 |
+
from peft import LoraConfig, get_peft_model
|
| 652 |
+
|
| 653 |
+
layers = self._get_llm_decoder_layers()
|
| 654 |
+
resolved_targets: List[str] = list(target_modules)
|
| 655 |
+
if layers is not None and top_k_layers > 0 and len(layers) > top_k_layers:
|
| 656 |
+
total_l = len(layers)
|
| 657 |
+
allowed_layer_indices = set(range(total_l - top_k_layers, total_l))
|
| 658 |
+
exact_targets: List[str] = []
|
| 659 |
+
for name, _ in self.backbone.language_model.named_modules():
|
| 660 |
+
leaf = name.split(".")[-1]
|
| 661 |
+
if leaf not in target_modules:
|
| 662 |
+
continue
|
| 663 |
+
if any(
|
| 664 |
+
f"layers.{idx}." in name or name.startswith(f"layers.{idx}.")
|
| 665 |
+
for idx in allowed_layer_indices
|
| 666 |
+
):
|
| 667 |
+
exact_targets.append(name)
|
| 668 |
+
if exact_targets:
|
| 669 |
+
resolved_targets = exact_targets
|
| 670 |
+
|
| 671 |
+
lora_cfg = LoraConfig(
|
| 672 |
+
r=r,
|
| 673 |
+
lora_alpha=lora_alpha,
|
| 674 |
+
lora_dropout=lora_dropout,
|
| 675 |
+
target_modules=resolved_targets,
|
| 676 |
+
bias="none",
|
| 677 |
+
)
|
| 678 |
+
self.backbone.language_model = get_peft_model(self.backbone.language_model, lora_cfg)
|
| 679 |
+
self._register_llm_layer_hooks()
|
| 680 |
+
self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
|
| 681 |
+
self.freeze_backbone = False
|
| 682 |
+
|
| 683 |
+
def _compute_window_index(
|
| 684 |
+
self,
|
| 685 |
+
grid_thw: torch.Tensor,
|
| 686 |
+
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
|
| 687 |
+
) -> torch.Tensor:
|
| 688 |
+
if hasattr(self.backbone.visual, "get_window_index"):
|
| 689 |
+
window_index, _ = self.backbone.visual.get_window_index(grid_thw)
|
| 690 |
+
return window_index
|
| 691 |
+
vis_cfg = getattr(self.backbone.config, "vision_config", None)
|
| 692 |
+
window_size = getattr(vis_cfg, "window_size", 112)
|
| 693 |
+
m = self.spatial_merge_size
|
| 694 |
+
vit_merger_window_size = max(window_size // m // self.patch_size, 1)
|
| 695 |
+
window_index: List[torch.Tensor] = []
|
| 696 |
+
window_index_id = 0
|
| 697 |
+
rows = grid_thw_list if grid_thw_list is not None else grid_thw.tolist()
|
| 698 |
+
for grid_t, grid_h, grid_w in rows:
|
| 699 |
+
llm_grid_h, llm_grid_w = grid_h // m, grid_w // m
|
| 700 |
+
index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
|
| 701 |
+
pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
|
| 702 |
+
pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
|
| 703 |
+
num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
|
| 704 |
+
num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
|
| 705 |
+
index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
|
| 706 |
+
index_padded = index_padded.reshape(
|
| 707 |
+
grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size
|
| 708 |
+
)
|
| 709 |
+
index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
|
| 710 |
+
grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size
|
| 711 |
+
)
|
| 712 |
+
index_padded = index_padded.reshape(-1)
|
| 713 |
+
index_new = index_padded[index_padded != -100]
|
| 714 |
+
window_index.append(index_new + window_index_id)
|
| 715 |
+
window_index_id += grid_t * llm_grid_h * llm_grid_w
|
| 716 |
+
return torch.cat(window_index, dim=0)
|
| 717 |
+
|
| 718 |
+
def _reconstruct_2d_vision_features(
|
| 719 |
+
self,
|
| 720 |
+
merged_tokens: Union[torch.Tensor, Sequence[torch.Tensor]],
|
| 721 |
+
unmerged_tokens: Optional[torch.Tensor],
|
| 722 |
+
image_grid_thw: torch.Tensor,
|
| 723 |
+
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
|
| 724 |
+
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
| 725 |
+
"""
|
| 726 |
+
Reconstructs per-image 2D spatial grids from Qwen2.5-VL's window-permuted ViT outputs:
|
| 727 |
+
- coarse_grids: list of [1, C_merged, H_m, W_m] at 1/28 resolution
|
| 728 |
+
- fine_grids: list of [1, C_unmerged, 2*H_m, 2*W_m] at 1/14 resolution
|
| 729 |
+
"""
|
| 730 |
+
m = self.spatial_merge_size
|
| 731 |
+
m_sq = m * m
|
| 732 |
+
rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
|
| 733 |
+
|
| 734 |
+
if isinstance(merged_tokens, (tuple, list)):
|
| 735 |
+
merged_cat = torch.cat(list(merged_tokens), dim=0)
|
| 736 |
+
else:
|
| 737 |
+
merged_cat = merged_tokens
|
| 738 |
+
|
| 739 |
+
# Un-permute the unmerged tokens from window attention order if the ViT uses window attention (Qwen2.5-VL);
|
| 740 |
+
# Qwen3.5-4B uses full ViT attention so its unmerged tokens are already in natural block-raster order.
|
| 741 |
+
vis_cfg = getattr(self.backbone.config, "vision_config", None)
|
| 742 |
+
has_window_attn = hasattr(vis_cfg, "window_size") and getattr(vis_cfg, "window_size", None) is not None
|
| 743 |
+
if unmerged_tokens is not None:
|
| 744 |
+
num_merged = merged_cat.shape[0]
|
| 745 |
+
if has_window_attn:
|
| 746 |
+
window_index = self._compute_window_index(image_grid_thw, grid_thw_list=rows)
|
| 747 |
+
reverse_indices = torch.argsort(window_index.to(unmerged_tokens.device))
|
| 748 |
+
unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)[reverse_indices]
|
| 749 |
+
else:
|
| 750 |
+
unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)
|
| 751 |
+
else:
|
| 752 |
+
unmerged_blocks = None
|
| 753 |
+
|
| 754 |
+
coarse_grids: List[torch.Tensor] = []
|
| 755 |
+
fine_grids: List[torch.Tensor] = []
|
| 756 |
+
|
| 757 |
+
offset = 0
|
| 758 |
+
for grid_t, grid_h, grid_w in rows:
|
| 759 |
+
h_m = grid_h // m
|
| 760 |
+
w_m = grid_w // m
|
| 761 |
+
n_tokens = grid_t * h_m * w_m
|
| 762 |
+
|
| 763 |
+
img_merged = merged_cat[offset : offset + n_tokens]
|
| 764 |
+
# Average across temporal dimension if grid_t > 1 (for static images grid_t == 1)
|
| 765 |
+
img_merged = img_merged.view(grid_t, h_m, w_m, -1).mean(dim=0)
|
| 766 |
+
f_coarse = img_merged.permute(2, 0, 1).unsqueeze(0).contiguous() # [1, C_out, H_m, W_m]
|
| 767 |
+
coarse_grids.append(f_coarse)
|
| 768 |
+
|
| 769 |
+
if unmerged_blocks is not None:
|
| 770 |
+
img_unmerged = unmerged_blocks[offset : offset + n_tokens]
|
| 771 |
+
# Matches Qwen2VLImageProcessor patch ordering: (T, H_m, W_m, m_h, m_w, C_in)
|
| 772 |
+
img_unmerged = img_unmerged.view(grid_t, h_m, w_m, m, m, -1).mean(dim=0)
|
| 773 |
+
f_fine = (
|
| 774 |
+
img_unmerged.permute(4, 0, 2, 1, 3)
|
| 775 |
+
.contiguous()
|
| 776 |
+
.view(1, self.vision_unmerged_dim, grid_h, grid_w)
|
| 777 |
+
)
|
| 778 |
+
fine_grids.append(f_fine)
|
| 779 |
+
else:
|
| 780 |
+
# Fallback if unmerged tokens are unavailable
|
| 781 |
+
f_fine = F.interpolate(f_coarse, size=(grid_h, grid_w), mode="bilinear", align_corners=False)
|
| 782 |
+
fine_grids.append(f_fine)
|
| 783 |
+
|
| 784 |
+
offset += n_tokens
|
| 785 |
+
|
| 786 |
+
return coarse_grids, fine_grids
|
| 787 |
+
|
| 788 |
+
def _extract_llm_visual_grids(
|
| 789 |
+
self,
|
| 790 |
+
last_hidden_states: torch.Tensor,
|
| 791 |
+
input_ids: torch.Tensor,
|
| 792 |
+
image_grid_thw: torch.Tensor,
|
| 793 |
+
layer_states: Optional[Sequence[torch.Tensor]] = None,
|
| 794 |
+
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
|
| 795 |
+
input_ids_cpu: Optional[torch.Tensor] = None,
|
| 796 |
+
) -> List[Optional[torch.Tensor]]:
|
| 797 |
+
"""
|
| 798 |
+
Extracts the LLM-contextualized `<|image_pad|>` visual tokens from the top LLM layers
|
| 799 |
+
and reshapes them into 2D grids [1, llm_dim, H_m, W_m] per image.
|
| 800 |
+
Because the user's JSON riddle precedes `<|image_pad|>`, these tokens carry
|
| 801 |
+
36 layers of riddle-to-image self-attention context.
|
| 802 |
+
"""
|
| 803 |
+
m = self.spatial_merge_size
|
| 804 |
+
base_dev = last_hidden_states.device
|
| 805 |
+
source_state = (
|
| 806 |
+
torch.stack([st.to(base_dev) for st in layer_states], dim=0).mean(dim=0)
|
| 807 |
+
if layer_states and len(layer_states) > 0
|
| 808 |
+
else last_hidden_states
|
| 809 |
+
)
|
| 810 |
+
rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
|
| 811 |
+
ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
|
| 812 |
+
target_dev = source_state.device
|
| 813 |
+
llm_grids: List[Optional[torch.Tensor]] = []
|
| 814 |
+
for b in range(ids_src.shape[0]):
|
| 815 |
+
grid_t, grid_h, grid_w = rows[b]
|
| 816 |
+
h_m, w_m = grid_h // m, grid_w // m
|
| 817 |
+
n_tokens = grid_t * h_m * w_m
|
| 818 |
+
img_pos = (ids_src[b] == self.image_token_id).nonzero(as_tuple=True)[0]
|
| 819 |
+
if img_pos.numel() == n_tokens:
|
| 820 |
+
if img_pos.device != target_dev:
|
| 821 |
+
img_pos = img_pos.to(device=target_dev, non_blocking=True)
|
| 822 |
+
vis_seq = source_state[b, img_pos, :] # [n_tokens, llm_dim]
|
| 823 |
+
vis_2d = (
|
| 824 |
+
vis_seq.view(grid_t, h_m, w_m, -1)
|
| 825 |
+
.mean(dim=0)
|
| 826 |
+
.permute(2, 0, 1)
|
| 827 |
+
.unsqueeze(0)
|
| 828 |
+
.contiguous()
|
| 829 |
+
)
|
| 830 |
+
llm_grids.append(vis_2d)
|
| 831 |
+
else:
|
| 832 |
+
llm_grids.append(None)
|
| 833 |
+
return llm_grids
|
| 834 |
+
|
| 835 |
+
def _extract_query_vectors(
|
| 836 |
+
self,
|
| 837 |
+
last_hidden_states: torch.Tensor,
|
| 838 |
+
input_ids: torch.Tensor,
|
| 839 |
+
layer_states: Optional[Sequence[torch.Tensor]] = None,
|
| 840 |
+
input_ids_cpu: Optional[torch.Tensor] = None,
|
| 841 |
+
) -> List[torch.Tensor]:
|
| 842 |
+
"""
|
| 843 |
+
Extracts the K `<|object_ref_end|>` decision vectors per batch item.
|
| 844 |
+
If multi-layer states are captured, returns [K, L, llm_dim] per batch item;
|
| 845 |
+
otherwise returns [K, llm_dim].
|
| 846 |
+
"""
|
| 847 |
+
states_to_index = list(layer_states) if layer_states else [last_hidden_states]
|
| 848 |
+
ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
|
| 849 |
+
target_dev = states_to_index[0].device
|
| 850 |
+
batch_queries: List[torch.Tensor] = []
|
| 851 |
+
for b in range(ids_src.shape[0]):
|
| 852 |
+
pos = (ids_src[b] == self.query_token_id).nonzero(as_tuple=True)[0]
|
| 853 |
+
if pos.numel() == 0:
|
| 854 |
+
pos = (ids_src[b] == self.im_end_token_id).nonzero(as_tuple=True)[0]
|
| 855 |
+
if pos.numel() > 0:
|
| 856 |
+
pos = pos[-1:]
|
| 857 |
+
else:
|
| 858 |
+
pos = torch.tensor([ids_src.shape[1] - 1], dtype=torch.long, device=ids_src.device)
|
| 859 |
+
if pos.device != target_dev:
|
| 860 |
+
pos = pos.to(device=target_dev, non_blocking=True)
|
| 861 |
+
if len(states_to_index) > 1:
|
| 862 |
+
per_layer = [
|
| 863 |
+
st[b, (pos.to(st.device) if pos.device != st.device else pos), :].to(target_dev)
|
| 864 |
+
for st in states_to_index
|
| 865 |
+
]
|
| 866 |
+
batch_queries.append(torch.stack(per_layer, dim=1)) # [K, L, llm_dim]
|
| 867 |
+
else:
|
| 868 |
+
batch_queries.append(states_to_index[0][b, pos, :]) # [K, llm_dim]
|
| 869 |
+
return batch_queries
|
| 870 |
+
|
| 871 |
+
def forward(
|
| 872 |
+
self,
|
| 873 |
+
input_ids: torch.LongTensor,
|
| 874 |
+
attention_mask: torch.Tensor,
|
| 875 |
+
pixel_values: torch.Tensor,
|
| 876 |
+
image_grid_thw: torch.LongTensor,
|
| 877 |
+
target_sizes: Optional[Sequence[Tuple[int, int]]] = None,
|
| 878 |
+
mm_token_type_ids: Optional[torch.Tensor] = None,
|
| 879 |
+
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
|
| 880 |
+
input_ids_cpu: Optional[torch.Tensor] = None,
|
| 881 |
+
) -> Dict[str, Any]:
|
| 882 |
+
"""
|
| 883 |
+
Single forward pass from tokenized JSON + image patches to pixel masks.
|
| 884 |
+
|
| 885 |
+
Args:
|
| 886 |
+
input_ids: [B, seq_len]
|
| 887 |
+
attention_mask: [B, seq_len]
|
| 888 |
+
pixel_values: [sum_patches, patch_dim]
|
| 889 |
+
image_grid_thw: [B, 3] (T, grid_h, grid_w)
|
| 890 |
+
target_sizes: optional list of (orig_H, orig_W) per image in batch.
|
| 891 |
+
Defaults to (grid_h * patch_size, grid_w * patch_size).
|
| 892 |
+
mm_token_type_ids: optional [B, seq_len] multimodal token type IDs (auto-computed if needed).
|
| 893 |
+
grid_thw_list: optional pre-extracted Python list of [T, grid_h, grid_w] per image to avoid CUDA syncs.
|
| 894 |
+
input_ids_cpu: optional CPU copy of `input_ids` to extract token positions without CUDA `.nonzero()` syncs.
|
| 895 |
+
|
| 896 |
+
Returns:
|
| 897 |
+
dict with:
|
| 898 |
+
- "mask_logits": List[Tensor] of shape [K_b, orig_H, orig_W] per image
|
| 899 |
+
- "mask_probs": List[Tensor] of shape [K_b, orig_H, orig_W] per image
|
| 900 |
+
- "pred_scores": List[Tensor] of shape [K_b] per image
|
| 901 |
+
"""
|
| 902 |
+
self._captured_visual.clear()
|
| 903 |
+
self._captured_llm_layers.clear()
|
| 904 |
+
|
| 905 |
+
resolved_grid_thw = list(grid_thw_list) if grid_thw_list is not None else image_grid_thw.tolist()
|
| 906 |
+
|
| 907 |
+
backbone_kwargs: Dict[str, Any] = {
|
| 908 |
+
"input_ids": input_ids,
|
| 909 |
+
"attention_mask": attention_mask,
|
| 910 |
+
"pixel_values": pixel_values,
|
| 911 |
+
"image_grid_thw": image_grid_thw,
|
| 912 |
+
"use_cache": False,
|
| 913 |
+
"output_hidden_states": False,
|
| 914 |
+
"return_dict": True,
|
| 915 |
+
}
|
| 916 |
+
if self._needs_mm_token_type_ids:
|
| 917 |
+
if mm_token_type_ids is None:
|
| 918 |
+
mm_token_type_ids = (input_ids == self.image_token_id).to(
|
| 919 |
+
dtype=torch.long, device=input_ids.device
|
| 920 |
+
)
|
| 921 |
+
backbone_kwargs["mm_token_type_ids"] = mm_token_type_ids
|
| 922 |
+
|
| 923 |
+
grad_ctx = torch.no_grad() if self.freeze_backbone else torch.enable_grad()
|
| 924 |
+
with grad_ctx:
|
| 925 |
+
outputs = self.backbone(**backbone_kwargs)
|
| 926 |
+
last_hidden_states = outputs.last_hidden_state
|
| 927 |
+
|
| 928 |
+
merged_vis = self._captured_visual.get("merged")
|
| 929 |
+
unmerged_vis = self._captured_visual.get("unmerged")
|
| 930 |
+
if merged_vis is None:
|
| 931 |
+
raise RuntimeError("Visual forward hook did not capture features; check pixel_values input.")
|
| 932 |
+
|
| 933 |
+
coarse_grids, fine_grids = self._reconstruct_2d_vision_features(
|
| 934 |
+
merged_tokens=merged_vis,
|
| 935 |
+
unmerged_tokens=unmerged_vis,
|
| 936 |
+
image_grid_thw=image_grid_thw,
|
| 937 |
+
grid_thw_list=resolved_grid_thw,
|
| 938 |
+
)
|
| 939 |
+
llm_vis_grids = self._extract_llm_visual_grids(
|
| 940 |
+
last_hidden_states=last_hidden_states,
|
| 941 |
+
input_ids=input_ids,
|
| 942 |
+
image_grid_thw=image_grid_thw,
|
| 943 |
+
layer_states=self._captured_llm_layers,
|
| 944 |
+
grid_thw_list=resolved_grid_thw,
|
| 945 |
+
input_ids_cpu=input_ids_cpu,
|
| 946 |
+
)
|
| 947 |
+
batch_queries = self._extract_query_vectors(
|
| 948 |
+
last_hidden_states=last_hidden_states,
|
| 949 |
+
input_ids=input_ids,
|
| 950 |
+
layer_states=self._captured_llm_layers,
|
| 951 |
+
input_ids_cpu=input_ids_cpu,
|
| 952 |
+
)
|
| 953 |
+
|
| 954 |
+
head_param = next(self.mask_head.parameters())
|
| 955 |
+
head_dtype = head_param.dtype
|
| 956 |
+
head_device = head_param.device
|
| 957 |
+
mask_logits_list: List[torch.Tensor] = []
|
| 958 |
+
mask_probs_list: List[torch.Tensor] = []
|
| 959 |
+
pred_scores_list: List[torch.Tensor] = []
|
| 960 |
+
|
| 961 |
+
for b in range(len(batch_queries)):
|
| 962 |
+
if target_sizes is not None and b < len(target_sizes):
|
| 963 |
+
tgt_size = (int(target_sizes[b][0]), int(target_sizes[b][1]))
|
| 964 |
+
else:
|
| 965 |
+
tgt_size = (
|
| 966 |
+
int(resolved_grid_thw[b][1] * self.patch_size),
|
| 967 |
+
int(resolved_grid_thw[b][2] * self.patch_size),
|
| 968 |
+
)
|
| 969 |
+
|
| 970 |
+
f_llm_b = (
|
| 971 |
+
llm_vis_grids[b].to(device=head_device, dtype=head_dtype)
|
| 972 |
+
if llm_vis_grids[b] is not None
|
| 973 |
+
else None
|
| 974 |
+
)
|
| 975 |
+
logits_b, scores_b = self.mask_head.forward_single_image(
|
| 976 |
+
queries=batch_queries[b].to(device=head_device, dtype=head_dtype),
|
| 977 |
+
f_coarse=coarse_grids[b].to(device=head_device, dtype=head_dtype),
|
| 978 |
+
f_fine=fine_grids[b].to(device=head_device, dtype=head_dtype),
|
| 979 |
+
target_size=tgt_size,
|
| 980 |
+
f_llm_coarse=f_llm_b,
|
| 981 |
+
)
|
| 982 |
+
mask_logits_list.append(logits_b)
|
| 983 |
+
mask_probs_list.append(torch.sigmoid(logits_b))
|
| 984 |
+
pred_scores_list.append(scores_b)
|
| 985 |
+
|
| 986 |
+
return {
|
| 987 |
+
"mask_logits": mask_logits_list,
|
| 988 |
+
"mask_probs": mask_probs_list,
|
| 989 |
+
"pred_scores": pred_scores_list,
|
| 990 |
+
}
|
| 991 |
+
|
| 992 |
+
@torch.no_grad()
|
| 993 |
+
def predict_json(
|
| 994 |
+
self,
|
| 995 |
+
image: Any,
|
| 996 |
+
query_json: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
|
| 997 |
+
threshold: float = 0.5,
|
| 998 |
+
) -> Dict[str, Any]:
|
| 999 |
+
"""
|
| 1000 |
+
End-to-end System 1 inference:
|
| 1001 |
+
Takes a PIL image + JSON object descriptions, runs 1 forward pass, and returns
|
| 1002 |
+
a JSON dict with COCO polygon segmentations (`segmentation`), `bbox`, `area`, and `score`.
|
| 1003 |
+
"""
|
| 1004 |
+
if self.processor is None:
|
| 1005 |
+
raise RuntimeError("`processor` is required for `predict_json`.")
|
| 1006 |
+
|
| 1007 |
+
self.eval()
|
| 1008 |
+
orig_w, orig_h = image.size
|
| 1009 |
+
messages, objects = build_segmentation_messages(image=image, query_input=query_json)
|
| 1010 |
+
|
| 1011 |
+
formatted_text = self.processor.apply_chat_template(
|
| 1012 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 1013 |
+
)
|
| 1014 |
+
inputs = self.processor(
|
| 1015 |
+
text=[formatted_text],
|
| 1016 |
+
images=[image],
|
| 1017 |
+
padding=True,
|
| 1018 |
+
return_tensors="pt",
|
| 1019 |
+
)
|
| 1020 |
+
input_ids_cpu = inputs["input_ids"]
|
| 1021 |
+
grid_thw_list = inputs["image_grid_thw"].tolist()
|
| 1022 |
+
device = next(self.mask_head.parameters()).device
|
| 1023 |
+
inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()}
|
| 1024 |
+
|
| 1025 |
+
outputs = self.forward(
|
| 1026 |
+
input_ids=inputs["input_ids"],
|
| 1027 |
+
attention_mask=inputs["attention_mask"],
|
| 1028 |
+
pixel_values=inputs["pixel_values"],
|
| 1029 |
+
image_grid_thw=inputs["image_grid_thw"],
|
| 1030 |
+
target_sizes=[(orig_h, orig_w)],
|
| 1031 |
+
grid_thw_list=grid_thw_list,
|
| 1032 |
+
input_ids_cpu=input_ids_cpu,
|
| 1033 |
+
)
|
| 1034 |
+
|
| 1035 |
+
probs = outputs["mask_probs"][0] # [K, orig_H, orig_W]
|
| 1036 |
+
scores = outputs["pred_scores"][0] # [K]
|
| 1037 |
+
|
| 1038 |
+
results = []
|
| 1039 |
+
for idx, obj in enumerate(objects):
|
| 1040 |
+
if idx >= probs.shape[0]:
|
| 1041 |
+
break
|
| 1042 |
+
prob_map = probs[idx]
|
| 1043 |
+
binary_mask = (prob_map >= threshold).to(torch.uint8)
|
| 1044 |
+
polygons, bbox, area = mask_to_polygons(binary_mask)
|
| 1045 |
+
|
| 1046 |
+
# Combine IoU head score with mean foreground probability
|
| 1047 |
+
fg_prob = float(prob_map[binary_mask > 0].mean().item()) if area > 0 else 0.0
|
| 1048 |
+
score = round(0.5 * float(scores[idx].item()) + 0.5 * fg_prob, 4)
|
| 1049 |
+
|
| 1050 |
+
results.append(
|
| 1051 |
+
{
|
| 1052 |
+
"id": obj["id"],
|
| 1053 |
+
"description": obj["description"],
|
| 1054 |
+
"score": score,
|
| 1055 |
+
"bbox": bbox,
|
| 1056 |
+
"area": area,
|
| 1057 |
+
"segmentation": polygons,
|
| 1058 |
+
}
|
| 1059 |
+
)
|
| 1060 |
+
|
| 1061 |
+
return {
|
| 1062 |
+
"image_size": {"height": orig_h, "width": orig_w},
|
| 1063 |
+
"objects": results,
|
| 1064 |
+
}
|
| 1065 |
+
|
| 1066 |
+
|
| 1067 |
+
class SegmentationLoss(nn.Module):
|
| 1068 |
+
"""
|
| 1069 |
+
Combined Focal + Soft Dice + BCE + IoU Score Calibration loss for training the mask head.
|
| 1070 |
+
"""
|
| 1071 |
+
|
| 1072 |
+
def __init__(
|
| 1073 |
+
self,
|
| 1074 |
+
bce_weight: float = 1.0,
|
| 1075 |
+
focal_weight: float = 2.0,
|
| 1076 |
+
dice_weight: float = 2.0,
|
| 1077 |
+
score_weight: float = 0.5,
|
| 1078 |
+
focal_alpha: float = 0.25,
|
| 1079 |
+
focal_gamma: float = 2.0,
|
| 1080 |
+
):
|
| 1081 |
+
super().__init__()
|
| 1082 |
+
self.bce_weight = bce_weight
|
| 1083 |
+
self.focal_weight = focal_weight
|
| 1084 |
+
self.dice_weight = dice_weight
|
| 1085 |
+
self.score_weight = score_weight
|
| 1086 |
+
self.focal_alpha = focal_alpha
|
| 1087 |
+
self.focal_gamma = focal_gamma
|
| 1088 |
+
|
| 1089 |
+
def _dice_loss(self, logits: torch.Tensor, targets: torch.Tensor, eps: float = 1.0) -> torch.Tensor:
|
| 1090 |
+
probs = torch.sigmoid(logits).flatten(1)
|
| 1091 |
+
tgt = targets.flatten(1)
|
| 1092 |
+
intersection = (probs * tgt).sum(dim=1)
|
| 1093 |
+
union = probs.sum(dim=1) + tgt.sum(dim=1)
|
| 1094 |
+
dice = (2.0 * intersection + eps) / (union + eps)
|
| 1095 |
+
return (1.0 - dice).mean()
|
| 1096 |
+
|
| 1097 |
+
def _sigmoid_focal_loss(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
|
| 1098 |
+
bce = F.binary_cross_entropy_with_logits(logits, targets, reduction="none")
|
| 1099 |
+
probs = torch.sigmoid(logits)
|
| 1100 |
+
p_t = probs * targets + (1.0 - probs) * (1.0 - targets)
|
| 1101 |
+
alpha_t = self.focal_alpha * targets + (1.0 - self.focal_alpha) * (1.0 - targets)
|
| 1102 |
+
focal = alpha_t * ((1.0 - p_t) ** self.focal_gamma) * bce
|
| 1103 |
+
return focal.mean()
|
| 1104 |
+
|
| 1105 |
+
def forward(
|
| 1106 |
+
self,
|
| 1107 |
+
mask_logits_list: Sequence[torch.Tensor],
|
| 1108 |
+
pred_scores_list: Sequence[torch.Tensor],
|
| 1109 |
+
gt_masks_list: Sequence[torch.Tensor],
|
| 1110 |
+
) -> Dict[str, torch.Tensor]:
|
| 1111 |
+
total_bce = torch.tensor(0.0, device=mask_logits_list[0].device)
|
| 1112 |
+
total_focal = torch.tensor(0.0, device=mask_logits_list[0].device)
|
| 1113 |
+
total_dice = torch.tensor(0.0, device=mask_logits_list[0].device)
|
| 1114 |
+
total_score = torch.tensor(0.0, device=mask_logits_list[0].device)
|
| 1115 |
+
|
| 1116 |
+
n_batches = max(len(mask_logits_list), 1)
|
| 1117 |
+
for logits, scores, gt in zip(mask_logits_list, pred_scores_list, gt_masks_list):
|
| 1118 |
+
gt = gt.to(device=logits.device, dtype=logits.dtype)
|
| 1119 |
+
if gt.shape[-2:] != logits.shape[-2:]:
|
| 1120 |
+
gt = F.interpolate(gt.unsqueeze(1), size=logits.shape[-2:], mode="nearest").squeeze(1)
|
| 1121 |
+
|
| 1122 |
+
total_bce = total_bce + F.binary_cross_entropy_with_logits(logits, gt)
|
| 1123 |
+
total_focal = total_focal + self._sigmoid_focal_loss(logits, gt)
|
| 1124 |
+
total_dice = total_dice + self._dice_loss(logits, gt)
|
| 1125 |
+
|
| 1126 |
+
with torch.no_grad():
|
| 1127 |
+
pred_bin = (torch.sigmoid(logits) >= 0.5).to(gt.dtype).flatten(1)
|
| 1128 |
+
gt_flat = gt.flatten(1)
|
| 1129 |
+
inter = (pred_bin * gt_flat).sum(dim=1)
|
| 1130 |
+
union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1).clamp(min=1e-6)
|
| 1131 |
+
true_iou = inter / union
|
| 1132 |
+
|
| 1133 |
+
total_score = total_score + F.mse_loss(scores, true_iou.to(scores.dtype))
|
| 1134 |
+
|
| 1135 |
+
total_bce = total_bce / n_batches
|
| 1136 |
+
total_focal = total_focal / n_batches
|
| 1137 |
+
total_dice = total_dice / n_batches
|
| 1138 |
+
total_score = total_score / n_batches
|
| 1139 |
+
|
| 1140 |
+
loss = (
|
| 1141 |
+
self.bce_weight * total_bce
|
| 1142 |
+
+ self.focal_weight * total_focal
|
| 1143 |
+
+ self.dice_weight * total_dice
|
| 1144 |
+
+ self.score_weight * total_score
|
| 1145 |
+
)
|
| 1146 |
+
return {
|
| 1147 |
+
"loss": loss,
|
| 1148 |
+
"bce_loss": total_bce.detach(),
|
| 1149 |
+
"focal_loss": total_focal.detach(),
|
| 1150 |
+
"dice_loss": total_dice.detach(),
|
| 1151 |
+
"score_loss": total_score.detach(),
|
| 1152 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
datasets:
|
| 4 |
+
- Wojtekb30/ref_coco_lichengunc
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
pipeline_tag: mask-generation
|
| 8 |
+
tags:
|
| 9 |
+
- mask-generation
|
| 10 |
+
- image-segmentation
|
| 11 |
+
- System1
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
<p align="center">
|
| 15 |
+
<img src="logo.png" alt="LIS1 Logo" width="320"/>
|
| 16 |
+
</p>
|
| 17 |
+
|
| 18 |
+
<h1 align="center">LIS1: Single-Pass Visual Reasoning & Referring Image Segmentation</h1>
|
| 19 |
+
|
| 20 |
+
**LIS1** (Linguistic Image Segmenter) is a fast, non-autoregressive visual reasoning and referring image segmentation model family built on **Qwen3.5** (`0.8B`, `2B`, `4B`, `9B`, and `27B`).
|
| 21 |
+
|
| 22 |
+
Unlike traditional reasoning segmentation models (such as LISA) that generate text tokens autoregressively and rely on a separate heavyweight SAM backbone, **LIS1** performs both complex commonsense/riddle reasoning and crisp pixel-level mask decoding in a **single forward pass** using Qwen3.5's native Vision Transformer and LLM hidden states—with **zero autoregressive token generation** and **no external SAM encoder**.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Example Output
|
| 27 |
+
|
| 28 |
+
Below is a single-image inference example (`image1.jpg` to `result_0_boxes_and_masks.png`) generated by **LIS1-0.8B** in 4-bit quantization (`example.py`) for three simultaneous queries combining direct object names and functional riddles:
|
| 29 |
+
- `"toast"` (Red — score `0.75`)
|
| 30 |
+
- `"red sauce"` (Green — score `0.80`)
|
| 31 |
+
- `"something you can drink"` (Blue — score `0.88`)
|
| 32 |
+
|
| 33 |
+
| Input Image (`image1.jpg`) | Rendered Output (`result_0_boxes_and_masks.png`) |
|
| 34 |
+
| :---: | :---: |
|
| 35 |
+
| <img src="image1.jpg" alt="Input Image" width="460"/> | <img src="result_0_boxes_and_masks.png" alt="Segmented Output" width="460"/> |
|
| 36 |
+
|
| 37 |
+
---
|
| 38 |
+
|
| 39 |
+
## ReasonSeg Benchmark Results
|
| 40 |
+
|
| 41 |
+
All **LIS1** models were trained **exclusively on the [RefCOCO dataset by lichengunc](https://huggingface.co/datasets/Wojtekb30/ref_coco_lichengunc)**, which includes RefCOCO, RefCOCO+, and RefCOCOg (with 35% synthetic functional riddle augmentation). They were evaluated **zero-shot on the ReasonSeg benchmark**, without seeing ReasonSeg training images, using the [ReasonSeg validation split](https://huggingface.co/datasets/Ricky06662/ReasonSeg_val) and [ReasonSeg test split](https://huggingface.co/datasets/Ricky06662/ReasonSeg_test).
|
| 42 |
+
|
| 43 |
+
Each repository provides two checkpoint variants (in both `.safetensors` and `.pt` formats):
|
| 44 |
+
- **`noLora` (Stage 1)**: Trained mask head with the base Qwen3.5 model kept fully frozen.
|
| 45 |
+
- **`forLora` (Stage 2)**: Trained mask head + lightweight PEFT LoRA adapter (`r=8`, `alpha=16`) applied only to the top 4 LLM layers.
|
| 46 |
+
|
| 47 |
+
| Method | Val overall gIoU | Val overall cIoU | Test short query gIoU | Test short query cIoU | Test long query gIoU | Test long query cIoU | Test overall gIoU | Test overall cIoU |
|
| 48 |
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 49 |
+
| OVSeg (Liang et al., 2023) | 28.5 | 18.6 | 18.0 | 15.5 | 28.7 | 22.5 | 26.1 | 20.8 |
|
| 50 |
+
| GRES (Liu et al., 2023b) | 22.4 | 19.9 | 17.6 | 15.0 | 22.6 | 23.8 | 21.3 | 22.0 |
|
| 51 |
+
| X-Decoder (Zou et al., 2023) | 22.6 | 17.9 | 20.4 | 11.6 | 22.2 | 17.5 | 21.7 | 16.3 |
|
| 52 |
+
| SEEM (Zou et al., 2023b) | 25.5 | 21.2 | 20.1 | 11.5 | 25.6 | 20.8 | 24.3 | 18.7 |
|
| 53 |
+
| LISA-7B | 44.4 | 46.0 | 37.6 | 34.4 | 36.6 | 34.7 | 36.8 | 34.1 |
|
| 54 |
+
| LISA-7B (ft) | 52.9 | 54.0 | 40.6 | 40.6 | 49.4 | 51.0 | 47.3 | 48.4 |
|
| 55 |
+
| LISA-13B | 48.9 | 46.9 | 39.9 | 43.3 | 46.4 | 46.5 | 44.8 | 45.8 |
|
| 56 |
+
| LISA-13B (ft) | 56.2 | 62.9 | 44.3 | 42.0 | 54.0 | 54.3 | 51.7 | 51.1 |
|
| 57 |
+
| LISA-Llama2-13B (ft) | 60.0 | 67.8 | 43.9 | 45.8 | 54.0 | 53.8 | 51.5 | 51.3 |
|
| 58 |
+
| LIS1-0.8B noLora | 31.7 | 32.5 | 28.6 | 23.7 | 27.3 | 30.6 | 27.5 | 29.6 |
|
| 59 |
+
| LIS1-0.8B forLora | 32.4 | 31.7 | 30.4 | 23.9 | 29.1 | 29.6 | 29.4 | 28.6 |
|
| 60 |
+
| LIS1-2B noLora | 37.4 | 33.1 | 35.1 | 27.2 | 34.5 | 33.6 | 34.6 | 32.6 |
|
| 61 |
+
| LIS1-2B forLora | 37.5 | 33.1 | 35.5 | 27.1 | 34.6 | 34.3 | 34.8 | 33.1 |
|
| 62 |
+
| LIS1-4B noLora | 42.3 | 38.3 | 39.9 | 29.9 | 43.5 | 40.5 | 42.8 | 38.7 |
|
| 63 |
+
| LIS1-4B forLora | 43.5 | 39.6 | 40.1 | 29.7 | 42.9 | 39.2 | 42.4 | 37.7 |
|
| 64 |
+
| LIS1-9B noLora | 45.2 | 38.3 | 43.1 | 35.7 | 42.7 | 37.3 | 42.8 | 37.0 |
|
| 65 |
+
| LIS1-9B forLora | 45.1 | 40.0 | 40.8 | 33.3 | 44.7 | 40.9 | 43.9 | 39.6 |
|
| 66 |
+
| LIS1-27B noLora | 54.1 | 54.3 | 46.9 | 44.0 | 49.9 | 49.8 | 49.3 | 49.0 |
|
| 67 |
+
| LIS1-27B forLora | 54.0 | 54.8 | 47.1 | 44.6 | 50.7 | 51.4 | 50.0 | 50.4 |
|
| 68 |
+
|
| 69 |
+
Other results come from [evaluation done by the team behind LISA](https://github.com/JIA-Lab-research/LISA/blob/main/imgs/table1.jpg)
|
| 70 |
+
|
| 71 |
+
LIS1 4B and bigger had better results than Ultralytics MobileSAM and YOLOE Open Vocabulary in iternal evaluation.
|
| 72 |
+
|
| 73 |
+
---
|
| 74 |
+
|
| 75 |
+
## Installation & Quick Start
|
| 76 |
+
|
| 77 |
+
### 1. Install Dependencies
|
| 78 |
+
|
| 79 |
+
```bash
|
| 80 |
+
pip install transformers peft safetensors pillow opencv-python numpy pyarrow torch torchvision
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
*(Optional)* For 4-bit or 8-bit quantization and faster Qwen3.5 linear-attention kernels on CUDA GPUs:
|
| 84 |
+
|
| 85 |
+
```bash
|
| 86 |
+
pip install bitsandbytes accelerate causal-conv1d flash-linear-attention
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
### 2. Minimal Inference Code
|
| 90 |
+
|
| 91 |
+
See **`example.py`** for the complete minimal script that produces the rendered image above. Make sure `LIS1ModelCode.py` and `lis1_infer.py` are in your working directory:
|
| 92 |
+
|
| 93 |
+
```python
|
| 94 |
+
from PIL import Image
|
| 95 |
+
from lis1_infer import LIS1Inferencer
|
| 96 |
+
|
| 97 |
+
# 1. Initialize the inferencer
|
| 98 |
+
inferencer = LIS1Inferencer(debug=True)
|
| 99 |
+
|
| 100 |
+
# 2. Load the base Qwen3.5 model, LIS1 mask head (.safetensors or .pt), and optional LoRA adapter
|
| 101 |
+
# Swap "0.8B" with "2B", "4B", "9B", or "27B" depending on the model size you use.
|
| 102 |
+
inferencer.load_model(
|
| 103 |
+
qwen_path="./Qwen3.5-9B",
|
| 104 |
+
lis_model_path="./LIS1-9B-forLora.safetensors",
|
| 105 |
+
lora_adapter="./LIS1-9B-LoraAdapter", # Pass None if using a *-noLora checkpoint
|
| 106 |
+
device="cuda",
|
| 107 |
+
max_pixels=524288,
|
| 108 |
+
load_in_4bit=True, # Set False for full bfloat16, or load_in_8bit=True for 8-bit
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
# 3. Prepare images and natural language / riddle queries
|
| 112 |
+
images = [Image.open("image1.jpg").convert("RGB")]
|
| 113 |
+
queries = [["toast", "red sauce", "something you can drink"]]
|
| 114 |
+
|
| 115 |
+
# 4. Run single-pass segmentation
|
| 116 |
+
outputs = inferencer.run_model(
|
| 117 |
+
images=images,
|
| 118 |
+
queries=queries,
|
| 119 |
+
threshold=0.7,
|
| 120 |
+
isolate_queries=True,
|
| 121 |
+
cache_vision=True,
|
| 122 |
+
exclusive_masks=True,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
# 5. Render masks, contours, bounding boxes, and non-overlapping labels
|
| 126 |
+
rendered = inferencer.render_image(images=images, outputs=outputs)
|
| 127 |
+
rendered[0].save("result_0_boxes_and_masks.png")
|
| 128 |
+
|
| 129 |
+
# 6. Free GPU/CPU memory when finished
|
| 130 |
+
inferencer.offload_model()
|
| 131 |
+
```
|
| 132 |
+
|
| 133 |
+
---
|
| 134 |
+
|
| 135 |
+
## How the LIS1 Architecture Works
|
| 136 |
+
|
| 137 |
+
LIS1 turns a pretrained Qwen3.5 vision-language model into a single-pass, multi-object pixel segmenter through six sequential stages:
|
| 138 |
+
|
| 139 |
+
### Step 1: Causal Riddle-Before-Image Prompt Construction
|
| 140 |
+
In causal language models, a token can only attend to tokens that appear earlier in the sequence. LIS1 formats the multimodal chat messages in a specific causal order:
|
| 141 |
+
1. **System Prompt**: Instructs the model to act as a single-pass visual reasoning and segmentation model.
|
| 142 |
+
2. **User Turn (JSON Queries First, Image Second)**: The target descriptions or riddles are serialized into a JSON block *before* the `<|image_pad|>` visual tokens. Because the text precedes the image, every visual patch token inside the LLM can attend directly to the user's riddles across all 24–64 layers.
|
| 143 |
+
3. **Prefilled Assistant Turn**: Instead of generating tokens one by one, LIS1 prefills the assistant response with a JSON structure wrapping each query inside Qwen's native grounding delimiters: `<|object_ref_start|>{description}<|object_ref_end|>`. Crossing into the assistant turn and reading `<|object_ref_end|>` activates Qwen's internal spatial localization circuit in a single forward pass.
|
| 144 |
+
|
| 145 |
+
### Step 2: Single-Pass Dual-Scale Vision Extraction (With LRU Caching)
|
| 146 |
+
When an image enters Qwen3.5's Vision Transformer (`backbone.visual`), a forward hook captures two spatial feature representations from the same pass without running an external segmentation backbone:
|
| 147 |
+
- **Fine Unmerged ViT Features (`f_fine`)**: Captured at 1/16 resolution before spatial patch merging, preserving sharp object edges and thin structures.
|
| 148 |
+
- **Coarse Merged ViT Features (`f_coarse`)**: Captured at 1/32 resolution after 2x2 spatial merging (`spatial_merge_size = 2`).
|
| 149 |
+
- **Vision Caching**: When segmenting multiple isolated queries on the same image (`isolate_queries=True, cache_vision=True`), LIS1 runs the ViT encoder only **once**, caches the reconstructed 2D spatial grids (`f_coarse`, `f_fine`) and merged visual embeddings in an LRU cache, and injects the cached embeddings directly into the LLM token stream via `masked_scatter`.
|
| 150 |
+
|
| 151 |
+
### Step 3: LLM Forward Pass & Multi-Layer Feature Readout
|
| 152 |
+
The sequence of text and visual tokens passes once through the Qwen3.5 hybrid backbone (which interleaves Gated DeltaNet linear-attention layers with full self-attention layers; the `lm_head` is dropped completely). Forward hooks on the **last 4 decoder layers** extract two sets of representations:
|
| 153 |
+
- **Riddle-Conditioned Visual Grid (`f_llm_coarse`)**: The hidden states at all `<|image_pad|>` positions are averaged across the top 4 layers and reshaped back into a 2D spatial grid that now carries full multimodal reasoning context.
|
| 154 |
+
- **Multi-Layer Object Decision Vectors (`queries`)**: For each of the K target objects, the hidden states at the `<|object_ref_end|>` token position are extracted from the last 4 layers (`[K, 4, llm_dim]`).
|
| 155 |
+
|
| 156 |
+
*(Note on Quantization)*: When 4-bit or 8-bit `bitsandbytes` quantization is enabled, LIS1 automatically excludes the ViT encoder (`visual`) and Qwen3.5's tiny Gated DeltaNet decay gates (`in_proj_a`, `in_proj_b`) from quantization so spatial accuracy and linear-attention stability are preserved.
|
| 157 |
+
|
| 158 |
+
### Step 4: Cross-Attention & Dynamic Feature Modulation
|
| 159 |
+
Inside `MultiScaleMaskHead` (kept in `float32` for crisp boundary decoding):
|
| 160 |
+
1. **Layer Fusion**: The 4 captured LLM layers for each query vector are normalized via per-layer `LayerNorm` and combined using learned softmax weights into a single decision vector per object (`[K, llm_dim]`).
|
| 161 |
+
2. **Visual Context Fusion**: The coarse ViT grid (`f_coarse`) and the LLM-contextualized visual grid (`f_llm_coarse`) are projected to `hidden_dim = 256` using 1x1 `Conv2d + GroupNorm + GELU` blocks and combined via a learned `tanh`-gated residual connection.
|
| 162 |
+
3. **2D Cross-Attention**: The K object queries are projected to `256` dimensions and refined via 4-head cross-attention over the coarse 2D visual tokens, which are augmented with parameter-free 2D sine-cosine positional embeddings.
|
| 163 |
+
4. **Calibrated IoU Score Prediction**: A 2-layer MLP (`score_head`) reads the refined query vector to predict an intrinsic mask confidence / IoU score in [0, 1].
|
| 164 |
+
|
| 165 |
+
### Step 5: Coarse-to-Fine Spatial Mask Decoding
|
| 166 |
+
All K object masks are decoded in parallel through a three-stage hierarchy:
|
| 167 |
+
1. **Coarse Stage (1/32 Scale)**: Each query computes a spatial dot-product cosine similarity map against both the ViT coarse features and the LLM visual features. Simultaneously, the query generates a channel-wise sigmoid gate to modulate the 2D coarse feature map. The modulated features and similarity map (`256 + 1` channels) are fused with a 3x3 convolution and a residual refinement block.
|
| 168 |
+
2. **Fine Skip-Connection Stage (1/16 Scale)**: The coarse features are bilinearly upsampled 2x and concatenated with the fine unmerged ViT skip features (`f_fine`, projected to `128` channels and gated by the query vector). Two consecutive 3x3 residual refinement blocks sharpen the object boundaries.
|
| 169 |
+
3. **Full-Resolution Output Stage**: A learned 2x transposed convolution (`ConvTranspose2d` to 1/8 scale) followed by a 1x1 convolution produces mask logits, which are bilinearly interpolated to the exact original image dimensions `(H, W)`.
|
| 170 |
+
|
| 171 |
+
### Step 6: Post-Processing & Two-Stage Training
|
| 172 |
+
- **Inference Post-Processing**: Pixel probabilities are thresholded, cleaned with connected-component filtering (`filter_mask_components` to drop tiny stray noise islands while preserving small single-component objects), optionally disambiguated across queries via pixel-wise winner-takes-all (`exclusive_masks=True`), and converted into both binary masks and COCO polygons/bounding boxes. The final confidence score blends the IoU score-head output (50%) with the mean foreground pixel probability (50%).
|
| 173 |
+
- **Two-Stage Training**:
|
| 174 |
+
- **Stage 1 (`noLora`)**: Freezes all of the Qwen3.5 backbone and trains only `MultiScaleMaskHead` for 5 epochs using a combination of Binary Cross-Entropy (1.0), Sigmoid Focal Loss (2.0), Soft Dice Loss (2.0), and MSE IoU Score Calibration Loss (0.5).
|
| 175 |
+
- **Stage 2 (`forLora`)**: Keeps the first 85-95% of the LLM layers completely frozen (preserving pretrained world knowledge and riddle-solving capabilities) and attaches low-rank LoRA adapters (`r=8`) **only to the top 4 LLM layers** (`q_proj, k_proj, v_proj, o_proj, in_proj_qkv, out_proj`) for 1–3 epochs of joint fine-tuning.
|
| 176 |
+
|
| 177 |
+
---
|
| 178 |
+
|
| 179 |
+
## Training and Benchmark Commands
|
| 180 |
+
|
| 181 |
+
The commands below assume the project scripts are in the current directory and the datasets are available locally. Replace `{SIZE}` with a Qwen3.5 model size such as `0.8B`, `2B`, `4B`, `9B`, or `27B`; update the model and dataset paths for your setup.
|
| 182 |
+
|
| 183 |
+
### Stage 1: Train the segmentation head
|
| 184 |
+
|
| 185 |
+
```bash
|
| 186 |
+
python3 train.py \
|
| 187 |
+
--stage 1 \
|
| 188 |
+
--model-path ./Qwen3.5-{SIZE} \
|
| 189 |
+
--data-root ./ref_coco_lichengunc \
|
| 190 |
+
--dataset all \
|
| 191 |
+
--riddle-prob 0.35 \
|
| 192 |
+
--epochs 5 \
|
| 193 |
+
--batch-size 4 \
|
| 194 |
+
--max-objects 4 \
|
| 195 |
+
--lr-head 2e-4 \
|
| 196 |
+
--output-dir ./checkpoints_{SIZE}
|
| 197 |
+
```
|
| 198 |
+
|
| 199 |
+
### Stage 2: Fine-tune with LoRA
|
| 200 |
+
|
| 201 |
+
Resume from the Stage 1 checkpoint you want to continue from (the example uses epoch 5):
|
| 202 |
+
|
| 203 |
+
```bash
|
| 204 |
+
python3 train.py \
|
| 205 |
+
--stage 2 \
|
| 206 |
+
--model-path ./Qwen3.5-{SIZE} \
|
| 207 |
+
--data-root ./ref_coco_lichengunc \
|
| 208 |
+
--dataset all \
|
| 209 |
+
--riddle-prob 0.35 \
|
| 210 |
+
--resume-ckpt ./checkpoints_{SIZE}/segmenter_stage1_epoch5.pt \
|
| 211 |
+
--lora-top-layers 4 \
|
| 212 |
+
--lora-r 8 \
|
| 213 |
+
--lora-alpha 16 \
|
| 214 |
+
--epochs 3 \
|
| 215 |
+
--batch-size 4 \
|
| 216 |
+
--lr-head 1e-4 \
|
| 217 |
+
--lr-lora 2e-5 \
|
| 218 |
+
--output-dir ./checkpoints_{SIZE}
|
| 219 |
+
```
|
| 220 |
+
|
| 221 |
+
### Evaluate on ReasonSeg
|
| 222 |
+
|
| 223 |
+
```bash
|
| 224 |
+
python3 benchmark_reasonseg.py \
|
| 225 |
+
--model-path ./Qwen3.5-{SIZE} \
|
| 226 |
+
--data-dir ./ReasonSeg_val \
|
| 227 |
+
--test-dir ./ReasonSeg_test \
|
| 228 |
+
--resume-ckpt ./checkpoints_{SIZE}/segmenter_stage2_epoch3.pt \
|
| 229 |
+
--method-name "LIS1-{SIZE} (Stage 2)" \
|
| 230 |
+
--out-md ./reasonseg_benchmark-{SIZE}.md
|
| 231 |
+
```
|
| 232 |
+
|
| 233 |
+
Use the Stage 1 checkpoint path instead to benchmark a `noLora` model. Stage 2 checkpoints require the corresponding LoRA adapter directory saved alongside the checkpoint; the path recorded in the checkpoint must resolve when running the benchmark. `train.py` defaults to CUDA when available, while the benchmark command can be run on CPU by adding `--device cpu`.
|
benchmark_reasonseg.py
ADDED
|
@@ -0,0 +1,791 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
from collections import deque
|
| 3 |
+
from concurrent.futures import Future, ThreadPoolExecutor
|
| 4 |
+
import copy
|
| 5 |
+
import io
|
| 6 |
+
import json
|
| 7 |
+
import pathlib
|
| 8 |
+
import threading
|
| 9 |
+
import time
|
| 10 |
+
from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
|
| 11 |
+
|
| 12 |
+
import cv2
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
from PIL import Image
|
| 17 |
+
|
| 18 |
+
from LIS1ModelCode import (
|
| 19 |
+
DEFAULT_SYSTEM_PROMPT,
|
| 20 |
+
OBJECT_QUERY_TOKEN,
|
| 21 |
+
Qwen2_5_VL_SinglePassSegmenter,
|
| 22 |
+
build_segmentation_messages,
|
| 23 |
+
)
|
| 24 |
+
from train import load_checkpoint
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
BASELINE_ROWS = [
|
| 28 |
+
("OVSeg (Liang et al., 2022)", 28.5, 18.6, 18.0, 15.5, 28.7, 22.5, 26.1, 20.8),
|
| 29 |
+
("GRES (Liu et al., 2023b)", 22.4, 19.9, 17.6, 15.0, 22.6, 23.8, 21.3, 22.0),
|
| 30 |
+
("X-Decoder (Zou et al., 2023)", 22.6, 17.9, 20.4, 11.6, 22.2, 17.5, 21.7, 16.3),
|
| 31 |
+
("SEEM (Zou et al., 2023b)", 25.5, 21.2, 20.1, 11.5, 25.6, 20.8, 24.3, 18.7),
|
| 32 |
+
("LISA-7B", 44.4, 46.0, 37.6, 34.4, 36.6, 34.7, 36.8, 34.1),
|
| 33 |
+
("LISA-7B (ft)", 52.9, 54.0, 40.6, 40.6, 49.4, 51.0, 47.3, 48.4),
|
| 34 |
+
("LISA-13B", 48.9, 46.9, 39.9, 43.3, 46.4, 46.5, 44.8, 45.8),
|
| 35 |
+
("LISA-13B (ft)", 56.2, 62.9, 44.3, 42.0, 54.0, 54.3, 51.7, 51.1),
|
| 36 |
+
("LISA-Llama2-13B (ft)", 60.0, 67.8, 43.9, 45.8, 54.0, 53.8, 51.5, 51.3),
|
| 37 |
+
]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def is_long_query(text: str, explicit_flag: Optional[bool] = None) -> bool:
|
| 41 |
+
"""
|
| 42 |
+
Determines whether a ReasonSeg query belongs to the 'long query' (sentence/riddle)
|
| 43 |
+
or 'short query' (phrase) subset.
|
| 44 |
+
Uses `is_sentence` if present in raw ReasonSeg JSON; otherwise checks sentence structure.
|
| 45 |
+
"""
|
| 46 |
+
if explicit_flag is not None:
|
| 47 |
+
return bool(explicit_flag)
|
| 48 |
+
cleaned = text.strip()
|
| 49 |
+
if "?" in cleaned or cleaned.endswith(".") or len(cleaned.split()) >= 10:
|
| 50 |
+
return True
|
| 51 |
+
return False
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _decode_parquet_image(img_entry: Any) -> Image.Image:
|
| 55 |
+
if isinstance(img_entry, dict) and img_entry.get("bytes"):
|
| 56 |
+
return Image.open(io.BytesIO(img_entry["bytes"])).convert("RGB")
|
| 57 |
+
if isinstance(img_entry, dict) and img_entry.get("path"):
|
| 58 |
+
return Image.open(img_entry["path"]).convert("RGB")
|
| 59 |
+
if isinstance(img_entry, (bytes, bytearray)):
|
| 60 |
+
return Image.open(io.BytesIO(img_entry)).convert("RGB")
|
| 61 |
+
if isinstance(img_entry, Image.Image):
|
| 62 |
+
return img_entry.convert("RGB")
|
| 63 |
+
raise ValueError(f"Unsupported parquet image entry format: {type(img_entry)}")
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _unpack_parquet_mask_slice(mask_slice: Any) -> np.ndarray:
|
| 67 |
+
"""
|
| 68 |
+
Unpacks a 1-row Arrow `mask` slice (`list<list<bool>>` / `list<list<uint8>>`) in C++
|
| 69 |
+
without materializing millions of nested Python `bool` objects via `to_pydict()`.
|
| 70 |
+
"""
|
| 71 |
+
import pyarrow as pa
|
| 72 |
+
import pyarrow.compute as pc
|
| 73 |
+
|
| 74 |
+
if pa.types.is_nested(mask_slice.type):
|
| 75 |
+
row_inner = pc.list_flatten(mask_slice)
|
| 76 |
+
h = len(row_inner)
|
| 77 |
+
if h == 0:
|
| 78 |
+
return np.zeros((0, 0), dtype=np.uint8)
|
| 79 |
+
if pa.types.is_nested(row_inner.type):
|
| 80 |
+
flat_arr = pc.list_flatten(row_inner)
|
| 81 |
+
flat_np = flat_arr.to_numpy(zero_copy_only=False)
|
| 82 |
+
if flat_np.dtype == np.bool_:
|
| 83 |
+
flat_u8 = flat_np.view(np.uint8)
|
| 84 |
+
else:
|
| 85 |
+
flat_u8 = (flat_np > 0).astype(np.uint8, copy=False)
|
| 86 |
+
w = flat_u8.size // h
|
| 87 |
+
if h * w == flat_u8.size:
|
| 88 |
+
out = flat_u8.reshape(h, w)
|
| 89 |
+
return out if out.flags.writeable else out.copy()
|
| 90 |
+
|
| 91 |
+
raw_val = mask_slice[0].as_py() if hasattr(mask_slice[0], "as_py") else mask_slice[0]
|
| 92 |
+
if isinstance(raw_val, dict) and raw_val.get("bytes"):
|
| 93 |
+
mask_img = Image.open(io.BytesIO(raw_val["bytes"])).convert("L")
|
| 94 |
+
return (np.asarray(mask_img, dtype=np.uint8) > 0).astype(np.uint8)
|
| 95 |
+
if isinstance(raw_val, (bytes, bytearray)):
|
| 96 |
+
mask_img = Image.open(io.BytesIO(raw_val)).convert("L")
|
| 97 |
+
return (np.asarray(mask_img, dtype=np.uint8) > 0).astype(np.uint8)
|
| 98 |
+
out = np.asarray(raw_val, dtype=np.uint8)
|
| 99 |
+
return out if out.flags.writeable else out.copy()
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _rasterize_json_shapes(shapes: List[Dict[str, Any]], height: int, width: int) -> np.ndarray:
|
| 103 |
+
mask = np.zeros((height, width), dtype=np.uint8)
|
| 104 |
+
for shape in shapes:
|
| 105 |
+
pts = np.asarray(shape.get("points", []), dtype=np.float32)
|
| 106 |
+
if pts.ndim == 2 and pts.shape[0] >= 3:
|
| 107 |
+
cv2.fillPoly(mask, [np.round(pts).astype(np.int32)], 1)
|
| 108 |
+
return mask
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _iter_raw_reasonseg_entries(
|
| 112 |
+
data_dir: pathlib.Path,
|
| 113 |
+
max_samples: int = 0,
|
| 114 |
+
) -> Iterator[Dict[str, Any]]:
|
| 115 |
+
"""
|
| 116 |
+
Streams lightweight, undecoded ReasonSeg entries so worker threads can decode JPEGs,
|
| 117 |
+
unpack Arrow bitmasks, and run `processor` in parallel.
|
| 118 |
+
"""
|
| 119 |
+
parquet_files = sorted(data_dir.rglob("*.parquet"))
|
| 120 |
+
yielded = 0
|
| 121 |
+
|
| 122 |
+
if parquet_files:
|
| 123 |
+
try:
|
| 124 |
+
import pyarrow.parquet as pq
|
| 125 |
+
except ImportError as exc:
|
| 126 |
+
raise ImportError(
|
| 127 |
+
"Reading ReasonSeg parquet files requires `pyarrow`. "
|
| 128 |
+
"Install it on your server with: `pip install pyarrow`"
|
| 129 |
+
) from exc
|
| 130 |
+
|
| 131 |
+
for pfile in parquet_files:
|
| 132 |
+
pf = pq.ParquetFile(pfile)
|
| 133 |
+
for batch in pf.iter_batches(batch_size=16):
|
| 134 |
+
col_names = set(batch.schema.names)
|
| 135 |
+
meta_cols = [c for c in ("image", "text", "image_id", "is_sentence") if c in col_names]
|
| 136 |
+
rows = batch.select(meta_cols).to_pydict()
|
| 137 |
+
mask_col = batch.column("mask")
|
| 138 |
+
n_rows = len(rows["text"])
|
| 139 |
+
explicit_flags = rows.get("is_sentence")
|
| 140 |
+
image_ids = rows.get("image_id")
|
| 141 |
+
|
| 142 |
+
for i in range(n_rows):
|
| 143 |
+
text = str(rows["text"][i]).strip()
|
| 144 |
+
explicit_flag = explicit_flags[i] if explicit_flags is not None else None
|
| 145 |
+
yield {
|
| 146 |
+
"kind": "parquet",
|
| 147 |
+
"image_id": str(image_ids[i]) if image_ids is not None else str(i),
|
| 148 |
+
"img_entry": rows["image"][i],
|
| 149 |
+
"mask_slice": mask_col.slice(i, 1),
|
| 150 |
+
"text": text,
|
| 151 |
+
"is_long": is_long_query(text, explicit_flag),
|
| 152 |
+
}
|
| 153 |
+
yielded += 1
|
| 154 |
+
if max_samples > 0 and yielded >= max_samples:
|
| 155 |
+
return
|
| 156 |
+
return
|
| 157 |
+
|
| 158 |
+
json_files = sorted(data_dir.rglob("*.json"))
|
| 159 |
+
for jpath in json_files:
|
| 160 |
+
data = json.loads(jpath.read_text(encoding="utf-8"))
|
| 161 |
+
if "text" not in data or "shapes" not in data:
|
| 162 |
+
continue
|
| 163 |
+
img_path = None
|
| 164 |
+
for ext in (".jpg", ".jpeg", ".png", ".JPG", ".PNG"):
|
| 165 |
+
cand = jpath.with_suffix(ext)
|
| 166 |
+
if cand.exists():
|
| 167 |
+
img_path = cand
|
| 168 |
+
break
|
| 169 |
+
if img_path is None:
|
| 170 |
+
continue
|
| 171 |
+
|
| 172 |
+
shapes = data.get("shapes", [])
|
| 173 |
+
texts = data["text"] if isinstance(data["text"], list) else [data["text"]]
|
| 174 |
+
explicit_flag = data.get("is_sentence", None)
|
| 175 |
+
for t in texts:
|
| 176 |
+
t_str = str(t).strip()
|
| 177 |
+
if not t_str:
|
| 178 |
+
continue
|
| 179 |
+
yield {
|
| 180 |
+
"kind": "json",
|
| 181 |
+
"image_id": jpath.stem,
|
| 182 |
+
"img_path": img_path,
|
| 183 |
+
"shapes": shapes,
|
| 184 |
+
"text": t_str,
|
| 185 |
+
"is_long": is_long_query(t_str, explicit_flag),
|
| 186 |
+
}
|
| 187 |
+
yielded += 1
|
| 188 |
+
if max_samples > 0 and yielded >= max_samples:
|
| 189 |
+
return
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def _decode_raw_entry(entry: Dict[str, Any]) -> Dict[str, Any]:
|
| 193 |
+
if entry["kind"] == "parquet":
|
| 194 |
+
pil_img = _decode_parquet_image(entry["img_entry"])
|
| 195 |
+
gt_mask = _unpack_parquet_mask_slice(entry["mask_slice"])
|
| 196 |
+
else:
|
| 197 |
+
pil_img = Image.open(entry["img_path"]).convert("RGB")
|
| 198 |
+
w, h = pil_img.size
|
| 199 |
+
gt_mask = _rasterize_json_shapes(entry["shapes"], height=h, width=w)
|
| 200 |
+
|
| 201 |
+
return {
|
| 202 |
+
"image_id": entry["image_id"],
|
| 203 |
+
"image": pil_img,
|
| 204 |
+
"text": entry["text"],
|
| 205 |
+
"gt_mask": gt_mask,
|
| 206 |
+
"is_long": entry["is_long"],
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _iter_parquet_samples(parquet_files: List[pathlib.Path]) -> Iterator[Dict[str, Any]]:
|
| 211 |
+
"""Streams ReasonSeg samples from HuggingFace parquet shards."""
|
| 212 |
+
try:
|
| 213 |
+
import pyarrow.parquet as pq
|
| 214 |
+
except ImportError as exc:
|
| 215 |
+
raise ImportError(
|
| 216 |
+
"Reading ReasonSeg parquet files requires `pyarrow`. "
|
| 217 |
+
"Install it on your server with: `pip install pyarrow`"
|
| 218 |
+
) from exc
|
| 219 |
+
|
| 220 |
+
for pfile in sorted(parquet_files):
|
| 221 |
+
pf = pq.ParquetFile(pfile)
|
| 222 |
+
for batch in pf.iter_batches(batch_size=16):
|
| 223 |
+
col_names = set(batch.schema.names)
|
| 224 |
+
meta_cols = [c for c in ("image", "text", "image_id", "is_sentence") if c in col_names]
|
| 225 |
+
rows = batch.select(meta_cols).to_pydict()
|
| 226 |
+
mask_col = batch.column("mask")
|
| 227 |
+
n_rows = len(rows["text"])
|
| 228 |
+
explicit_flags = rows.get("is_sentence")
|
| 229 |
+
image_ids = rows.get("image_id")
|
| 230 |
+
|
| 231 |
+
for i in range(n_rows):
|
| 232 |
+
pil_img = _decode_parquet_image(rows["image"][i])
|
| 233 |
+
gt_mask = _unpack_parquet_mask_slice(mask_col.slice(i, 1))
|
| 234 |
+
text = str(rows["text"][i]).strip()
|
| 235 |
+
explicit_flag = explicit_flags[i] if explicit_flags is not None else None
|
| 236 |
+
yield {
|
| 237 |
+
"image_id": str(image_ids[i]) if image_ids is not None else str(i),
|
| 238 |
+
"image": pil_img,
|
| 239 |
+
"text": text,
|
| 240 |
+
"gt_mask": gt_mask,
|
| 241 |
+
"is_long": is_long_query(text, explicit_flag),
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _iter_json_folder_samples(folder: pathlib.Path) -> Iterator[Dict[str, Any]]:
|
| 246 |
+
"""Streams ReasonSeg samples from a raw directory of .jpg + .json pairs (original LISA format)."""
|
| 247 |
+
json_files = sorted(folder.rglob("*.json"))
|
| 248 |
+
for jpath in json_files:
|
| 249 |
+
data = json.loads(jpath.read_text(encoding="utf-8"))
|
| 250 |
+
if "text" not in data or "shapes" not in data:
|
| 251 |
+
continue
|
| 252 |
+
img_path = None
|
| 253 |
+
for ext in (".jpg", ".jpeg", ".png", ".JPG", ".PNG"):
|
| 254 |
+
cand = jpath.with_suffix(ext)
|
| 255 |
+
if cand.exists():
|
| 256 |
+
img_path = cand
|
| 257 |
+
break
|
| 258 |
+
if img_path is None:
|
| 259 |
+
continue
|
| 260 |
+
|
| 261 |
+
pil_img = Image.open(img_path).convert("RGB")
|
| 262 |
+
w, h = pil_img.size
|
| 263 |
+
mask = _rasterize_json_shapes(data.get("shapes", []), height=h, width=w)
|
| 264 |
+
|
| 265 |
+
texts = data["text"] if isinstance(data["text"], list) else [data["text"]]
|
| 266 |
+
explicit_flag = data.get("is_sentence", None)
|
| 267 |
+
for t in texts:
|
| 268 |
+
t_str = str(t).strip()
|
| 269 |
+
if not t_str:
|
| 270 |
+
continue
|
| 271 |
+
yield {
|
| 272 |
+
"image_id": jpath.stem,
|
| 273 |
+
"image": pil_img,
|
| 274 |
+
"text": t_str,
|
| 275 |
+
"gt_mask": mask,
|
| 276 |
+
"is_long": is_long_query(t_str, explicit_flag),
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def iter_reasonseg_dataset(data_dir: pathlib.Path) -> Iterator[Dict[str, Any]]:
|
| 281 |
+
parquet_files = sorted(data_dir.rglob("*.parquet"))
|
| 282 |
+
if parquet_files:
|
| 283 |
+
yield from _iter_parquet_samples(parquet_files)
|
| 284 |
+
return
|
| 285 |
+
yield from _iter_json_folder_samples(data_dir)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
def _apply_max_pixels(processor: Any, max_pixels: int) -> None:
|
| 289 |
+
if max_pixels <= 0 or processor is None:
|
| 290 |
+
return
|
| 291 |
+
ip = getattr(processor, "image_processor", None)
|
| 292 |
+
if ip is None:
|
| 293 |
+
return
|
| 294 |
+
if hasattr(ip, "max_pixels"):
|
| 295 |
+
ip.max_pixels = int(max_pixels)
|
| 296 |
+
if hasattr(ip, "min_pixels") and getattr(ip, "min_pixels", 0) > max_pixels:
|
| 297 |
+
ip.min_pixels = int(max_pixels)
|
| 298 |
+
size = getattr(ip, "size", None)
|
| 299 |
+
if isinstance(size, dict):
|
| 300 |
+
if "longest_edge" in size:
|
| 301 |
+
size["longest_edge"] = int(max_pixels)
|
| 302 |
+
if "shortest_edge" in size and size["shortest_edge"] > max_pixels:
|
| 303 |
+
size["shortest_edge"] = int(max_pixels)
|
| 304 |
+
if "max_pixels" in size:
|
| 305 |
+
size["max_pixels"] = int(max_pixels)
|
| 306 |
+
if "min_pixels" in size and size["min_pixels"] > max_pixels:
|
| 307 |
+
size["min_pixels"] = int(max_pixels)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _make_processor_getter(
|
| 311 |
+
base_processor: Any,
|
| 312 |
+
use_threads: bool,
|
| 313 |
+
) -> Tuple[Callable[[], Any], Optional[threading.Lock]]:
|
| 314 |
+
if not use_threads:
|
| 315 |
+
return (lambda: base_processor), None
|
| 316 |
+
|
| 317 |
+
tls = threading.local()
|
| 318 |
+
fallback_lock = threading.Lock()
|
| 319 |
+
state = {"deepcopy_ok": True}
|
| 320 |
+
|
| 321 |
+
def _get() -> Any:
|
| 322 |
+
proc = getattr(tls, "processor", None)
|
| 323 |
+
if proc is not None:
|
| 324 |
+
return proc
|
| 325 |
+
if state["deepcopy_ok"]:
|
| 326 |
+
try:
|
| 327 |
+
proc = copy.deepcopy(base_processor)
|
| 328 |
+
tls.processor = proc
|
| 329 |
+
return proc
|
| 330 |
+
except Exception:
|
| 331 |
+
state["deepcopy_ok"] = False
|
| 332 |
+
return base_processor
|
| 333 |
+
|
| 334 |
+
return _get, fallback_lock
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def _prepare_sample_for_eval(
|
| 338 |
+
entry: Dict[str, Any],
|
| 339 |
+
base_processor: Any,
|
| 340 |
+
get_processor: Callable[[], Any],
|
| 341 |
+
fallback_lock: Optional[threading.Lock],
|
| 342 |
+
needs_mm_token_type_ids: bool,
|
| 343 |
+
image_token_id: int,
|
| 344 |
+
) -> Dict[str, Any]:
|
| 345 |
+
decoded = _decode_raw_entry(entry)
|
| 346 |
+
pil_img: Image.Image = decoded["image"]
|
| 347 |
+
orig_w, orig_h = pil_img.size
|
| 348 |
+
|
| 349 |
+
messages, _ = build_segmentation_messages(
|
| 350 |
+
image=pil_img,
|
| 351 |
+
query_input={"objects": [{"id": "0", "description": decoded["text"]}]},
|
| 352 |
+
system_prompt=DEFAULT_SYSTEM_PROMPT,
|
| 353 |
+
query_token=OBJECT_QUERY_TOKEN,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
proc = get_processor()
|
| 357 |
+
if proc is base_processor and fallback_lock is not None:
|
| 358 |
+
with fallback_lock:
|
| 359 |
+
formatted_text = proc.apply_chat_template(
|
| 360 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 361 |
+
)
|
| 362 |
+
inputs = proc(
|
| 363 |
+
text=[formatted_text],
|
| 364 |
+
images=[pil_img],
|
| 365 |
+
padding=True,
|
| 366 |
+
return_tensors="pt",
|
| 367 |
+
)
|
| 368 |
+
else:
|
| 369 |
+
formatted_text = proc.apply_chat_template(
|
| 370 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 371 |
+
)
|
| 372 |
+
inputs = proc(
|
| 373 |
+
text=[formatted_text],
|
| 374 |
+
images=[pil_img],
|
| 375 |
+
padding=True,
|
| 376 |
+
return_tensors="pt",
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
input_ids_cpu = inputs["input_ids"]
|
| 380 |
+
grid_thw_list = inputs["image_grid_thw"].tolist()
|
| 381 |
+
if needs_mm_token_type_ids:
|
| 382 |
+
inputs["mm_token_type_ids"] = (input_ids_cpu == image_token_id).to(dtype=torch.long)
|
| 383 |
+
|
| 384 |
+
gt_mask_np = np.ascontiguousarray(decoded["gt_mask"])
|
| 385 |
+
if not gt_mask_np.flags.writeable:
|
| 386 |
+
gt_mask_np = gt_mask_np.copy()
|
| 387 |
+
gt_mask_t = torch.from_numpy(gt_mask_np)
|
| 388 |
+
|
| 389 |
+
return {
|
| 390 |
+
"inputs": inputs,
|
| 391 |
+
"input_ids_cpu": input_ids_cpu,
|
| 392 |
+
"grid_thw_list": grid_thw_list,
|
| 393 |
+
"orig_size": (orig_h, orig_w),
|
| 394 |
+
"gt_mask": gt_mask_t,
|
| 395 |
+
"is_long": decoded["is_long"],
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def _iter_prepared_samples(
|
| 400 |
+
data_dir: pathlib.Path,
|
| 401 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 402 |
+
max_samples: int = 0,
|
| 403 |
+
num_workers: int = 4,
|
| 404 |
+
) -> Iterator[Dict[str, Any]]:
|
| 405 |
+
raw_stream = _iter_raw_reasonseg_entries(data_dir, max_samples=max_samples)
|
| 406 |
+
needs_mm = getattr(model, "_needs_mm_token_type_ids", False)
|
| 407 |
+
image_token_id = model.image_token_id
|
| 408 |
+
base_processor = model.processor
|
| 409 |
+
use_threads = num_workers > 0
|
| 410 |
+
get_processor, fallback_lock = _make_processor_getter(base_processor, use_threads=use_threads)
|
| 411 |
+
|
| 412 |
+
if not use_threads:
|
| 413 |
+
for entry in raw_stream:
|
| 414 |
+
yield _prepare_sample_for_eval(
|
| 415 |
+
entry=entry,
|
| 416 |
+
base_processor=base_processor,
|
| 417 |
+
get_processor=get_processor,
|
| 418 |
+
fallback_lock=fallback_lock,
|
| 419 |
+
needs_mm_token_type_ids=needs_mm,
|
| 420 |
+
image_token_id=image_token_id,
|
| 421 |
+
)
|
| 422 |
+
return
|
| 423 |
+
|
| 424 |
+
max_inflight = max(num_workers * 2, 2)
|
| 425 |
+
inflight: deque[Future] = deque()
|
| 426 |
+
raw_it = iter(raw_stream)
|
| 427 |
+
exhausted = False
|
| 428 |
+
|
| 429 |
+
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
| 430 |
+
def _fill_queue() -> None:
|
| 431 |
+
nonlocal exhausted
|
| 432 |
+
while not exhausted and len(inflight) < max_inflight:
|
| 433 |
+
try:
|
| 434 |
+
item = next(raw_it)
|
| 435 |
+
except StopIteration:
|
| 436 |
+
exhausted = True
|
| 437 |
+
break
|
| 438 |
+
inflight.append(
|
| 439 |
+
pool.submit(
|
| 440 |
+
_prepare_sample_for_eval,
|
| 441 |
+
item,
|
| 442 |
+
base_processor,
|
| 443 |
+
get_processor,
|
| 444 |
+
fallback_lock,
|
| 445 |
+
needs_mm,
|
| 446 |
+
image_token_id,
|
| 447 |
+
)
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
try:
|
| 451 |
+
_fill_queue()
|
| 452 |
+
while inflight:
|
| 453 |
+
fut = inflight.popleft()
|
| 454 |
+
_fill_queue()
|
| 455 |
+
yield fut.result()
|
| 456 |
+
finally:
|
| 457 |
+
for fut in inflight:
|
| 458 |
+
fut.cancel()
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
@torch.no_grad()
|
| 462 |
+
def predict_single_mask(
|
| 463 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 464 |
+
image: Image.Image,
|
| 465 |
+
description: str,
|
| 466 |
+
device: torch.device,
|
| 467 |
+
threshold: float = 0.5,
|
| 468 |
+
) -> np.ndarray:
|
| 469 |
+
"""Runs 1 forward pass and returns the binary numpy mask (H, W) in {0, 1}."""
|
| 470 |
+
orig_w, orig_h = image.size
|
| 471 |
+
messages, _ = build_segmentation_messages(
|
| 472 |
+
image=image,
|
| 473 |
+
query_input={"objects": [{"id": "0", "description": description}]},
|
| 474 |
+
system_prompt=DEFAULT_SYSTEM_PROMPT,
|
| 475 |
+
query_token=OBJECT_QUERY_TOKEN,
|
| 476 |
+
)
|
| 477 |
+
formatted_text = model.processor.apply_chat_template(
|
| 478 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 479 |
+
)
|
| 480 |
+
inputs = model.processor(
|
| 481 |
+
text=[formatted_text],
|
| 482 |
+
images=[image],
|
| 483 |
+
padding=True,
|
| 484 |
+
return_tensors="pt",
|
| 485 |
+
)
|
| 486 |
+
input_ids_cpu = inputs["input_ids"]
|
| 487 |
+
grid_thw_list = inputs["image_grid_thw"].tolist()
|
| 488 |
+
inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()}
|
| 489 |
+
|
| 490 |
+
outputs = model(
|
| 491 |
+
input_ids=inputs["input_ids"],
|
| 492 |
+
attention_mask=inputs["attention_mask"],
|
| 493 |
+
pixel_values=inputs["pixel_values"],
|
| 494 |
+
image_grid_thw=inputs["image_grid_thw"],
|
| 495 |
+
target_sizes=[(orig_h, orig_w)],
|
| 496 |
+
grid_thw_list=grid_thw_list,
|
| 497 |
+
input_ids_cpu=input_ids_cpu,
|
| 498 |
+
)
|
| 499 |
+
prob = outputs["mask_probs"][0][0]
|
| 500 |
+
return (prob >= threshold).to(torch.uint8).cpu().numpy()
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
class MetricBucket:
|
| 504 |
+
def __init__(self) -> None:
|
| 505 |
+
self.ious: List[float] = []
|
| 506 |
+
self.total_inter: float = 0.0
|
| 507 |
+
self.total_union: float = 0.0
|
| 508 |
+
|
| 509 |
+
def update_counts(self, inter: float, union: float) -> float:
|
| 510 |
+
iou = 1.0 if union == 0 else (inter / union)
|
| 511 |
+
self.ious.append(iou)
|
| 512 |
+
self.total_inter += inter
|
| 513 |
+
self.total_union += union
|
| 514 |
+
return iou
|
| 515 |
+
|
| 516 |
+
def update(self, pred_mask: np.ndarray, gt_mask: np.ndarray) -> float:
|
| 517 |
+
pred_b = pred_mask > 0
|
| 518 |
+
gt_b = gt_mask > 0
|
| 519 |
+
inter = float(np.logical_and(pred_b, gt_b).sum())
|
| 520 |
+
union = float(np.logical_or(pred_b, gt_b).sum())
|
| 521 |
+
return self.update_counts(inter, union)
|
| 522 |
+
|
| 523 |
+
@property
|
| 524 |
+
def giou(self) -> float:
|
| 525 |
+
return (sum(self.ious) / len(self.ious) * 100.0) if self.ious else 0.0
|
| 526 |
+
|
| 527 |
+
@property
|
| 528 |
+
def ciou(self) -> float:
|
| 529 |
+
return (self.total_inter / self.total_union * 100.0) if self.total_union > 0 else 0.0
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
@torch.no_grad()
|
| 533 |
+
def evaluate_split(
|
| 534 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 535 |
+
data_dir: pathlib.Path,
|
| 536 |
+
device: torch.device,
|
| 537 |
+
threshold: float = 0.5,
|
| 538 |
+
max_samples: int = 0,
|
| 539 |
+
log_every: int = 25,
|
| 540 |
+
num_workers: int = 4,
|
| 541 |
+
) -> Dict[str, MetricBucket]:
|
| 542 |
+
buckets = {
|
| 543 |
+
"short": MetricBucket(),
|
| 544 |
+
"long": MetricBucket(),
|
| 545 |
+
"overall": MetricBucket(),
|
| 546 |
+
}
|
| 547 |
+
is_cuda = device.type == "cuda"
|
| 548 |
+
pending: List[Tuple[torch.Tensor, bool, Optional[Tuple[Any, Any]]]] = []
|
| 549 |
+
gpu_ms_total = 0.0
|
| 550 |
+
|
| 551 |
+
def _flush_pending() -> None:
|
| 552 |
+
nonlocal gpu_ms_total
|
| 553 |
+
if not pending:
|
| 554 |
+
return
|
| 555 |
+
pairs_cpu = torch.stack([p for p, _, _ in pending], dim=0).cpu().tolist()
|
| 556 |
+
if is_cuda:
|
| 557 |
+
torch.cuda.synchronize(device)
|
| 558 |
+
for (inter_i, union_i), (_, is_long_i, ev_pair) in zip(pairs_cpu, pending):
|
| 559 |
+
inter_f = float(inter_i)
|
| 560 |
+
union_f = float(union_i)
|
| 561 |
+
buckets["overall"].update_counts(inter_f, union_f)
|
| 562 |
+
if is_long_i:
|
| 563 |
+
buckets["long"].update_counts(inter_f, union_f)
|
| 564 |
+
else:
|
| 565 |
+
buckets["short"].update_counts(inter_f, union_f)
|
| 566 |
+
if ev_pair is not None:
|
| 567 |
+
gpu_ms_total += float(ev_pair[0].elapsed_time(ev_pair[1]))
|
| 568 |
+
pending.clear()
|
| 569 |
+
|
| 570 |
+
t0 = time.perf_counter()
|
| 571 |
+
stream = _iter_prepared_samples(
|
| 572 |
+
data_dir=data_dir,
|
| 573 |
+
model=model,
|
| 574 |
+
max_samples=max_samples,
|
| 575 |
+
num_workers=num_workers,
|
| 576 |
+
)
|
| 577 |
+
for idx, prepared in enumerate(stream, start=1):
|
| 578 |
+
dev_inputs = {
|
| 579 |
+
k: (v.to(device) if isinstance(v, torch.Tensor) else v)
|
| 580 |
+
for k, v in prepared["inputs"].items()
|
| 581 |
+
}
|
| 582 |
+
gt_t = prepared["gt_mask"].to(device=device)
|
| 583 |
+
|
| 584 |
+
ev_pair: Optional[Tuple[Any, Any]] = None
|
| 585 |
+
if is_cuda:
|
| 586 |
+
ev_start = torch.cuda.Event(enable_timing=True)
|
| 587 |
+
ev_end = torch.cuda.Event(enable_timing=True)
|
| 588 |
+
ev_start.record()
|
| 589 |
+
ev_pair = (ev_start, ev_end)
|
| 590 |
+
|
| 591 |
+
outputs = model(
|
| 592 |
+
input_ids=dev_inputs["input_ids"],
|
| 593 |
+
attention_mask=dev_inputs["attention_mask"],
|
| 594 |
+
pixel_values=dev_inputs["pixel_values"],
|
| 595 |
+
image_grid_thw=dev_inputs["image_grid_thw"],
|
| 596 |
+
target_sizes=[prepared["orig_size"]],
|
| 597 |
+
mm_token_type_ids=dev_inputs.get("mm_token_type_ids"),
|
| 598 |
+
grid_thw_list=prepared["grid_thw_list"],
|
| 599 |
+
input_ids_cpu=prepared["input_ids_cpu"],
|
| 600 |
+
)
|
| 601 |
+
prob = outputs["mask_probs"][0][0]
|
| 602 |
+
pred_b = prob >= threshold
|
| 603 |
+
if pred_b.shape != gt_t.shape:
|
| 604 |
+
pred_b = (
|
| 605 |
+
F.interpolate(
|
| 606 |
+
pred_b.to(torch.float32).unsqueeze(0).unsqueeze(0),
|
| 607 |
+
size=gt_t.shape,
|
| 608 |
+
mode="nearest",
|
| 609 |
+
)
|
| 610 |
+
.squeeze(0)
|
| 611 |
+
.squeeze(0)
|
| 612 |
+
> 0.5
|
| 613 |
+
)
|
| 614 |
+
gt_b = gt_t > 0
|
| 615 |
+
inter_t = torch.logical_and(pred_b, gt_b).sum(dtype=torch.int64)
|
| 616 |
+
union_t = torch.logical_or(pred_b, gt_b).sum(dtype=torch.int64)
|
| 617 |
+
pair_t = torch.stack((inter_t, union_t))
|
| 618 |
+
|
| 619 |
+
if ev_pair is not None:
|
| 620 |
+
ev_pair[1].record()
|
| 621 |
+
|
| 622 |
+
pending.append((pair_t, prepared["is_long"], ev_pair))
|
| 623 |
+
|
| 624 |
+
if log_every > 0 and idx % log_every == 0:
|
| 625 |
+
_flush_pending()
|
| 626 |
+
elapsed = time.perf_counter() - t0
|
| 627 |
+
timing_str = f"{elapsed / idx * 1000:.1f} ms/sample"
|
| 628 |
+
if is_cuda and gpu_ms_total > 0.0:
|
| 629 |
+
timing_str += f" (gpu {gpu_ms_total / idx:.1f} ms)"
|
| 630 |
+
print(
|
| 631 |
+
f"[{data_dir.name} | {idx} samples | {timing_str}] "
|
| 632 |
+
f"overall gIoU={buckets['overall'].giou:.1f} cIoU={buckets['overall'].ciou:.1f} | "
|
| 633 |
+
f"short gIoU={buckets['short'].giou:.1f} | long gIoU={buckets['long'].giou:.1f}",
|
| 634 |
+
flush=True,
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
_flush_pending()
|
| 638 |
+
return buckets
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def format_markdown_table(
|
| 642 |
+
method_name: str,
|
| 643 |
+
val_buckets: Dict[str, MetricBucket],
|
| 644 |
+
test_buckets: Dict[str, MetricBucket],
|
| 645 |
+
) -> str:
|
| 646 |
+
lines = [
|
| 647 |
+
"| Method | Val overall gIoU | Val overall cIoU | Test short query gIoU | Test short query cIoU | Test long query gIoU | Test long query cIoU | Test overall gIoU | Test overall cIoU |",
|
| 648 |
+
"|---|---:|---:|---:|---:|---:|---:|---:|---:|",
|
| 649 |
+
]
|
| 650 |
+
for row in BASELINE_ROWS:
|
| 651 |
+
name, v_g, v_c, ts_g, ts_c, tl_g, tl_c, to_g, to_c = row
|
| 652 |
+
lines.append(
|
| 653 |
+
f"| {name} | {v_g:.1f} | {v_c:.1f} | {ts_g:.1f} | {ts_c:.1f} | {tl_g:.1f} | {tl_c:.1f} | {to_g:.1f} | {to_c:.1f} |"
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
lines.append(
|
| 657 |
+
f"| {method_name} | "
|
| 658 |
+
f"{val_buckets['overall'].giou:.1f} | {val_buckets['overall'].ciou:.1f} | "
|
| 659 |
+
f"{test_buckets['short'].giou:.1f} | {test_buckets['short'].ciou:.1f} | "
|
| 660 |
+
f"{test_buckets['long'].giou:.1f} | {test_buckets['long'].ciou:.1f} | "
|
| 661 |
+
f"{test_buckets['overall'].giou:.1f} | {test_buckets['overall'].ciou:.1f} |"
|
| 662 |
+
)
|
| 663 |
+
lines.append("")
|
| 664 |
+
lines.append("*Table 1: Reasoning segmentation results among LISA (ours) and previous related works.*")
|
| 665 |
+
return "\n".join(lines)
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
def parse_args() -> argparse.Namespace:
|
| 669 |
+
parser = argparse.ArgumentParser(description="Benchmark Single-Pass Segmenter on ReasonSeg")
|
| 670 |
+
parser.add_argument(
|
| 671 |
+
"--data-dir",
|
| 672 |
+
default="./ReasonSeg_val",
|
| 673 |
+
help="Default ReasonSeg directory (used for both Val and Test columns unless --val-dir / --test-dir are set)",
|
| 674 |
+
)
|
| 675 |
+
parser.add_argument("--val-dir", default="", help="Optional separate ReasonSeg val split directory")
|
| 676 |
+
parser.add_argument("--test-dir", default="", help="Optional separate ReasonSeg test split directory")
|
| 677 |
+
parser.add_argument("--resume-ckpt", default="./checkpoints/segmenter_stage1_epoch5.pt")
|
| 678 |
+
parser.add_argument("--model-path", default="./Qwen3.5-4B")
|
| 679 |
+
parser.add_argument("--method-name", default="TurboSmartSegment-Qwen3.5-4B (Ours)")
|
| 680 |
+
parser.add_argument("--fused-layers", type=int, default=4)
|
| 681 |
+
parser.add_argument("--threshold", type=float, default=0.5)
|
| 682 |
+
parser.add_argument("--max-samples", type=int, default=0, help="0 = evaluate all samples")
|
| 683 |
+
parser.add_argument(
|
| 684 |
+
"--num-workers",
|
| 685 |
+
type=int,
|
| 686 |
+
default=4,
|
| 687 |
+
help="Background CPU threads for parallel Parquet/image decoding and preprocessing (0 = main thread)",
|
| 688 |
+
)
|
| 689 |
+
parser.add_argument(
|
| 690 |
+
"--max-pixels",
|
| 691 |
+
type=int,
|
| 692 |
+
default=0,
|
| 693 |
+
help="Optional max image pixels for processor (0 = use model default without capping)",
|
| 694 |
+
)
|
| 695 |
+
parser.add_argument("--log-every", type=int, default=25)
|
| 696 |
+
parser.add_argument("--out-md", default="./reasonseg_benchmark.md")
|
| 697 |
+
parser.add_argument("--device", default="cuda")
|
| 698 |
+
return parser.parse_args()
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
def main() -> None:
|
| 702 |
+
args = parse_args()
|
| 703 |
+
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
|
| 704 |
+
dtype = (
|
| 705 |
+
torch.bfloat16
|
| 706 |
+
if (device.type == "cuda" and torch.cuda.is_bf16_supported())
|
| 707 |
+
else torch.float32
|
| 708 |
+
)
|
| 709 |
+
|
| 710 |
+
model = Qwen2_5_VL_SinglePassSegmenter(
|
| 711 |
+
qwen_model_name=args.model_path,
|
| 712 |
+
torch_dtype=dtype,
|
| 713 |
+
freeze_backbone=True,
|
| 714 |
+
num_fused_layers=args.fused_layers,
|
| 715 |
+
)
|
| 716 |
+
_apply_max_pixels(model.processor, args.max_pixels)
|
| 717 |
+
model.to(device)
|
| 718 |
+
model.mask_head.to(device=device, dtype=torch.float32)
|
| 719 |
+
model.eval()
|
| 720 |
+
|
| 721 |
+
if args.resume_ckpt:
|
| 722 |
+
ckpt_path = pathlib.Path(args.resume_ckpt)
|
| 723 |
+
load_checkpoint(model, ckpt_path, device)
|
| 724 |
+
state = torch.load(ckpt_path, map_location=device)
|
| 725 |
+
if "lora_dir" not in state or not state["lora_dir"]:
|
| 726 |
+
print(
|
| 727 |
+
"LoRA adapter: checkpoint has no adapter path; evaluating without a Stage 2 adapter.",
|
| 728 |
+
flush=True,
|
| 729 |
+
)
|
| 730 |
+
else:
|
| 731 |
+
lora_dir = pathlib.Path(state["lora_dir"])
|
| 732 |
+
if not lora_dir.exists():
|
| 733 |
+
print(
|
| 734 |
+
f"LoRA adapter: not found at {lora_dir.resolve()} "
|
| 735 |
+
"(path is resolved from the current working directory); evaluating without it.",
|
| 736 |
+
flush=True,
|
| 737 |
+
)
|
| 738 |
+
else:
|
| 739 |
+
print(f"LoRA adapter: found at {lora_dir.resolve()}; loading...", flush=True)
|
| 740 |
+
from peft import PeftModel
|
| 741 |
+
|
| 742 |
+
model.backbone.language_model = PeftModel.from_pretrained(
|
| 743 |
+
model.backbone.language_model, str(lora_dir)
|
| 744 |
+
).to(device=device, dtype=dtype)
|
| 745 |
+
model._register_llm_layer_hooks()
|
| 746 |
+
model.eval()
|
| 747 |
+
print(
|
| 748 |
+
f"LoRA adapter: successfully applied from {lora_dir.resolve()}.",
|
| 749 |
+
flush=True,
|
| 750 |
+
)
|
| 751 |
+
|
| 752 |
+
val_dir = pathlib.Path(args.val_dir) if args.val_dir else pathlib.Path(args.data_dir)
|
| 753 |
+
test_dir = pathlib.Path(args.test_dir) if args.test_dir else pathlib.Path(args.data_dir)
|
| 754 |
+
|
| 755 |
+
print(f"Evaluating on: {val_dir} ...", flush=True)
|
| 756 |
+
val_buckets = evaluate_split(
|
| 757 |
+
model=model,
|
| 758 |
+
data_dir=val_dir,
|
| 759 |
+
device=device,
|
| 760 |
+
threshold=args.threshold,
|
| 761 |
+
max_samples=args.max_samples,
|
| 762 |
+
log_every=args.log_every,
|
| 763 |
+
num_workers=args.num_workers,
|
| 764 |
+
)
|
| 765 |
+
|
| 766 |
+
if test_dir.resolve() == val_dir.resolve():
|
| 767 |
+
test_buckets = val_buckets
|
| 768 |
+
else:
|
| 769 |
+
print(f"Evaluating on: {test_dir} ...", flush=True)
|
| 770 |
+
test_buckets = evaluate_split(
|
| 771 |
+
model=model,
|
| 772 |
+
data_dir=test_dir,
|
| 773 |
+
device=device,
|
| 774 |
+
threshold=args.threshold,
|
| 775 |
+
max_samples=args.max_samples,
|
| 776 |
+
log_every=args.log_every,
|
| 777 |
+
num_workers=args.num_workers,
|
| 778 |
+
)
|
| 779 |
+
|
| 780 |
+
table_md = format_markdown_table(args.method_name, val_buckets, test_buckets)
|
| 781 |
+
print("\n" + table_md + "\n", flush=True)
|
| 782 |
+
|
| 783 |
+
if args.out_md:
|
| 784 |
+
out_path = pathlib.Path(args.out_md)
|
| 785 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 786 |
+
out_path.write_text(table_md + "\n", encoding="utf-8")
|
| 787 |
+
print(f"Saved Markdown benchmark table to: {out_path}", flush=True)
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
if __name__ == "__main__":
|
| 791 |
+
main()
|
dataset.py
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import pathlib
|
| 3 |
+
import pickle
|
| 4 |
+
import random
|
| 5 |
+
from collections import defaultdict
|
| 6 |
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from torch.utils.data import Dataset
|
| 11 |
+
|
| 12 |
+
from LIS1ModelCode import (
|
| 13 |
+
DEFAULT_SYSTEM_PROMPT,
|
| 14 |
+
OBJECT_QUERY_TOKEN,
|
| 15 |
+
build_segmentation_messages,
|
| 16 |
+
build_segmentation_prompt,
|
| 17 |
+
polygons_to_mask,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# Functional & commonsense descriptions for COCO categories to bridge RefCOCO -> ReasonSeg riddles
|
| 22 |
+
COCO_FUNCTIONAL_CLUES: Dict[str, List[str]] = {
|
| 23 |
+
"person": ["someone who is present in the scene", "the human individual shown here"],
|
| 24 |
+
"bicycle": ["a two-wheeled vehicle propelled by pedaling", "what you can ride for eco-friendly commuting"],
|
| 25 |
+
"car": ["a motor vehicle used for driving on roads", "the automobile parked or moving in the scene"],
|
| 26 |
+
"motorcycle": ["a two-wheeled motor vehicle for riding", "the motorbike in the picture"],
|
| 27 |
+
"airplane": ["an aircraft built for flying through the sky", "what transports passengers by air"],
|
| 28 |
+
"bus": ["a large road vehicle designed to carry many passengers", "public transit vehicle"],
|
| 29 |
+
"train": ["a rail vehicle that travels along tracks", "locomotive or rail car"],
|
| 30 |
+
"truck": ["a heavy motor vehicle built for hauling cargo", "the utility or freight vehicle"],
|
| 31 |
+
"boat": ["a watercraft used for traveling across water", "vessel floating on the water"],
|
| 32 |
+
"traffic light": ["the signaling device that controls road traffic flow", "where drivers look to know whether to stop or go"],
|
| 33 |
+
"bench": ["a long seat where multiple people can sit outdoors or indoors", "place to sit down and rest"],
|
| 34 |
+
"backpack": ["a bag carried on the back to hold personal belongings", "what someone wears on their shoulders to carry gear"],
|
| 35 |
+
"umbrella": ["what you should hold overhead to stay dry when it rains", "item used for shade or protection from rain"],
|
| 36 |
+
"handbag": ["a personal bag held in the hand or over the shoulder", "accessory used to carry daily essentials"],
|
| 37 |
+
"tie": ["a formal cloth accessory worn around the neck with a collared shirt", "neckwear piece"],
|
| 38 |
+
"suitcase": ["luggage used to pack clothes when traveling on a trip", "travel baggage"],
|
| 39 |
+
"bottle": ["a narrow-necked container used to hold liquids or drinks", "vessel you can pour a beverage from"],
|
| 40 |
+
"wine glass": ["a stemmed glass vessel used for drinking wine", "glassware for a formal beverage"],
|
| 41 |
+
"cup": ["what you can drink from if you are thirsty", "vessel used to hold a hot or cold beverage"],
|
| 42 |
+
"fork": ["utensil with prongs used to spear and eat solid food", "cutlery item for dining"],
|
| 43 |
+
"knife": ["bladed utensil used for cutting food at the table or kitchen", "sharp tool for slicing"],
|
| 44 |
+
"spoon": ["curved utensil used for eating soup or stirring liquids", "utensil for scooping food"],
|
| 45 |
+
"bowl": ["a round deep dish used to serve soup, cereal, or salad", "container holding food"],
|
| 46 |
+
"banana": ["a curved yellow fruit you peel before eating", "potassium-rich snack fruit"],
|
| 47 |
+
"apple": ["a crisp round fruit that keeps the doctor away", "fresh fruit snack"],
|
| 48 |
+
"sandwich": ["a meal made of fillings placed between slices of bread", "prepared food you can eat for lunch"],
|
| 49 |
+
"orange": ["a citrus fruit rich in vitamin C", "round citrus snack"],
|
| 50 |
+
"broccoli": ["a green tree-shaped vegetable rich in nutrients", "green vegetable side"],
|
| 51 |
+
"carrot": ["an crunchy orange root vegetable", "root vegetable"],
|
| 52 |
+
"pizza": ["a flat baked dough topped with tomato sauce and melted cheese", "sliced Italian dish"],
|
| 53 |
+
"donut": ["a sweet fried ring-shaped pastry", "sugary dessert treat"],
|
| 54 |
+
"cake": ["a sweet baked dessert often served at celebrations or birthdays", "baked dessert"],
|
| 55 |
+
"chair": ["a piece of furniture designed for one person to sit on", "where someone can sit down"],
|
| 56 |
+
"couch": ["an upholstered sofa where several people can sit or lounge comfortably", "living room seating furniture"],
|
| 57 |
+
"potted plant": ["indoor or outdoor greenery growing inside a container", "decorative living plant in a pot"],
|
| 58 |
+
"bed": ["furniture with a mattress where someone can lie down and sleep", "place to rest and sleep at night"],
|
| 59 |
+
"dining table": ["the surface where meals are served and people gather to eat", "table used for dining"],
|
| 60 |
+
"toilet": ["bathroom fixture used for sanitation", "plumbing fixture in the restroom"],
|
| 61 |
+
"tv": ["the screen used to watch broadcasts, movies, or visual presentations", "display screen for viewing video"],
|
| 62 |
+
"laptop": ["a portable personal computer with a clamshell screen and keyboard", "device used for mobile work or browsing"],
|
| 63 |
+
"mouse": ["handheld pointing device used to move a cursor on a computer screen", "computer input peripheral"],
|
| 64 |
+
"remote": ["handheld controller used to operate a television or device from a distance", "what you press to change channels"],
|
| 65 |
+
"keyboard": ["panel of keys used for typing text into a computer", "input device for typing"],
|
| 66 |
+
"cell phone": ["portable handheld device used to make calls, text, or take photos", "smartphone or mobile device"],
|
| 67 |
+
"microwave": ["kitchen appliance that quickly heats or reheats food using electromagnetic waves", "appliance for warming up leftovers"],
|
| 68 |
+
"oven": ["heated chamber used for baking or roasting food in the kitchen", "cooking appliance"],
|
| 69 |
+
"toaster": ["small electric kitchen appliance used to brown slices of bread", "breakfast appliance"],
|
| 70 |
+
"sink": ["basin with a faucet where you can wash your hands or dishes", "where to wash hands"],
|
| 71 |
+
"refrigerator": ["insulated kitchen appliance that keeps food and beverages cold and fresh", "where perishable food is stored"],
|
| 72 |
+
"book": ["printed work of pages bound together that you can read for knowledge or leisure", "something to read"],
|
| 73 |
+
"clock": ["the instrument you look at to see what time it is", "timepiece showing the hour"],
|
| 74 |
+
"vase": ["decorative container used to hold cut flowers", "vessel holding stems or flowers"],
|
| 75 |
+
"scissors": ["handheld tool with two pivoted blades used for cutting paper or fabric", "cutting tool"],
|
| 76 |
+
"teddy bear": ["a soft stuffed toy bear made for comfort or play", "plush toy"],
|
| 77 |
+
"hair drier": ["electromechanical device that blows warm air to dry wet hair", "grooming appliance"],
|
| 78 |
+
"toothbrush": ["small brush used with toothpaste to clean teeth", "oral hygiene tool"],
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
SHORT_RIDDLE_TEMPLATES: List[str] = [
|
| 82 |
+
"{clue} ({desc})",
|
| 83 |
+
"the {cat} that is {clue}, specifically {desc}",
|
| 84 |
+
"where to find {clue}: {desc}",
|
| 85 |
+
]
|
| 86 |
+
|
| 87 |
+
LONG_RIDDLE_TEMPLATES: List[str] = [
|
| 88 |
+
"In this scene, suppose you are looking for {clue}. Which specific object—described as '{desc}'—matches this need?",
|
| 89 |
+
"Sometimes we identify an object by its function or role, such as {clue}. What part of the picture corresponds to '{desc}'?",
|
| 90 |
+
"If someone asked you to point out {clue} in this image, specifically the one that looks like '{desc}', which object would you select?",
|
| 91 |
+
]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class RefCOCOJsonDataset(Dataset):
|
| 95 |
+
"""
|
| 96 |
+
Loads RefCOCO / RefCOCO+ / RefCOCOg entries (or all combined when dataset_name='all')
|
| 97 |
+
and groups referring expressions by image_id for single-pass multi-object JSON training.
|
| 98 |
+
Includes optional functional/riddle prompt augmentation to generalize to ReasonSeg.
|
| 99 |
+
"""
|
| 100 |
+
|
| 101 |
+
def __init__(
|
| 102 |
+
self,
|
| 103 |
+
data_root: Union[str, pathlib.Path] = "./ref_coco_lichengunc",
|
| 104 |
+
dataset_name: str = "refcoco",
|
| 105 |
+
split_by: str = "unc",
|
| 106 |
+
split: str = "train",
|
| 107 |
+
max_objects_per_image: int = 4,
|
| 108 |
+
min_objects_per_image: int = 1,
|
| 109 |
+
group_by_image: bool = True,
|
| 110 |
+
riddle_prob: float = 0.35,
|
| 111 |
+
seed: int = 42,
|
| 112 |
+
):
|
| 113 |
+
super().__init__()
|
| 114 |
+
self.data_root = pathlib.Path(data_root)
|
| 115 |
+
self.dataset_name = dataset_name
|
| 116 |
+
self.split_by = split_by
|
| 117 |
+
self.split = split
|
| 118 |
+
self.max_objects_per_image = max_objects_per_image
|
| 119 |
+
self.min_objects_per_image = min_objects_per_image
|
| 120 |
+
self.group_by_image = group_by_image
|
| 121 |
+
self.riddle_prob = riddle_prob if split == "train" else 0.0
|
| 122 |
+
self.rng = random.Random(seed)
|
| 123 |
+
|
| 124 |
+
candidate_dirs = [
|
| 125 |
+
self.data_root / "images" / "train2014",
|
| 126 |
+
self.data_root / "images" / "train2014" / "train2014",
|
| 127 |
+
self.data_root / "train2014",
|
| 128 |
+
self.data_root / "images",
|
| 129 |
+
]
|
| 130 |
+
self.image_dir = candidate_dirs[0]
|
| 131 |
+
for cand in candidate_dirs:
|
| 132 |
+
if cand.exists() and any(cand.glob("*.jpg")):
|
| 133 |
+
self.image_dir = cand
|
| 134 |
+
break
|
| 135 |
+
|
| 136 |
+
if dataset_name == "all":
|
| 137 |
+
specs = [("refcoco", "unc"), ("refcoco+", "unc"), ("refcocog", "umd")]
|
| 138 |
+
else:
|
| 139 |
+
default_split_by = "umd" if (dataset_name == "refcocog" and split_by == "unc") else split_by
|
| 140 |
+
specs = [(dataset_name, default_split_by)]
|
| 141 |
+
|
| 142 |
+
self.images_by_id: Dict[int, Dict[str, Any]] = {}
|
| 143 |
+
self.anns_by_id: Dict[Union[int, str], Dict[str, Any]] = {}
|
| 144 |
+
self.cats_by_id: Dict[int, str] = {}
|
| 145 |
+
filtered_refs: List[Dict[str, Any]] = []
|
| 146 |
+
|
| 147 |
+
for ds_name, sp_by in specs:
|
| 148 |
+
sub_dir = self.data_root / ds_name
|
| 149 |
+
refs_path = sub_dir / f"refs({sp_by}).p"
|
| 150 |
+
instances_path = sub_dir / "instances.json"
|
| 151 |
+
if not refs_path.exists() or not instances_path.exists():
|
| 152 |
+
if dataset_name != "all":
|
| 153 |
+
raise FileNotFoundError(f"Could not find {refs_path} or {instances_path}")
|
| 154 |
+
continue
|
| 155 |
+
|
| 156 |
+
with open(refs_path, "rb") as f:
|
| 157 |
+
refs = pickle.load(f)
|
| 158 |
+
with open(instances_path, "r", encoding="utf-8") as f:
|
| 159 |
+
instances = json.load(f)
|
| 160 |
+
|
| 161 |
+
for img in instances.get("images", []):
|
| 162 |
+
self.images_by_id[img["id"]] = img
|
| 163 |
+
for ann in instances.get("annotations", []):
|
| 164 |
+
self.anns_by_id[ann["id"]] = ann
|
| 165 |
+
for cat in instances.get("categories", []):
|
| 166 |
+
self.cats_by_id[cat["id"]] = cat["name"]
|
| 167 |
+
|
| 168 |
+
for ref in refs:
|
| 169 |
+
if ref.get("split") != split:
|
| 170 |
+
continue
|
| 171 |
+
img_id = ref["image_id"]
|
| 172 |
+
ann_id = ref["ann_id"]
|
| 173 |
+
if img_id not in self.images_by_id or ann_id not in self.anns_by_id:
|
| 174 |
+
continue
|
| 175 |
+
img_info = self.images_by_id[img_id]
|
| 176 |
+
img_path = self.image_dir / img_info["file_name"]
|
| 177 |
+
if not img_path.exists():
|
| 178 |
+
continue
|
| 179 |
+
sentences = [s["sent"].strip() for s in ref.get("sentences", []) if s.get("sent", "").strip()]
|
| 180 |
+
if not sentences:
|
| 181 |
+
continue
|
| 182 |
+
filtered_refs.append(
|
| 183 |
+
{
|
| 184 |
+
"ref_id": f"{ds_name}_{ref['ref_id']}",
|
| 185 |
+
"image_id": img_id,
|
| 186 |
+
"ann_id": ann_id,
|
| 187 |
+
"category_id": ref.get("category_id"),
|
| 188 |
+
"sentences": sentences,
|
| 189 |
+
}
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
if group_by_image:
|
| 193 |
+
by_img: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
|
| 194 |
+
for item in filtered_refs:
|
| 195 |
+
by_img[item["image_id"]].append(item)
|
| 196 |
+
self.samples: List[List[Dict[str, Any]]] = list(by_img.values())
|
| 197 |
+
else:
|
| 198 |
+
self.samples = [[item] for item in filtered_refs]
|
| 199 |
+
|
| 200 |
+
if not self.samples:
|
| 201 |
+
raise FileNotFoundError(
|
| 202 |
+
f"No valid samples found for dataset='{dataset_name}' split='{split}'. "
|
| 203 |
+
f"Checked image directory: {self.image_dir} (ensure COCO_train2014_*.jpg files are present)."
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
def __len__(self) -> int:
|
| 207 |
+
return len(self.samples)
|
| 208 |
+
|
| 209 |
+
def _augment_as_riddle(self, desc: str, category_id: Optional[int]) -> str:
|
| 210 |
+
cat_name = self.cats_by_id.get(category_id or -1, "object")
|
| 211 |
+
clues = COCO_FUNCTIONAL_CLUES.get(cat_name, [f"an instance of {cat_name} in the scene"])
|
| 212 |
+
clue = self.rng.choice(clues)
|
| 213 |
+
if self.rng.random() < 0.5:
|
| 214 |
+
tpl = self.rng.choice(SHORT_RIDDLE_TEMPLATES)
|
| 215 |
+
else:
|
| 216 |
+
tpl = self.rng.choice(LONG_RIDDLE_TEMPLATES)
|
| 217 |
+
return tpl.format(clue=clue, desc=desc, cat=cat_name)
|
| 218 |
+
|
| 219 |
+
def __getitem__(self, idx: int) -> Dict[str, Any]:
|
| 220 |
+
ref_group = self.samples[idx]
|
| 221 |
+
image_id = ref_group[0]["image_id"]
|
| 222 |
+
img_info = self.images_by_id[image_id]
|
| 223 |
+
img_path = self.image_dir / img_info["file_name"]
|
| 224 |
+
|
| 225 |
+
image = Image.open(img_path).convert("RGB")
|
| 226 |
+
width, height = image.size
|
| 227 |
+
|
| 228 |
+
if self.split == "train" and len(ref_group) > 1:
|
| 229 |
+
k_max = min(len(ref_group), self.max_objects_per_image)
|
| 230 |
+
k_min = min(k_max, self.min_objects_per_image)
|
| 231 |
+
k = self.rng.randint(k_min, k_max)
|
| 232 |
+
chosen_refs = self.rng.sample(ref_group, k=k)
|
| 233 |
+
else:
|
| 234 |
+
chosen_refs = ref_group[: self.max_objects_per_image]
|
| 235 |
+
|
| 236 |
+
query_objects: List[Dict[str, str]] = []
|
| 237 |
+
gt_masks = []
|
| 238 |
+
gt_polygons = []
|
| 239 |
+
gt_bboxes = []
|
| 240 |
+
|
| 241 |
+
for obj_idx, ref in enumerate(chosen_refs):
|
| 242 |
+
ann = self.anns_by_id[ref["ann_id"]]
|
| 243 |
+
if self.split == "train":
|
| 244 |
+
desc = self.rng.choice(ref["sentences"])
|
| 245 |
+
if self.riddle_prob > 0.0 and self.rng.random() < self.riddle_prob:
|
| 246 |
+
desc = self._augment_as_riddle(desc, ref.get("category_id"))
|
| 247 |
+
else:
|
| 248 |
+
desc = ref["sentences"][0]
|
| 249 |
+
|
| 250 |
+
obj_id = f"obj_{obj_idx}"
|
| 251 |
+
query_objects.append({"id": obj_id, "description": desc})
|
| 252 |
+
|
| 253 |
+
mask_np = polygons_to_mask(ann["segmentation"], height=height, width=width)
|
| 254 |
+
gt_masks.append(torch.from_numpy(mask_np).to(torch.float32))
|
| 255 |
+
gt_polygons.append(ann["segmentation"])
|
| 256 |
+
gt_bboxes.append(ann.get("bbox", [0.0, 0.0, 0.0, 0.0]))
|
| 257 |
+
|
| 258 |
+
prompt_text, normalized_objects = build_segmentation_prompt(
|
| 259 |
+
{"objects": query_objects},
|
| 260 |
+
query_token=OBJECT_QUERY_TOKEN,
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
return {
|
| 264 |
+
"image_id": image_id,
|
| 265 |
+
"image_path": str(img_path),
|
| 266 |
+
"image": image,
|
| 267 |
+
"target_size": (height, width),
|
| 268 |
+
"query_json": {"objects": normalized_objects},
|
| 269 |
+
"prompt_text": prompt_text,
|
| 270 |
+
"gt_masks": torch.stack(gt_masks, dim=0), # [K, H, W]
|
| 271 |
+
"gt_polygons": gt_polygons,
|
| 272 |
+
"gt_bboxes": gt_bboxes,
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class RefCOCOJsonCollator:
|
| 277 |
+
"""
|
| 278 |
+
Collates a batch of (PIL Image, multi-object JSON prompt, ground-truth masks)
|
| 279 |
+
into Qwen2.5-VL processor tensors using the causal reasoning message order:
|
| 280 |
+
[System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>]
|
| 281 |
+
"""
|
| 282 |
+
|
| 283 |
+
def __init__(self, processor: Any, system_prompt: str = DEFAULT_SYSTEM_PROMPT):
|
| 284 |
+
self.processor = processor
|
| 285 |
+
self.system_prompt = system_prompt
|
| 286 |
+
|
| 287 |
+
def __call__(self, batch: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
|
| 288 |
+
texts: List[str] = []
|
| 289 |
+
images: List[Image.Image] = []
|
| 290 |
+
target_sizes: List[Tuple[int, int]] = []
|
| 291 |
+
gt_masks_list: List[torch.Tensor] = []
|
| 292 |
+
query_jsons: List[Dict[str, Any]] = []
|
| 293 |
+
gt_polygons_list: List[List[Any]] = []
|
| 294 |
+
|
| 295 |
+
for item in batch:
|
| 296 |
+
messages, _ = build_segmentation_messages(
|
| 297 |
+
image=item["image"],
|
| 298 |
+
query_input=item["query_json"],
|
| 299 |
+
system_prompt=self.system_prompt,
|
| 300 |
+
)
|
| 301 |
+
chat_text = self.processor.apply_chat_template(
|
| 302 |
+
messages,
|
| 303 |
+
tokenize=False,
|
| 304 |
+
add_generation_prompt=False,
|
| 305 |
+
)
|
| 306 |
+
texts.append(chat_text)
|
| 307 |
+
images.append(item["image"])
|
| 308 |
+
target_sizes.append(item["target_size"])
|
| 309 |
+
gt_masks_list.append(item["gt_masks"])
|
| 310 |
+
query_jsons.append(item["query_json"])
|
| 311 |
+
gt_polygons_list.append(item["gt_polygons"])
|
| 312 |
+
|
| 313 |
+
proc_inputs = self.processor(
|
| 314 |
+
text=texts,
|
| 315 |
+
images=images,
|
| 316 |
+
padding=True,
|
| 317 |
+
return_tensors="pt",
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
+
return {
|
| 321 |
+
"input_ids": proc_inputs["input_ids"],
|
| 322 |
+
"attention_mask": proc_inputs["attention_mask"],
|
| 323 |
+
"pixel_values": proc_inputs["pixel_values"],
|
| 324 |
+
"image_grid_thw": proc_inputs["image_grid_thw"],
|
| 325 |
+
"target_sizes": target_sizes,
|
| 326 |
+
"gt_masks_list": gt_masks_list,
|
| 327 |
+
"query_jsons": query_jsons,
|
| 328 |
+
"gt_polygons_list": gt_polygons_list,
|
| 329 |
+
}
|
example.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pathlib
|
| 2 |
+
from PIL import Image
|
| 3 |
+
from lis1_infer import LIS1Inferencer
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def main() -> None:
|
| 7 |
+
# 1. Instantiate the inference class and enable debug logging
|
| 8 |
+
inferencer = LIS1Inferencer()
|
| 9 |
+
inferencer.debug = True
|
| 10 |
+
|
| 11 |
+
# 2. Load Qwen3.5 backbone + LIS1 decoder + LoRA adapter (with 4-bit bitsandbytes quantization)
|
| 12 |
+
inferencer.load_model(
|
| 13 |
+
qwen_path="./Qwen3.5-9B",
|
| 14 |
+
lis_model_path="./LIS1-9B-forLora.safetensors",
|
| 15 |
+
lora_adapter="./LIS1-9B-LoraAdapter",
|
| 16 |
+
device="cuda",
|
| 17 |
+
max_pixels=524288,
|
| 18 |
+
load_in_4bit=True, # Or set load_in_8bit=True for 8-bit quantization
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
# 3. Load "image1.jpg" and define target queries
|
| 22 |
+
pil_images = [Image.open("image1.jpg").convert("RGB")]
|
| 23 |
+
queries_per_image = [["toast", "red sauce", "something you can drink"]]
|
| 24 |
+
|
| 25 |
+
# 4. Run inference (isolate_queries=True avoids multi-query cross-bleed while reusing cached ViT features)
|
| 26 |
+
outputs = inferencer.run_model(
|
| 27 |
+
images=pil_images,
|
| 28 |
+
queries=queries_per_image,
|
| 29 |
+
threshold=0.7,
|
| 30 |
+
batch_size=4,
|
| 31 |
+
isolate_queries=True, # Compute queries separately
|
| 32 |
+
cache_vision=True, # Cache vision when above is True for better performance
|
| 33 |
+
exclusive_masks=True, # Make masks not overlap, choose the best one
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
for img_idx, out in enumerate(outputs):
|
| 37 |
+
print(f"\nImage {img_idx} ({out['image_size']['width']}x{out['image_size']['height']}):")
|
| 38 |
+
for obj in out["objects"]:
|
| 39 |
+
print(
|
| 40 |
+
f" - [{obj['id']}] '{obj['description']}': "
|
| 41 |
+
f"score={obj['score']:.4f}, bbox={obj['bbox']}, area={obj['area']}"
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# 5. Render masks and bounding boxes onto the image and save
|
| 45 |
+
rendered_images = inferencer.render_image(
|
| 46 |
+
images=pil_images,
|
| 47 |
+
outputs=outputs,
|
| 48 |
+
draw_masks=True,
|
| 49 |
+
draw_boxes=True,
|
| 50 |
+
draw_labels=True,
|
| 51 |
+
draw_scores=True,
|
| 52 |
+
draw_contours=True,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
out_dir = pathlib.Path(".")
|
| 56 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 57 |
+
for i, img_full in enumerate(rendered_images):
|
| 58 |
+
full_path = out_dir / f"result_{i}_boxes_and_masks.png"
|
| 59 |
+
img_full.save(full_path)
|
| 60 |
+
print(f"Saved rendered output to: {full_path}")
|
| 61 |
+
|
| 62 |
+
# 6. Offload model from GPU/CPU memory when done
|
| 63 |
+
inferencer.offload_model()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
if __name__ == "__main__":
|
| 67 |
+
main()
|
image1.jpg
ADDED
|
Git LFS Details
|
lis1_infer.py
ADDED
|
@@ -0,0 +1,1461 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from concurrent.futures import Future, ThreadPoolExecutor
|
| 2 |
+
from collections import OrderedDict, deque
|
| 3 |
+
import copy
|
| 4 |
+
import gc
|
| 5 |
+
import hashlib
|
| 6 |
+
import pathlib
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
import warnings
|
| 10 |
+
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union
|
| 11 |
+
|
| 12 |
+
import cv2
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 16 |
+
from safetensors import safe_open
|
| 17 |
+
from safetensors.torch import load_file
|
| 18 |
+
|
| 19 |
+
from LIS1ModelCode import (
|
| 20 |
+
DEFAULT_SYSTEM_PROMPT,
|
| 21 |
+
OBJECT_QUERY_TOKEN,
|
| 22 |
+
Qwen2_5_VL_SinglePassSegmenter,
|
| 23 |
+
build_segmentation_messages,
|
| 24 |
+
mask_to_polygons,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# Distinct vibrant RGB palette for multi-object mask & box rendering
|
| 29 |
+
DEFAULT_PALETTE: List[Tuple[int, int, int]] = [
|
| 30 |
+
(255, 56, 56), # Red
|
| 31 |
+
(44, 160, 44), # Green
|
| 32 |
+
(31, 119, 180), # Blue
|
| 33 |
+
(255, 157, 151), # Coral
|
| 34 |
+
(255, 112, 31), # Orange
|
| 35 |
+
(148, 103, 189), # Purple
|
| 36 |
+
(23, 190, 207), # Cyan
|
| 37 |
+
(188, 189, 34), # Olive
|
| 38 |
+
(227, 119, 194), # Pink
|
| 39 |
+
(140, 86, 75), # Brown
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def inspect_lis_checkpoint(
|
| 44 |
+
lis_model_path: Union[str, pathlib.Path],
|
| 45 |
+
) -> Tuple[Dict[str, torch.Tensor], Optional[str], Dict[str, Any]]:
|
| 46 |
+
"""
|
| 47 |
+
Loads a LIS1 mask_head checkpoint from either `.pt` or `.safetensors` format.
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
mask_head_state: state_dict for `model.mask_head`
|
| 51 |
+
lora_dir: optional LoRA directory string stored in checkpoint metadata (if `forLora`)
|
| 52 |
+
raw_metadata: dictionary of raw metadata from the file
|
| 53 |
+
"""
|
| 54 |
+
ckpt_path = pathlib.Path(lis_model_path)
|
| 55 |
+
if not ckpt_path.exists():
|
| 56 |
+
raise FileNotFoundError(f"LIS model checkpoint not found: {ckpt_path}")
|
| 57 |
+
|
| 58 |
+
suffix = ckpt_path.suffix.lower()
|
| 59 |
+
if suffix == ".safetensors":
|
| 60 |
+
with safe_open(str(ckpt_path), framework="pt", device="cpu") as f:
|
| 61 |
+
metadata = dict(f.metadata() or {})
|
| 62 |
+
tensors = load_file(str(ckpt_path), device="cpu")
|
| 63 |
+
lora_dir = metadata.get("lora_dir")
|
| 64 |
+
if any(k.startswith("mask_head.") for k in tensors):
|
| 65 |
+
mask_head_state = {
|
| 66 |
+
(k[len("mask_head.") :] if k.startswith("mask_head.") else k): v
|
| 67 |
+
for k, v in tensors.items()
|
| 68 |
+
}
|
| 69 |
+
else:
|
| 70 |
+
mask_head_state = tensors
|
| 71 |
+
return mask_head_state, (lora_dir if lora_dir else None), metadata
|
| 72 |
+
|
| 73 |
+
# PyTorch .pt / .pth checkpoint
|
| 74 |
+
try:
|
| 75 |
+
state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
|
| 76 |
+
except TypeError:
|
| 77 |
+
state = torch.load(ckpt_path, map_location="cpu")
|
| 78 |
+
|
| 79 |
+
if isinstance(state, dict) and "mask_head" in state:
|
| 80 |
+
mask_head_state = state["mask_head"]
|
| 81 |
+
lora_dir = state.get("lora_dir")
|
| 82 |
+
metadata = {k: v for k, v in state.items() if k != "mask_head"}
|
| 83 |
+
elif isinstance(state, dict):
|
| 84 |
+
mask_head_state = state
|
| 85 |
+
lora_dir = state.get("lora_dir") if isinstance(state.get("lora_dir"), str) else None
|
| 86 |
+
metadata = {"lora_dir": lora_dir} if lora_dir else {}
|
| 87 |
+
else:
|
| 88 |
+
raise ValueError(f"Unsupported checkpoint structure in {ckpt_path}")
|
| 89 |
+
|
| 90 |
+
return mask_head_state, (str(lora_dir) if lora_dir else None), metadata
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _apply_max_pixels(processor: Any, max_pixels: int) -> None:
|
| 94 |
+
if max_pixels <= 0 or processor is None:
|
| 95 |
+
return
|
| 96 |
+
ip = getattr(processor, "image_processor", None)
|
| 97 |
+
if ip is None:
|
| 98 |
+
return
|
| 99 |
+
if hasattr(ip, "max_pixels") and getattr(ip, "max_pixels", None) is not None:
|
| 100 |
+
ip.max_pixels = int(max_pixels)
|
| 101 |
+
if hasattr(ip, "min_pixels") and getattr(ip, "min_pixels", None) is not None:
|
| 102 |
+
if ip.min_pixels > max_pixels:
|
| 103 |
+
ip.min_pixels = int(max_pixels)
|
| 104 |
+
size = getattr(ip, "size", None)
|
| 105 |
+
if size is not None:
|
| 106 |
+
if isinstance(size, dict) or hasattr(size, "get"):
|
| 107 |
+
if size.get("longest_edge") is not None:
|
| 108 |
+
size["longest_edge"] = int(max_pixels)
|
| 109 |
+
if size.get("shortest_edge") is not None and size["shortest_edge"] > max_pixels:
|
| 110 |
+
size["shortest_edge"] = int(max_pixels)
|
| 111 |
+
if size.get("max_pixels") is not None:
|
| 112 |
+
size["max_pixels"] = int(max_pixels)
|
| 113 |
+
if size.get("min_pixels") is not None and size["min_pixels"] > max_pixels:
|
| 114 |
+
size["min_pixels"] = int(max_pixels)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def filter_mask_components(
|
| 118 |
+
mask_np: np.ndarray,
|
| 119 |
+
min_component_ratio: float = 0.05,
|
| 120 |
+
min_component_area: int = 16,
|
| 121 |
+
) -> np.ndarray:
|
| 122 |
+
"""
|
| 123 |
+
Removes tiny disconnected foreground islands whose area is smaller than
|
| 124 |
+
`max(min_component_area, min_component_ratio * largest_component_area)`.
|
| 125 |
+
Single-component masks are returned unchanged so genuine small objects are preserved.
|
| 126 |
+
"""
|
| 127 |
+
if min_component_ratio <= 0.0 and min_component_area <= 0:
|
| 128 |
+
return mask_np
|
| 129 |
+
if not np.any(mask_np):
|
| 130 |
+
return mask_np
|
| 131 |
+
|
| 132 |
+
num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(mask_np, connectivity=8)
|
| 133 |
+
if num_labels <= 2:
|
| 134 |
+
return mask_np
|
| 135 |
+
|
| 136 |
+
fg_areas = stats[1:, cv2.CC_STAT_AREA]
|
| 137 |
+
max_area = int(fg_areas.max())
|
| 138 |
+
area_cutoff = max(float(min_component_area), float(min_component_ratio) * float(max_area))
|
| 139 |
+
|
| 140 |
+
keep_lut = np.zeros(num_labels, dtype=np.uint8)
|
| 141 |
+
for comp_idx in range(1, num_labels):
|
| 142 |
+
if stats[comp_idx, cv2.CC_STAT_AREA] >= area_cutoff:
|
| 143 |
+
keep_lut[comp_idx] = 1
|
| 144 |
+
return keep_lut[labels]
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _rects_overlap(
|
| 148 |
+
a: Tuple[float, float, float, float],
|
| 149 |
+
b: Tuple[float, float, float, float],
|
| 150 |
+
margin: float = 2.0,
|
| 151 |
+
) -> bool:
|
| 152 |
+
return not (
|
| 153 |
+
a[2] + margin <= b[0]
|
| 154 |
+
or b[2] + margin <= a[0]
|
| 155 |
+
or a[3] + margin <= b[1]
|
| 156 |
+
or b[3] + margin <= a[1]
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _place_non_overlapping_label(
|
| 161 |
+
x1: float,
|
| 162 |
+
y1: float,
|
| 163 |
+
label_w: float,
|
| 164 |
+
label_h: float,
|
| 165 |
+
img_w: int,
|
| 166 |
+
img_h: int,
|
| 167 |
+
occupied: List[Tuple[float, float, float, float]],
|
| 168 |
+
) -> Tuple[float, float, float, float]:
|
| 169 |
+
"""
|
| 170 |
+
Finds a non-overlapping rectangle (tx1, ty1, tx2, ty2) for a bounding-box label,
|
| 171 |
+
preferring above top-left of the box, then stacking upward or downward on collision.
|
| 172 |
+
"""
|
| 173 |
+
tx1 = max(0.0, min(x1, max(0.0, float(img_w) - label_w)))
|
| 174 |
+
tx2 = min(float(img_w), tx1 + label_w)
|
| 175 |
+
|
| 176 |
+
pref_ty1 = (y1 - label_h) if (y1 - label_h >= 0.0) else y1
|
| 177 |
+
pref_ty1 = max(0.0, min(pref_ty1, max(0.0, float(img_h) - label_h)))
|
| 178 |
+
cand = (tx1, pref_ty1, tx2, pref_ty1 + label_h)
|
| 179 |
+
|
| 180 |
+
if not any(_rects_overlap(cand, occ) for occ in occupied):
|
| 181 |
+
return cand
|
| 182 |
+
|
| 183 |
+
# 1. Try stacking upward above colliding labels
|
| 184 |
+
ty1 = pref_ty1
|
| 185 |
+
for _ in range(len(occupied) + 2):
|
| 186 |
+
hits = [occ for occ in occupied if _rects_overlap((tx1, ty1, tx2, ty1 + label_h), occ)]
|
| 187 |
+
if not hits:
|
| 188 |
+
return (tx1, ty1, tx2, ty1 + label_h)
|
| 189 |
+
min_occ_top = min(occ[1] for occ in hits)
|
| 190 |
+
ty1 = min_occ_top - label_h - 2.0
|
| 191 |
+
if ty1 < 0.0:
|
| 192 |
+
break
|
| 193 |
+
|
| 194 |
+
# 2. Otherwise stack downward starting just inside/below the top of the box
|
| 195 |
+
ty1 = max(0.0, y1 + 2.0)
|
| 196 |
+
for _ in range(len(occupied) + 4):
|
| 197 |
+
if ty1 + label_h > float(img_h):
|
| 198 |
+
ty1 = max(0.0, float(img_h) - label_h)
|
| 199 |
+
break
|
| 200 |
+
hits = [occ for occ in occupied if _rects_overlap((tx1, ty1, tx2, ty1 + label_h), occ)]
|
| 201 |
+
if not hits:
|
| 202 |
+
return (tx1, ty1, tx2, ty1 + label_h)
|
| 203 |
+
max_occ_bot = max(occ[3] for occ in hits)
|
| 204 |
+
ty1 = max_occ_bot + 2.0
|
| 205 |
+
|
| 206 |
+
return (tx1, ty1, tx2, min(float(img_h), ty1 + label_h))
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _compute_image_cache_key(pil_rgb: Image.Image, max_pixels: int) -> Tuple[int, int, int, bytes]:
|
| 210 |
+
arr = np.ascontiguousarray(np.asarray(pil_rgb, dtype=np.uint8))
|
| 211 |
+
digest = hashlib.blake2b(memoryview(arr), digest_size=16).digest()
|
| 212 |
+
w, h = pil_rgb.size
|
| 213 |
+
return (h, w, int(max_pixels), digest)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class LIS1Inferencer:
|
| 217 |
+
"""
|
| 218 |
+
High-efficiency single-pass inference wrapper for Qwen3.5 + LIS1 segmentation models.
|
| 219 |
+
|
| 220 |
+
Provides:
|
| 221 |
+
1. `load_model(qwen_path, lis_model_path, lora_adapter=None, device="cuda", ...)`
|
| 222 |
+
2. `offload_model()`
|
| 223 |
+
3. `run_model(images, queries, isolate_queries=False, cache_vision=True, ...)`
|
| 224 |
+
4. `render_image(images, outputs, ...)`
|
| 225 |
+
5. `debug` property + setter (`inferencer.debug = True` or `inferencer.set_debug(True)`)
|
| 226 |
+
"""
|
| 227 |
+
|
| 228 |
+
def __init__(self, debug: bool = False, vision_cache_size: int = 8) -> None:
|
| 229 |
+
self._debug: bool = bool(debug)
|
| 230 |
+
self.model: Optional[Qwen2_5_VL_SinglePassSegmenter] = None
|
| 231 |
+
self.device: torch.device = torch.device("cpu")
|
| 232 |
+
self.dtype: torch.dtype = torch.float32
|
| 233 |
+
self.quantization_mode: str = "none"
|
| 234 |
+
self.max_pixels: int = 0
|
| 235 |
+
self.vision_cache_size: int = max(0, int(vision_cache_size))
|
| 236 |
+
self._vision_cache: "OrderedDict[Tuple[int, int, int, bytes], Dict[str, Any]]" = OrderedDict()
|
| 237 |
+
self.qwen_path: Optional[pathlib.Path] = None
|
| 238 |
+
self.lis_model_path: Optional[pathlib.Path] = None
|
| 239 |
+
self.lora_adapter_path: Optional[pathlib.Path] = None
|
| 240 |
+
|
| 241 |
+
def clear_vision_cache(self) -> None:
|
| 242 |
+
"""Clears cached ViT visual features from memory."""
|
| 243 |
+
self._vision_cache.clear()
|
| 244 |
+
|
| 245 |
+
@property
|
| 246 |
+
def debug(self) -> bool:
|
| 247 |
+
return self._debug
|
| 248 |
+
|
| 249 |
+
@debug.setter
|
| 250 |
+
def debug(self, value: bool) -> None:
|
| 251 |
+
self._debug = bool(value)
|
| 252 |
+
if self._debug:
|
| 253 |
+
print("[LIS1 Debug] Debug mode enabled.", flush=True)
|
| 254 |
+
|
| 255 |
+
def set_debug(self, value: bool) -> None:
|
| 256 |
+
"""Setter method for enabling or disabling debug prints."""
|
| 257 |
+
self.debug = value
|
| 258 |
+
|
| 259 |
+
def _log(self, msg: str) -> None:
|
| 260 |
+
if self._debug:
|
| 261 |
+
print(f"[LIS1 Debug] {msg}", flush=True)
|
| 262 |
+
|
| 263 |
+
def load_model(
|
| 264 |
+
self,
|
| 265 |
+
qwen_path: Union[str, pathlib.Path],
|
| 266 |
+
lis_model_path: Union[str, pathlib.Path],
|
| 267 |
+
lora_adapter: Optional[Union[str, pathlib.Path]] = None,
|
| 268 |
+
device: Union[str, torch.device] = "cuda",
|
| 269 |
+
fused_layers: int = 4,
|
| 270 |
+
max_pixels: int = 0,
|
| 271 |
+
load_in_4bit: bool = False,
|
| 272 |
+
load_in_8bit: bool = False,
|
| 273 |
+
bnb_4bit_quant_type: str = "nf4",
|
| 274 |
+
bnb_4bit_use_double_quant: bool = True,
|
| 275 |
+
bnb_4bit_compute_dtype: Optional[torch.dtype] = None,
|
| 276 |
+
bnb_skip_vision: bool = True,
|
| 277 |
+
quantization_config: Optional[Any] = None,
|
| 278 |
+
) -> "LIS1Inferencer":
|
| 279 |
+
"""
|
| 280 |
+
Loads the Qwen3.5 backbone, the LIS1 mask_head weights (.pt or .safetensors),
|
| 281 |
+
and optionally the Stage-2 PEFT LoRA adapter and bitsandbytes quantization.
|
| 282 |
+
|
| 283 |
+
Args:
|
| 284 |
+
qwen_path: Path or HuggingFace ID of the base Qwen3.5 model.
|
| 285 |
+
lis_model_path: Path to the LIS1 mask_head checkpoint (.pt or .safetensors).
|
| 286 |
+
lora_adapter: Optional path to Stage-2 PEFT LoRA adapter directory.
|
| 287 |
+
If None, treats the LIS model as a noLora model, but prints a warning
|
| 288 |
+
if checkpoint metadata indicates it was trained for LoRA (`forLora`).
|
| 289 |
+
device: Target Torch device (e.g. "cuda", "cuda:0", "cpu", or "auto").
|
| 290 |
+
fused_layers: Number of top LLM layers fused into the mask head.
|
| 291 |
+
max_pixels: Optional cap on image pixels for faster ViT processing (0 = uncapped).
|
| 292 |
+
load_in_4bit: Enable bitsandbytes 4-bit quantization (NF4/FP4).
|
| 293 |
+
load_in_8bit: Enable bitsandbytes 8-bit quantization (LLM.int8()).
|
| 294 |
+
bnb_4bit_quant_type: 4-bit quantization data type ("nf4" or "fp4").
|
| 295 |
+
bnb_4bit_use_double_quant: Enable nested double quantization in 4-bit mode.
|
| 296 |
+
bnb_4bit_compute_dtype: Optional compute dtype override for 4-bit matmuls.
|
| 297 |
+
bnb_skip_vision: Keep the ViT `visual` encoder unquantized to preserve spatial mask precision.
|
| 298 |
+
quantization_config: Optional custom `transformers.BitsAndBytesConfig` instance.
|
| 299 |
+
"""
|
| 300 |
+
if load_in_4bit and load_in_8bit:
|
| 301 |
+
raise ValueError("Cannot enable both `load_in_4bit=True` and `load_in_8bit=True`. Choose one.")
|
| 302 |
+
|
| 303 |
+
if self.model is not None:
|
| 304 |
+
self._log("Existing model detected in memory; offloading before loading new model.")
|
| 305 |
+
self.offload_model()
|
| 306 |
+
|
| 307 |
+
t0 = time.perf_counter()
|
| 308 |
+
|
| 309 |
+
use_auto_device_map = isinstance(device, str) and device.strip().lower() == "auto"
|
| 310 |
+
if use_auto_device_map:
|
| 311 |
+
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
| 312 |
+
else:
|
| 313 |
+
req_device = torch.device(device)
|
| 314 |
+
if req_device.type == "cuda" and not torch.cuda.is_available():
|
| 315 |
+
print(
|
| 316 |
+
f"WARNING: CUDA device '{device}' requested but CUDA is not available. Falling back to CPU.",
|
| 317 |
+
flush=True,
|
| 318 |
+
)
|
| 319 |
+
req_device = torch.device("cpu")
|
| 320 |
+
self.device = req_device
|
| 321 |
+
|
| 322 |
+
use_bnb = bool(load_in_4bit or load_in_8bit or quantization_config is not None)
|
| 323 |
+
if quantization_config is not None:
|
| 324 |
+
resolved_bnb_cfg = quantization_config
|
| 325 |
+
is_4bit = bool(getattr(resolved_bnb_cfg, "load_in_4bit", False))
|
| 326 |
+
is_8bit = bool(getattr(resolved_bnb_cfg, "load_in_8bit", False))
|
| 327 |
+
self.quantization_mode = "4bit" if is_4bit else ("8bit" if is_8bit else "custom_bnb")
|
| 328 |
+
elif load_in_4bit or load_in_8bit:
|
| 329 |
+
try:
|
| 330 |
+
import accelerate # noqa: F401
|
| 331 |
+
import bitsandbytes # noqa: F401
|
| 332 |
+
from transformers import BitsAndBytesConfig
|
| 333 |
+
except ImportError as exc:
|
| 334 |
+
raise ImportError(
|
| 335 |
+
"bitsandbytes quantization requires `bitsandbytes` and `accelerate`. "
|
| 336 |
+
"Install them with: `pip install -U bitsandbytes accelerate`"
|
| 337 |
+
) from exc
|
| 338 |
+
resolved_bnb_cfg = None
|
| 339 |
+
is_4bit = bool(load_in_4bit)
|
| 340 |
+
is_8bit = bool(load_in_8bit)
|
| 341 |
+
self.quantization_mode = "4bit" if is_4bit else "8bit"
|
| 342 |
+
else:
|
| 343 |
+
resolved_bnb_cfg = None
|
| 344 |
+
is_4bit = False
|
| 345 |
+
is_8bit = False
|
| 346 |
+
self.quantization_mode = "none"
|
| 347 |
+
|
| 348 |
+
if self.device.type == "cuda":
|
| 349 |
+
if is_8bit:
|
| 350 |
+
self.dtype = torch.float16
|
| 351 |
+
elif is_4bit:
|
| 352 |
+
self.dtype = bnb_4bit_compute_dtype or (
|
| 353 |
+
torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
|
| 354 |
+
)
|
| 355 |
+
else:
|
| 356 |
+
self.dtype = (
|
| 357 |
+
torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
|
| 358 |
+
)
|
| 359 |
+
else:
|
| 360 |
+
# On CPU, float32 avoids 100x+ software-emulated fp16/bf16 matmul slowdowns on x86 CPUs
|
| 361 |
+
self.dtype = bnb_4bit_compute_dtype or torch.float32
|
| 362 |
+
|
| 363 |
+
if resolved_bnb_cfg is None and (load_in_4bit or load_in_8bit):
|
| 364 |
+
from transformers import BitsAndBytesConfig
|
| 365 |
+
|
| 366 |
+
# Protect Qwen3.5 tiny 16xD Gated DeltaNet decay gates (in_proj_a/b) and optionally ViT
|
| 367 |
+
skip_modules = ["in_proj_a", "in_proj_b"]
|
| 368 |
+
if bnb_skip_vision:
|
| 369 |
+
skip_modules.insert(0, "visual")
|
| 370 |
+
if load_in_4bit:
|
| 371 |
+
resolved_bnb_cfg = BitsAndBytesConfig(
|
| 372 |
+
load_in_4bit=True,
|
| 373 |
+
bnb_4bit_quant_type=bnb_4bit_quant_type,
|
| 374 |
+
bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,
|
| 375 |
+
bnb_4bit_compute_dtype=self.dtype,
|
| 376 |
+
llm_int8_skip_modules=skip_modules,
|
| 377 |
+
)
|
| 378 |
+
else:
|
| 379 |
+
resolved_bnb_cfg = BitsAndBytesConfig(
|
| 380 |
+
load_in_8bit=True,
|
| 381 |
+
llm_int8_skip_modules=skip_modules,
|
| 382 |
+
)
|
| 383 |
+
|
| 384 |
+
resolved_device_map: Optional[Union[str, Dict[str, Any]]]
|
| 385 |
+
if use_auto_device_map:
|
| 386 |
+
resolved_device_map = "auto"
|
| 387 |
+
elif use_bnb:
|
| 388 |
+
if self.device.type == "cuda":
|
| 389 |
+
dev_idx = (
|
| 390 |
+
self.device.index
|
| 391 |
+
if self.device.index is not None
|
| 392 |
+
else torch.cuda.current_device()
|
| 393 |
+
)
|
| 394 |
+
resolved_device_map = {"": f"cuda:{dev_idx}"}
|
| 395 |
+
else:
|
| 396 |
+
resolved_device_map = {"": str(self.device)}
|
| 397 |
+
else:
|
| 398 |
+
resolved_device_map = None
|
| 399 |
+
|
| 400 |
+
self.qwen_path = pathlib.Path(qwen_path)
|
| 401 |
+
self.lis_model_path = pathlib.Path(lis_model_path)
|
| 402 |
+
self.max_pixels = int(max_pixels)
|
| 403 |
+
self.clear_vision_cache()
|
| 404 |
+
|
| 405 |
+
# 1. Inspect LIS checkpoint (.pt or .safetensors) first to validate paths & metadata
|
| 406 |
+
mask_head_state, ckpt_lora_dir, _ = inspect_lis_checkpoint(self.lis_model_path)
|
| 407 |
+
has_lora_arg = lora_adapter is not None and str(lora_adapter).strip() != ""
|
| 408 |
+
|
| 409 |
+
if not has_lora_arg and ckpt_lora_dir:
|
| 410 |
+
warn_msg = (
|
| 411 |
+
f"WARNING: LIS model '{self.lis_model_path.name}' is a 'forLora' checkpoint "
|
| 412 |
+
f"(metadata contains lora_dir='{ckpt_lora_dir}'), but `lora_adapter` was set to None. "
|
| 413 |
+
"Proceeding without a LoRA adapter as a noLora model, which may reduce segmentation accuracy."
|
| 414 |
+
)
|
| 415 |
+
print(warn_msg, flush=True)
|
| 416 |
+
warnings.warn(warn_msg, UserWarning, stacklevel=2)
|
| 417 |
+
|
| 418 |
+
self._log(
|
| 419 |
+
f"Loading Qwen3.5 backbone from '{self.qwen_path}' on device={self.device} "
|
| 420 |
+
f"(dtype={self.dtype}, quantization={self.quantization_mode})..."
|
| 421 |
+
)
|
| 422 |
+
self.model = Qwen2_5_VL_SinglePassSegmenter(
|
| 423 |
+
qwen_model_name=str(self.qwen_path),
|
| 424 |
+
torch_dtype=self.dtype,
|
| 425 |
+
device_map=resolved_device_map,
|
| 426 |
+
quantization_config=resolved_bnb_cfg,
|
| 427 |
+
freeze_backbone=True,
|
| 428 |
+
num_fused_layers=fused_layers,
|
| 429 |
+
)
|
| 430 |
+
_apply_max_pixels(self.model.processor, max_pixels)
|
| 431 |
+
|
| 432 |
+
# 2. Load LIS mask_head weights (kept in float32 for crisp spatial decoding)
|
| 433 |
+
self._log(
|
| 434 |
+
f"Loading LIS mask head from '{self.lis_model_path}' ({self.lis_model_path.suffix})..."
|
| 435 |
+
)
|
| 436 |
+
self.model.mask_head.load_state_dict(mask_head_state, strict=True)
|
| 437 |
+
if resolved_device_map is None:
|
| 438 |
+
self.model.to(self.device)
|
| 439 |
+
self.model.mask_head.to(device=self.device, dtype=torch.float32)
|
| 440 |
+
|
| 441 |
+
# 3. Load LoRA adapter if provided
|
| 442 |
+
if has_lora_arg:
|
| 443 |
+
lora_cand = pathlib.Path(lora_adapter) # type: ignore[arg-type]
|
| 444 |
+
if not lora_cand.exists():
|
| 445 |
+
rel_cand = self.lis_model_path.parent / lora_cand
|
| 446 |
+
if rel_cand.exists():
|
| 447 |
+
lora_cand = rel_cand
|
| 448 |
+
else:
|
| 449 |
+
raise FileNotFoundError(
|
| 450 |
+
f"LoRA adapter directory not found at '{lora_cand}' or '{rel_cand}'."
|
| 451 |
+
)
|
| 452 |
+
|
| 453 |
+
self._log(f"Loading LoRA adapter from '{lora_cand.resolve()}'...")
|
| 454 |
+
from peft import PeftModel
|
| 455 |
+
|
| 456 |
+
if use_bnb or resolved_device_map is not None:
|
| 457 |
+
# Propagate top-level quantization flags to language_model so PEFT uses bnb LoRA layers
|
| 458 |
+
lm_sub = self.model.backbone.language_model
|
| 459 |
+
for q_attr in ("is_loaded_in_4bit", "is_loaded_in_8bit", "hf_quantizer", "quantization_method"):
|
| 460 |
+
if hasattr(self.model.qwen, q_attr) and not hasattr(lm_sub, q_attr):
|
| 461 |
+
setattr(lm_sub, q_attr, getattr(self.model.qwen, q_attr))
|
| 462 |
+
if hasattr(self.model.qwen.config, "quantization_config") and hasattr(lm_sub, "config"):
|
| 463 |
+
lm_sub.config.quantization_config = self.model.qwen.config.quantization_config
|
| 464 |
+
|
| 465 |
+
self.model.backbone.language_model = PeftModel.from_pretrained(
|
| 466 |
+
lm_sub, str(lora_cand)
|
| 467 |
+
)
|
| 468 |
+
else:
|
| 469 |
+
self.model.backbone.language_model = PeftModel.from_pretrained(
|
| 470 |
+
self.model.backbone.language_model, str(lora_cand)
|
| 471 |
+
).to(device=self.device, dtype=self.dtype)
|
| 472 |
+
self.model._register_llm_layer_hooks()
|
| 473 |
+
self.model._needs_mm_token_type_ids = self.model._check_needs_mm_token_type_ids()
|
| 474 |
+
self.lora_adapter_path = lora_cand
|
| 475 |
+
else:
|
| 476 |
+
self.lora_adapter_path = None
|
| 477 |
+
|
| 478 |
+
self.model.eval()
|
| 479 |
+
|
| 480 |
+
elapsed = (time.perf_counter() - t0) * 1000.0
|
| 481 |
+
self._log(
|
| 482 |
+
f"Model loaded successfully in {elapsed:.1f} ms "
|
| 483 |
+
f"(quantization={self.quantization_mode}, LoRA={'enabled' if self.lora_adapter_path else 'disabled'})."
|
| 484 |
+
)
|
| 485 |
+
return self
|
| 486 |
+
|
| 487 |
+
def offload_model(self) -> None:
|
| 488 |
+
"""
|
| 489 |
+
Unloads the Qwen3.5 backbone, LoRA adapter, and LIS mask head from memory
|
| 490 |
+
and frees GPU/CPU caches.
|
| 491 |
+
"""
|
| 492 |
+
t0 = time.perf_counter()
|
| 493 |
+
if self.model is not None:
|
| 494 |
+
# Remove forward hooks to break circular references before GC
|
| 495 |
+
if getattr(self.model, "_visual_hook", None) is not None:
|
| 496 |
+
try:
|
| 497 |
+
self.model._visual_hook.remove()
|
| 498 |
+
except Exception:
|
| 499 |
+
pass
|
| 500 |
+
for h in getattr(self.model, "_llm_hooks", []):
|
| 501 |
+
try:
|
| 502 |
+
h.remove()
|
| 503 |
+
except Exception:
|
| 504 |
+
pass
|
| 505 |
+
if hasattr(self.model, "_llm_hooks"):
|
| 506 |
+
self.model._llm_hooks.clear()
|
| 507 |
+
if hasattr(self.model, "_captured_visual"):
|
| 508 |
+
self.model._captured_visual.clear()
|
| 509 |
+
if hasattr(self.model, "_captured_llm_layers"):
|
| 510 |
+
self.model._captured_llm_layers.clear()
|
| 511 |
+
|
| 512 |
+
if self.quantization_mode == "none":
|
| 513 |
+
try:
|
| 514 |
+
self.model.to("cpu")
|
| 515 |
+
except Exception:
|
| 516 |
+
pass
|
| 517 |
+
del self.model
|
| 518 |
+
self.model = None
|
| 519 |
+
self.quantization_mode = "none"
|
| 520 |
+
|
| 521 |
+
self.clear_vision_cache()
|
| 522 |
+
gc.collect()
|
| 523 |
+
if torch.cuda.is_available():
|
| 524 |
+
torch.cuda.empty_cache()
|
| 525 |
+
torch.cuda.ipc_collect()
|
| 526 |
+
|
| 527 |
+
elapsed = (time.perf_counter() - t0) * 1000.0
|
| 528 |
+
self._log(f"Model offloaded from memory in {elapsed:.1f} ms.")
|
| 529 |
+
|
| 530 |
+
def _encode_image_vision(
|
| 531 |
+
self,
|
| 532 |
+
pil_rgb: Image.Image,
|
| 533 |
+
) -> Tuple[Dict[str, Any], bool, float]:
|
| 534 |
+
"""
|
| 535 |
+
Runs `image_processor` and the ViT `visual` encoder once for `pil_rgb` (or retrieves
|
| 536 |
+
from the LRU `_vision_cache`), returning `(cache_entry, cache_hit, elapsed_ms)`.
|
| 537 |
+
"""
|
| 538 |
+
assert self.model is not None
|
| 539 |
+
t0 = time.perf_counter()
|
| 540 |
+
|
| 541 |
+
cache_key: Optional[Tuple[int, int, int, bytes]] = None
|
| 542 |
+
if self.vision_cache_size > 0:
|
| 543 |
+
cache_key = _compute_image_cache_key(pil_rgb, self.max_pixels)
|
| 544 |
+
cached = self._vision_cache.get(cache_key)
|
| 545 |
+
if cached is not None:
|
| 546 |
+
self._vision_cache.move_to_end(cache_key)
|
| 547 |
+
return cached, True, (time.perf_counter() - t0) * 1000.0
|
| 548 |
+
|
| 549 |
+
proc = self.model.processor
|
| 550 |
+
vis_mod = self.model.backbone.visual
|
| 551 |
+
vis_param = next(vis_mod.parameters(), None)
|
| 552 |
+
vis_dtype = getattr(vis_mod, "dtype", None) or (
|
| 553 |
+
vis_param.dtype if vis_param is not None else self.dtype
|
| 554 |
+
)
|
| 555 |
+
vis_device = vis_param.device if vis_param is not None else self.device
|
| 556 |
+
|
| 557 |
+
ip_out = proc.image_processor(images=[pil_rgb], return_tensors="pt")
|
| 558 |
+
pv_1 = ip_out["pixel_values"].to(device=vis_device, dtype=vis_dtype)
|
| 559 |
+
gthw_1 = ip_out["image_grid_thw"].to(device=vis_device)
|
| 560 |
+
gthw_row = ip_out["image_grid_thw"][0].tolist()
|
| 561 |
+
|
| 562 |
+
self.model._captured_visual.clear()
|
| 563 |
+
_ = self.model.backbone.get_image_features(pv_1, gthw_1, return_dict=True)
|
| 564 |
+
merged_vis = self.model._captured_visual.get("merged")
|
| 565 |
+
unmerged_vis = self.model._captured_visual.get("unmerged")
|
| 566 |
+
if merged_vis is None:
|
| 567 |
+
raise RuntimeError("Visual forward hook did not capture features during vision encoding.")
|
| 568 |
+
|
| 569 |
+
merged_1 = (
|
| 570 |
+
torch.cat(list(merged_vis), dim=0)
|
| 571 |
+
if isinstance(merged_vis, (tuple, list))
|
| 572 |
+
else merged_vis
|
| 573 |
+
)
|
| 574 |
+
coarse_grids, fine_grids = self.model._reconstruct_2d_vision_features(
|
| 575 |
+
merged_tokens=merged_1,
|
| 576 |
+
unmerged_tokens=unmerged_vis,
|
| 577 |
+
image_grid_thw=gthw_1,
|
| 578 |
+
grid_thw_list=[gthw_row],
|
| 579 |
+
)
|
| 580 |
+
|
| 581 |
+
head_param = next(self.model.mask_head.parameters())
|
| 582 |
+
head_device = head_param.device
|
| 583 |
+
head_dtype = head_param.dtype
|
| 584 |
+
|
| 585 |
+
orig_w, orig_h = pil_rgb.size
|
| 586 |
+
entry: Dict[str, Any] = {
|
| 587 |
+
"merged": merged_1.detach(),
|
| 588 |
+
"f_coarse": coarse_grids[0].detach().to(device=head_device, dtype=head_dtype),
|
| 589 |
+
"f_fine": fine_grids[0].detach().to(device=head_device, dtype=head_dtype),
|
| 590 |
+
"grid_thw_1": gthw_1.detach(),
|
| 591 |
+
"grid_thw_row": gthw_row,
|
| 592 |
+
"num_image_tokens": int(merged_1.shape[0]),
|
| 593 |
+
"orig_size": (orig_h, orig_w),
|
| 594 |
+
}
|
| 595 |
+
|
| 596 |
+
if cache_key is not None and self.vision_cache_size > 0:
|
| 597 |
+
self._vision_cache[cache_key] = entry
|
| 598 |
+
while len(self._vision_cache) > self.vision_cache_size:
|
| 599 |
+
self._vision_cache.popitem(last=False)
|
| 600 |
+
|
| 601 |
+
return entry, False, (time.perf_counter() - t0) * 1000.0
|
| 602 |
+
|
| 603 |
+
def _prepare_batch(
|
| 604 |
+
self,
|
| 605 |
+
batch_images: Sequence[Image.Image],
|
| 606 |
+
batch_prompt_objs: Sequence[Sequence[Dict[str, str]]],
|
| 607 |
+
processor: Any,
|
| 608 |
+
lock: Optional[threading.Lock],
|
| 609 |
+
needs_mm_token_type_ids: bool,
|
| 610 |
+
image_token_id: int,
|
| 611 |
+
) -> Dict[str, Any]:
|
| 612 |
+
texts: List[str] = []
|
| 613 |
+
rgb_images: List[Image.Image] = []
|
| 614 |
+
orig_sizes: List[Tuple[int, int]] = []
|
| 615 |
+
batch_objects: List[List[Dict[str, str]]] = []
|
| 616 |
+
|
| 617 |
+
for img, query_objs in zip(batch_images, batch_prompt_objs):
|
| 618 |
+
pil_rgb = img.convert("RGB") if img.mode != "RGB" else img
|
| 619 |
+
orig_w, orig_h = pil_rgb.size
|
| 620 |
+
orig_sizes.append((orig_h, orig_w))
|
| 621 |
+
rgb_images.append(pil_rgb)
|
| 622 |
+
|
| 623 |
+
messages, norm_objs = build_segmentation_messages(
|
| 624 |
+
image=pil_rgb,
|
| 625 |
+
query_input={"objects": list(query_objs)},
|
| 626 |
+
system_prompt=DEFAULT_SYSTEM_PROMPT,
|
| 627 |
+
query_token=OBJECT_QUERY_TOKEN,
|
| 628 |
+
)
|
| 629 |
+
batch_objects.append(norm_objs)
|
| 630 |
+
|
| 631 |
+
if lock is not None:
|
| 632 |
+
with lock:
|
| 633 |
+
chat_text = processor.apply_chat_template(
|
| 634 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 635 |
+
)
|
| 636 |
+
else:
|
| 637 |
+
chat_text = processor.apply_chat_template(
|
| 638 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 639 |
+
)
|
| 640 |
+
texts.append(chat_text)
|
| 641 |
+
|
| 642 |
+
if lock is not None:
|
| 643 |
+
with lock:
|
| 644 |
+
inputs = processor(
|
| 645 |
+
text=texts,
|
| 646 |
+
images=rgb_images,
|
| 647 |
+
padding=True,
|
| 648 |
+
return_tensors="pt",
|
| 649 |
+
)
|
| 650 |
+
else:
|
| 651 |
+
inputs = processor(
|
| 652 |
+
text=texts,
|
| 653 |
+
images=rgb_images,
|
| 654 |
+
padding=True,
|
| 655 |
+
return_tensors="pt",
|
| 656 |
+
)
|
| 657 |
+
|
| 658 |
+
input_ids_cpu = inputs["input_ids"]
|
| 659 |
+
grid_thw_list = inputs["image_grid_thw"].tolist()
|
| 660 |
+
if needs_mm_token_type_ids:
|
| 661 |
+
inputs["mm_token_type_ids"] = (input_ids_cpu == image_token_id).to(dtype=torch.long)
|
| 662 |
+
|
| 663 |
+
return {
|
| 664 |
+
"inputs": inputs,
|
| 665 |
+
"input_ids_cpu": input_ids_cpu,
|
| 666 |
+
"grid_thw_list": grid_thw_list,
|
| 667 |
+
"orig_sizes": orig_sizes,
|
| 668 |
+
"batch_objects": batch_objects,
|
| 669 |
+
}
|
| 670 |
+
|
| 671 |
+
def _postprocess_task_outputs(
|
| 672 |
+
self,
|
| 673 |
+
probs_b: torch.Tensor,
|
| 674 |
+
scores_b: torch.Tensor,
|
| 675 |
+
output_meta_objs: Sequence[Dict[str, str]],
|
| 676 |
+
threshold: float,
|
| 677 |
+
min_component_ratio: float,
|
| 678 |
+
min_component_area: int,
|
| 679 |
+
) -> List[Dict[str, Any]]:
|
| 680 |
+
bin_masks_gpu = (probs_b >= threshold).to(torch.uint8)
|
| 681 |
+
bin_masks_np = bin_masks_gpu.cpu().numpy()
|
| 682 |
+
probs_np: Optional[np.ndarray] = None
|
| 683 |
+
scores_cpu = scores_b.detach().cpu().tolist()
|
| 684 |
+
|
| 685 |
+
task_results: List[Dict[str, Any]] = []
|
| 686 |
+
for q_idx, meta_obj in enumerate(output_meta_objs):
|
| 687 |
+
if q_idx >= bin_masks_np.shape[0]:
|
| 688 |
+
break
|
| 689 |
+
raw_mask_np = bin_masks_np[q_idx]
|
| 690 |
+
mask_np = filter_mask_components(
|
| 691 |
+
raw_mask_np,
|
| 692 |
+
min_component_ratio=min_component_ratio,
|
| 693 |
+
min_component_area=min_component_area,
|
| 694 |
+
)
|
| 695 |
+
polygons, bbox, area = mask_to_polygons(mask_np)
|
| 696 |
+
|
| 697 |
+
if area > 0:
|
| 698 |
+
if mask_np is raw_mask_np:
|
| 699 |
+
fg_prob = float(probs_b[q_idx][bin_masks_gpu[q_idx] > 0].mean().item())
|
| 700 |
+
else:
|
| 701 |
+
if probs_np is None:
|
| 702 |
+
probs_np = probs_b.detach().cpu().to(torch.float32).numpy()
|
| 703 |
+
fg_prob = float(probs_np[q_idx][mask_np > 0].mean())
|
| 704 |
+
else:
|
| 705 |
+
fg_prob = 0.0
|
| 706 |
+
|
| 707 |
+
pred_s = (
|
| 708 |
+
float(scores_cpu[q_idx])
|
| 709 |
+
if isinstance(scores_cpu, list)
|
| 710 |
+
else float(scores_b[q_idx].item())
|
| 711 |
+
)
|
| 712 |
+
score = round(0.5 * pred_s + 0.5 * fg_prob, 4)
|
| 713 |
+
|
| 714 |
+
x, y, w, h = bbox
|
| 715 |
+
bbox_xyxy = (
|
| 716 |
+
[round(x, 2), round(y, 2), round(x + w, 2), round(y + h, 2)]
|
| 717 |
+
if area > 0
|
| 718 |
+
else [0.0, 0.0, 0.0, 0.0]
|
| 719 |
+
)
|
| 720 |
+
|
| 721 |
+
task_results.append(
|
| 722 |
+
{
|
| 723 |
+
"id": meta_obj["id"],
|
| 724 |
+
"description": meta_obj["description"],
|
| 725 |
+
"score": score,
|
| 726 |
+
"bbox": bbox,
|
| 727 |
+
"bbox_xyxy": bbox_xyxy,
|
| 728 |
+
"area": area,
|
| 729 |
+
"mask": mask_np,
|
| 730 |
+
"segmentation": polygons,
|
| 731 |
+
}
|
| 732 |
+
)
|
| 733 |
+
return task_results
|
| 734 |
+
|
| 735 |
+
def _finalize_exclusive_image_outputs(
|
| 736 |
+
self,
|
| 737 |
+
items: Sequence[Tuple[np.ndarray, float, Dict[str, str]]],
|
| 738 |
+
threshold: float,
|
| 739 |
+
min_component_ratio: float,
|
| 740 |
+
min_component_area: int,
|
| 741 |
+
) -> List[Dict[str, Any]]:
|
| 742 |
+
"""
|
| 743 |
+
Resolves pixel-level mask overlaps across all queries on the same image via
|
| 744 |
+
winner-takes-all (`argmax` over pixel probabilities among queries >= `threshold`).
|
| 745 |
+
"""
|
| 746 |
+
num_q = len(items)
|
| 747 |
+
if num_q == 0:
|
| 748 |
+
return []
|
| 749 |
+
|
| 750 |
+
probs_stack = np.stack([it[0] for it in items], axis=0) # [K, H, W] float32
|
| 751 |
+
eff_probs = np.full_like(probs_stack, -1.0)
|
| 752 |
+
for k in range(num_q):
|
| 753 |
+
raw_bin = (probs_stack[k] >= threshold).astype(np.uint8)
|
| 754 |
+
clean_bin = filter_mask_components(
|
| 755 |
+
raw_bin,
|
| 756 |
+
min_component_ratio=min_component_ratio,
|
| 757 |
+
min_component_area=min_component_area,
|
| 758 |
+
)
|
| 759 |
+
eff_probs[k] = np.where(clean_bin > 0, probs_stack[k], -1.0)
|
| 760 |
+
|
| 761 |
+
winner_idx = np.argmax(eff_probs, axis=0)
|
| 762 |
+
max_eff_prob = np.max(eff_probs, axis=0)
|
| 763 |
+
valid_fg = max_eff_prob >= threshold
|
| 764 |
+
|
| 765 |
+
img_results: List[Dict[str, Any]] = []
|
| 766 |
+
for k, (prob_np, pred_s, meta_obj) in enumerate(items):
|
| 767 |
+
win_mask = ((winner_idx == k) & valid_fg).astype(np.uint8)
|
| 768 |
+
mask_np = filter_mask_components(
|
| 769 |
+
win_mask,
|
| 770 |
+
min_component_ratio=min_component_ratio,
|
| 771 |
+
min_component_area=min_component_area,
|
| 772 |
+
)
|
| 773 |
+
polygons, bbox, area = mask_to_polygons(mask_np)
|
| 774 |
+
|
| 775 |
+
if area > 0:
|
| 776 |
+
fg_prob = float(prob_np[mask_np > 0].mean())
|
| 777 |
+
else:
|
| 778 |
+
fg_prob = 0.0
|
| 779 |
+
score = round(0.5 * float(pred_s) + 0.5 * fg_prob, 4)
|
| 780 |
+
|
| 781 |
+
x, y, w, h = bbox
|
| 782 |
+
bbox_xyxy = (
|
| 783 |
+
[round(x, 2), round(y, 2), round(x + w, 2), round(y + h, 2)]
|
| 784 |
+
if area > 0
|
| 785 |
+
else [0.0, 0.0, 0.0, 0.0]
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
img_results.append(
|
| 789 |
+
{
|
| 790 |
+
"id": meta_obj["id"],
|
| 791 |
+
"description": meta_obj["description"],
|
| 792 |
+
"score": score,
|
| 793 |
+
"bbox": bbox,
|
| 794 |
+
"bbox_xyxy": bbox_xyxy,
|
| 795 |
+
"area": area,
|
| 796 |
+
"mask": mask_np,
|
| 797 |
+
"segmentation": polygons,
|
| 798 |
+
}
|
| 799 |
+
)
|
| 800 |
+
return img_results
|
| 801 |
+
|
| 802 |
+
@torch.no_grad()
|
| 803 |
+
def run_model(
|
| 804 |
+
self,
|
| 805 |
+
images: Sequence[Image.Image],
|
| 806 |
+
queries: Sequence[Sequence[str]],
|
| 807 |
+
threshold: float = 0.5,
|
| 808 |
+
batch_size: int = 4,
|
| 809 |
+
num_workers: int = 2,
|
| 810 |
+
isolate_queries: bool = False,
|
| 811 |
+
cache_vision: bool = True,
|
| 812 |
+
exclusive_masks: bool = False,
|
| 813 |
+
min_component_ratio: float = 0.05,
|
| 814 |
+
min_component_area: int = 16,
|
| 815 |
+
) -> List[Dict[str, Any]]:
|
| 816 |
+
"""
|
| 817 |
+
Runs single-pass segmentation over a list of PIL images and a list of query string lists.
|
| 818 |
+
|
| 819 |
+
Args:
|
| 820 |
+
images: List of PIL.Image.Image (length N).
|
| 821 |
+
queries: List of lists of query/riddle strings for each image (length N).
|
| 822 |
+
threshold: Binary mask probability threshold in [0, 1] (default 0.5).
|
| 823 |
+
batch_size: Maximum number of prompts processed simultaneously in one LLM batch.
|
| 824 |
+
num_workers: Background CPU threads used for uncached batch preprocessing.
|
| 825 |
+
isolate_queries: Optional flag. When True, runs each query in its own isolated prompt
|
| 826 |
+
(matching ReasonSeg benchmark evaluation and preventing multi-query cross-bleed),
|
| 827 |
+
while sharing a single ViT vision pass per image when `cache_vision=True`.
|
| 828 |
+
cache_vision: Reuse `image_processor` and ViT visual features across queries on the
|
| 829 |
+
same image (and across consecutive `run_model` calls via an LRU cache).
|
| 830 |
+
exclusive_masks: Optional flag. When True, assigns each pixel to at most one query
|
| 831 |
+
(pixel-wise winner-takes-all `argmax` probability among queries >= `threshold`)
|
| 832 |
+
so predicted masks on the same image never overlap.
|
| 833 |
+
min_component_ratio: Drops disconnected stray mask islands smaller than this fraction
|
| 834 |
+
of the largest component's area (default 0.05; set 0.0 to disable).
|
| 835 |
+
min_component_area: Minimum pixel area floor when filtering multi-component masks.
|
| 836 |
+
|
| 837 |
+
Returns:
|
| 838 |
+
List of dicts (one per image), where each dict contains:
|
| 839 |
+
- "image_size": {"height": H, "width": W}
|
| 840 |
+
- "objects": List of dicts with keys:
|
| 841 |
+
"id", "description", "score", "bbox" ([x, y, w, h]),
|
| 842 |
+
"bbox_xyxy" ([x1, y1, x2, y2]), "area",
|
| 843 |
+
"mask" (np.ndarray of shape (H, W) uint8 in {0, 1}),
|
| 844 |
+
"segmentation" (COCO polygons)
|
| 845 |
+
"""
|
| 846 |
+
if self.model is None:
|
| 847 |
+
raise RuntimeError("Model is not loaded. Call `load_model(...)` before `run_model(...)`.")
|
| 848 |
+
|
| 849 |
+
if isinstance(images, Image.Image):
|
| 850 |
+
images = [images]
|
| 851 |
+
if queries and isinstance(queries[0], str):
|
| 852 |
+
queries = [queries] # type: ignore[list-item]
|
| 853 |
+
|
| 854 |
+
if len(images) != len(queries):
|
| 855 |
+
raise ValueError(
|
| 856 |
+
f"Length mismatch: got {len(images)} images and {len(queries)} query lists."
|
| 857 |
+
)
|
| 858 |
+
|
| 859 |
+
num_images = len(images)
|
| 860 |
+
if num_images == 0:
|
| 861 |
+
return []
|
| 862 |
+
|
| 863 |
+
self.model.eval()
|
| 864 |
+
t_start = time.perf_counter()
|
| 865 |
+
is_cuda = self.device.type == "cuda"
|
| 866 |
+
|
| 867 |
+
rgb_images_all: List[Image.Image] = [
|
| 868 |
+
(img.convert("RGB") if img.mode != "RGB" else img) for img in images
|
| 869 |
+
]
|
| 870 |
+
results: List[Dict[str, Any]] = []
|
| 871 |
+
tasks: List[Dict[str, Any]] = []
|
| 872 |
+
valid_img_indices: List[int] = []
|
| 873 |
+
expected_query_counts: Dict[int, int] = {}
|
| 874 |
+
|
| 875 |
+
for idx, (pil_rgb, q_list) in enumerate(zip(rgb_images_all, queries)):
|
| 876 |
+
w, h = pil_rgb.size
|
| 877 |
+
results.append({"image_size": {"height": h, "width": w}, "objects": []})
|
| 878 |
+
|
| 879 |
+
cleaned_q = [str(q).strip() for q in q_list if str(q).strip()]
|
| 880 |
+
if not cleaned_q:
|
| 881 |
+
continue
|
| 882 |
+
valid_img_indices.append(idx)
|
| 883 |
+
expected_query_counts[idx] = len(cleaned_q)
|
| 884 |
+
|
| 885 |
+
if isolate_queries:
|
| 886 |
+
for q_i, desc in enumerate(cleaned_q):
|
| 887 |
+
tasks.append(
|
| 888 |
+
{
|
| 889 |
+
"img_idx": idx,
|
| 890 |
+
"prompt_objs": [{"id": "0", "description": desc}],
|
| 891 |
+
"meta_objs": [{"id": str(q_i), "description": desc}],
|
| 892 |
+
}
|
| 893 |
+
)
|
| 894 |
+
else:
|
| 895 |
+
obj_specs = [
|
| 896 |
+
{"id": str(q_i), "description": desc}
|
| 897 |
+
for q_i, desc in enumerate(cleaned_q)
|
| 898 |
+
]
|
| 899 |
+
tasks.append(
|
| 900 |
+
{
|
| 901 |
+
"img_idx": idx,
|
| 902 |
+
"prompt_objs": obj_specs,
|
| 903 |
+
"meta_objs": obj_specs,
|
| 904 |
+
}
|
| 905 |
+
)
|
| 906 |
+
|
| 907 |
+
if not tasks:
|
| 908 |
+
return results
|
| 909 |
+
|
| 910 |
+
pending_exclusive: Dict[int, List[Tuple[np.ndarray, float, Dict[str, str]]]] = {}
|
| 911 |
+
|
| 912 |
+
def _record_task_output(
|
| 913 |
+
task: Dict[str, Any],
|
| 914 |
+
probs_b: torch.Tensor,
|
| 915 |
+
scores_b: torch.Tensor,
|
| 916 |
+
) -> None:
|
| 917 |
+
img_idx = task["img_idx"]
|
| 918 |
+
meta_objs = task["meta_objs"]
|
| 919 |
+
if exclusive_masks and expected_query_counts.get(img_idx, 0) > 1:
|
| 920 |
+
probs_np = probs_b.detach().cpu().to(torch.float32).numpy()
|
| 921 |
+
scores_cpu = scores_b.detach().cpu().tolist()
|
| 922 |
+
buf = pending_exclusive.setdefault(img_idx, [])
|
| 923 |
+
for q_idx, meta_obj in enumerate(meta_objs):
|
| 924 |
+
if q_idx >= probs_np.shape[0]:
|
| 925 |
+
break
|
| 926 |
+
pred_s = (
|
| 927 |
+
float(scores_cpu[q_idx])
|
| 928 |
+
if isinstance(scores_cpu, list)
|
| 929 |
+
else float(scores_b[q_idx].item())
|
| 930 |
+
)
|
| 931 |
+
buf.append((probs_np[q_idx], pred_s, meta_obj))
|
| 932 |
+
if len(buf) >= expected_query_counts[img_idx]:
|
| 933 |
+
results[img_idx]["objects"] = self._finalize_exclusive_image_outputs(
|
| 934 |
+
items=buf,
|
| 935 |
+
threshold=threshold,
|
| 936 |
+
min_component_ratio=min_component_ratio,
|
| 937 |
+
min_component_area=min_component_area,
|
| 938 |
+
)
|
| 939 |
+
del pending_exclusive[img_idx]
|
| 940 |
+
else:
|
| 941 |
+
task_objs = self._postprocess_task_outputs(
|
| 942 |
+
probs_b=probs_b,
|
| 943 |
+
scores_b=scores_b,
|
| 944 |
+
output_meta_objs=meta_objs,
|
| 945 |
+
threshold=threshold,
|
| 946 |
+
min_component_ratio=min_component_ratio,
|
| 947 |
+
min_component_area=min_component_area,
|
| 948 |
+
)
|
| 949 |
+
results[img_idx]["objects"].extend(task_objs)
|
| 950 |
+
|
| 951 |
+
eff_batch_size = max(1, int(batch_size))
|
| 952 |
+
task_chunks: List[List[Dict[str, Any]]] = [
|
| 953 |
+
tasks[i : i + eff_batch_size] for i in range(0, len(tasks), eff_batch_size)
|
| 954 |
+
]
|
| 955 |
+
|
| 956 |
+
needs_mm = getattr(self.model, "_needs_mm_token_type_ids", False)
|
| 957 |
+
image_token_id = self.model.image_token_id
|
| 958 |
+
base_processor = self.model.processor
|
| 959 |
+
|
| 960 |
+
can_use_vision_cache = bool(
|
| 961 |
+
cache_vision
|
| 962 |
+
and base_processor is not None
|
| 963 |
+
and hasattr(base_processor, "image_processor")
|
| 964 |
+
and hasattr(base_processor, "tokenizer")
|
| 965 |
+
and hasattr(self.model.backbone, "get_image_features")
|
| 966 |
+
and hasattr(self.model.backbone, "get_input_embeddings")
|
| 967 |
+
)
|
| 968 |
+
|
| 969 |
+
total_queries = sum(len(t["meta_objs"]) for t in tasks)
|
| 970 |
+
self._log(
|
| 971 |
+
f"Starting inference on {len(valid_img_indices)} image(s) with {total_queries} query(ies) "
|
| 972 |
+
f"-> {len(tasks)} prompt(s) across {len(task_chunks)} batch(es) "
|
| 973 |
+
f"(batch_size={eff_batch_size}, isolate_queries={isolate_queries}, "
|
| 974 |
+
f"cache_vision={can_use_vision_cache}, exclusive_masks={exclusive_masks})."
|
| 975 |
+
)
|
| 976 |
+
|
| 977 |
+
total_prep_ms = 0.0
|
| 978 |
+
total_vit_ms = 0.0
|
| 979 |
+
total_forward_ms = 0.0
|
| 980 |
+
total_gpu_ms = 0.0
|
| 981 |
+
total_post_ms = 0.0
|
| 982 |
+
|
| 983 |
+
# =====================================================================
|
| 984 |
+
# Path A: Vision-Cached Execution (1x ViT per unique image + batched LLM)
|
| 985 |
+
# =====================================================================
|
| 986 |
+
if can_use_vision_cache:
|
| 987 |
+
vis_entries_by_img: Dict[int, Dict[str, Any]] = {}
|
| 988 |
+
cache_hits = 0
|
| 989 |
+
cache_misses = 0
|
| 990 |
+
for img_idx in valid_img_indices:
|
| 991 |
+
entry, hit, vit_ms = self._encode_image_vision(rgb_images_all[img_idx])
|
| 992 |
+
vis_entries_by_img[img_idx] = entry
|
| 993 |
+
total_vit_ms += vit_ms
|
| 994 |
+
if hit:
|
| 995 |
+
cache_hits += 1
|
| 996 |
+
else:
|
| 997 |
+
cache_misses += 1
|
| 998 |
+
|
| 999 |
+
if self._debug:
|
| 1000 |
+
self._log(
|
| 1001 |
+
f"Vision encoding: {total_vit_ms:.1f} ms across {len(valid_img_indices)} image(s) "
|
| 1002 |
+
f"(cache_hits={cache_hits}, cache_misses={cache_misses})."
|
| 1003 |
+
)
|
| 1004 |
+
|
| 1005 |
+
img_tok_str = getattr(base_processor, "image_token", "<|image_pad|>")
|
| 1006 |
+
head_param = next(self.model.mask_head.parameters())
|
| 1007 |
+
head_device = head_param.device
|
| 1008 |
+
head_dtype = head_param.dtype
|
| 1009 |
+
|
| 1010 |
+
for batch_num, chunk in enumerate(task_chunks, start=1):
|
| 1011 |
+
tp0 = time.perf_counter()
|
| 1012 |
+
expanded_texts: List[str] = []
|
| 1013 |
+
chunk_vis: List[Dict[str, Any]] = []
|
| 1014 |
+
|
| 1015 |
+
for task in chunk:
|
| 1016 |
+
img_idx = task["img_idx"]
|
| 1017 |
+
vis_entry = vis_entries_by_img[img_idx]
|
| 1018 |
+
chunk_vis.append(vis_entry)
|
| 1019 |
+
messages, _ = build_segmentation_messages(
|
| 1020 |
+
image=rgb_images_all[img_idx],
|
| 1021 |
+
query_input={"objects": task["prompt_objs"]},
|
| 1022 |
+
system_prompt=DEFAULT_SYSTEM_PROMPT,
|
| 1023 |
+
query_token=OBJECT_QUERY_TOKEN,
|
| 1024 |
+
)
|
| 1025 |
+
chat_text = base_processor.apply_chat_template(
|
| 1026 |
+
messages, tokenize=False, add_generation_prompt=False
|
| 1027 |
+
)
|
| 1028 |
+
expanded_texts.append(
|
| 1029 |
+
chat_text.replace(
|
| 1030 |
+
img_tok_str, img_tok_str * vis_entry["num_image_tokens"], 1
|
| 1031 |
+
)
|
| 1032 |
+
)
|
| 1033 |
+
|
| 1034 |
+
tok_out = base_processor.tokenizer(
|
| 1035 |
+
expanded_texts, padding=True, return_tensors="pt"
|
| 1036 |
+
)
|
| 1037 |
+
input_ids_cpu = tok_out["input_ids"]
|
| 1038 |
+
attention_mask_cpu = tok_out["attention_mask"]
|
| 1039 |
+
mm_ids_cpu = (
|
| 1040 |
+
(input_ids_cpu == image_token_id).to(dtype=torch.long)
|
| 1041 |
+
if needs_mm
|
| 1042 |
+
else None
|
| 1043 |
+
)
|
| 1044 |
+
prep_ms = (time.perf_counter() - tp0) * 1000.0
|
| 1045 |
+
total_prep_ms += prep_ms
|
| 1046 |
+
|
| 1047 |
+
t_fwd0 = time.perf_counter()
|
| 1048 |
+
ev_start = None
|
| 1049 |
+
ev_end = None
|
| 1050 |
+
if is_cuda and self._debug:
|
| 1051 |
+
ev_start = torch.cuda.Event(enable_timing=True)
|
| 1052 |
+
ev_end = torch.cuda.Event(enable_timing=True)
|
| 1053 |
+
ev_start.record()
|
| 1054 |
+
|
| 1055 |
+
input_ids = input_ids_cpu.to(self.device, non_blocking=is_cuda)
|
| 1056 |
+
attention_mask = attention_mask_cpu.to(self.device, non_blocking=is_cuda)
|
| 1057 |
+
batch_grid_thw = torch.cat(
|
| 1058 |
+
[v["grid_thw_1"] for v in chunk_vis], dim=0
|
| 1059 |
+
).to(device=self.device)
|
| 1060 |
+
grid_thw_list = [v["grid_thw_row"] for v in chunk_vis]
|
| 1061 |
+
|
| 1062 |
+
inputs_embeds = self.model.backbone.get_input_embeddings()(input_ids)
|
| 1063 |
+
merged_cat = torch.cat(
|
| 1064 |
+
[
|
| 1065 |
+
v["merged"].to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
|
| 1066 |
+
for v in chunk_vis
|
| 1067 |
+
],
|
| 1068 |
+
dim=0,
|
| 1069 |
+
)
|
| 1070 |
+
img_mask = (input_ids == image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
|
| 1071 |
+
inputs_embeds = inputs_embeds.masked_scatter(img_mask, merged_cat)
|
| 1072 |
+
|
| 1073 |
+
bb_kwargs: Dict[str, Any] = {
|
| 1074 |
+
"input_ids": input_ids,
|
| 1075 |
+
"inputs_embeds": inputs_embeds,
|
| 1076 |
+
"attention_mask": attention_mask,
|
| 1077 |
+
"pixel_values": None,
|
| 1078 |
+
"image_grid_thw": batch_grid_thw,
|
| 1079 |
+
"use_cache": False,
|
| 1080 |
+
"output_hidden_states": False,
|
| 1081 |
+
"return_dict": True,
|
| 1082 |
+
}
|
| 1083 |
+
if needs_mm and mm_ids_cpu is not None:
|
| 1084 |
+
bb_kwargs["mm_token_type_ids"] = mm_ids_cpu.to(
|
| 1085 |
+
self.device, non_blocking=is_cuda
|
| 1086 |
+
)
|
| 1087 |
+
|
| 1088 |
+
self.model._captured_llm_layers.clear()
|
| 1089 |
+
llm_outputs = self.model.backbone(**bb_kwargs)
|
| 1090 |
+
last_hs = llm_outputs.last_hidden_state
|
| 1091 |
+
|
| 1092 |
+
llm_vis_grids = self.model._extract_llm_visual_grids(
|
| 1093 |
+
last_hidden_states=last_hs,
|
| 1094 |
+
input_ids=input_ids,
|
| 1095 |
+
image_grid_thw=batch_grid_thw,
|
| 1096 |
+
layer_states=self.model._captured_llm_layers,
|
| 1097 |
+
grid_thw_list=grid_thw_list,
|
| 1098 |
+
input_ids_cpu=input_ids_cpu,
|
| 1099 |
+
)
|
| 1100 |
+
batch_queries = self.model._extract_query_vectors(
|
| 1101 |
+
last_hidden_states=last_hs,
|
| 1102 |
+
input_ids=input_ids,
|
| 1103 |
+
layer_states=self.model._captured_llm_layers,
|
| 1104 |
+
input_ids_cpu=input_ids_cpu,
|
| 1105 |
+
)
|
| 1106 |
+
|
| 1107 |
+
probs_list: List[torch.Tensor] = []
|
| 1108 |
+
scores_list: List[torch.Tensor] = []
|
| 1109 |
+
for b_pos, vis_entry in enumerate(chunk_vis):
|
| 1110 |
+
f_llm_b = (
|
| 1111 |
+
llm_vis_grids[b_pos].to(device=head_device, dtype=head_dtype)
|
| 1112 |
+
if llm_vis_grids[b_pos] is not None
|
| 1113 |
+
else None
|
| 1114 |
+
)
|
| 1115 |
+
logits_b, scores_b = self.model.mask_head.forward_single_image(
|
| 1116 |
+
queries=batch_queries[b_pos].to(device=head_device, dtype=head_dtype),
|
| 1117 |
+
f_coarse=vis_entry["f_coarse"],
|
| 1118 |
+
f_fine=vis_entry["f_fine"],
|
| 1119 |
+
target_size=vis_entry["orig_size"],
|
| 1120 |
+
f_llm_coarse=f_llm_b,
|
| 1121 |
+
)
|
| 1122 |
+
probs_list.append(torch.sigmoid(logits_b))
|
| 1123 |
+
scores_list.append(scores_b)
|
| 1124 |
+
|
| 1125 |
+
if ev_end is not None and ev_start is not None:
|
| 1126 |
+
ev_end.record()
|
| 1127 |
+
torch.cuda.synchronize(self.device)
|
| 1128 |
+
batch_gpu_ms = float(ev_start.elapsed_time(ev_end))
|
| 1129 |
+
total_gpu_ms += batch_gpu_ms
|
| 1130 |
+
else:
|
| 1131 |
+
batch_gpu_ms = 0.0
|
| 1132 |
+
|
| 1133 |
+
batch_fwd_ms = (time.perf_counter() - t_fwd0) * 1000.0
|
| 1134 |
+
total_forward_ms += batch_fwd_ms
|
| 1135 |
+
|
| 1136 |
+
t_post0 = time.perf_counter()
|
| 1137 |
+
for b_pos, task in enumerate(chunk):
|
| 1138 |
+
_record_task_output(
|
| 1139 |
+
task=task,
|
| 1140 |
+
probs_b=probs_list[b_pos],
|
| 1141 |
+
scores_b=scores_list[b_pos],
|
| 1142 |
+
)
|
| 1143 |
+
|
| 1144 |
+
batch_post_ms = (time.perf_counter() - t_post0) * 1000.0
|
| 1145 |
+
total_post_ms += batch_post_ms
|
| 1146 |
+
|
| 1147 |
+
if self._debug:
|
| 1148 |
+
gpu_info = f", gpu_forward={batch_gpu_ms:.1f} ms" if is_cuda else ""
|
| 1149 |
+
self._log(
|
| 1150 |
+
f"Batch {batch_num}/{len(task_chunks)} ({len(chunk)} prompt(s)): "
|
| 1151 |
+
f"prep={prep_ms:.1f} ms, llm+head={batch_fwd_ms:.1f} ms{gpu_info}, "
|
| 1152 |
+
f"post={batch_post_ms:.1f} ms"
|
| 1153 |
+
)
|
| 1154 |
+
|
| 1155 |
+
total_ms = (time.perf_counter() - t_start) * 1000.0
|
| 1156 |
+
if self._debug:
|
| 1157 |
+
per_img_ms = total_ms / max(len(valid_img_indices), 1)
|
| 1158 |
+
self._log(
|
| 1159 |
+
f"Inference finished in {total_ms:.1f} ms ({per_img_ms:.1f} ms/img) | "
|
| 1160 |
+
f"vision={total_vit_ms:.1f} ms, prep={total_prep_ms:.1f} ms, "
|
| 1161 |
+
f"llm+head={total_forward_ms:.1f} ms, post={total_post_ms:.1f} ms."
|
| 1162 |
+
)
|
| 1163 |
+
return results
|
| 1164 |
+
|
| 1165 |
+
# =====================================================================
|
| 1166 |
+
# Path B: Uncached Standard Execution (`cache_vision=False` or custom processor)
|
| 1167 |
+
# =====================================================================
|
| 1168 |
+
use_threads = num_workers > 0 and len(task_chunks) > 1
|
| 1169 |
+
tls = threading.local()
|
| 1170 |
+
fallback_lock = threading.Lock()
|
| 1171 |
+
deepcopy_ok = {"ok": True}
|
| 1172 |
+
|
| 1173 |
+
def _get_proc() -> Tuple[Any, Optional[threading.Lock]]:
|
| 1174 |
+
if not use_threads:
|
| 1175 |
+
return base_processor, None
|
| 1176 |
+
proc = getattr(tls, "proc", None)
|
| 1177 |
+
if proc is not None:
|
| 1178 |
+
return proc, None
|
| 1179 |
+
if deepcopy_ok["ok"]:
|
| 1180 |
+
try:
|
| 1181 |
+
proc = copy.deepcopy(base_processor)
|
| 1182 |
+
tls.proc = proc
|
| 1183 |
+
return proc, None
|
| 1184 |
+
except Exception:
|
| 1185 |
+
deepcopy_ok["ok"] = False
|
| 1186 |
+
return base_processor, fallback_lock
|
| 1187 |
+
|
| 1188 |
+
def _prep_chunk(chunk: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], Dict[str, Any], float]:
|
| 1189 |
+
tp0 = time.perf_counter()
|
| 1190 |
+
proc, lck = _get_proc()
|
| 1191 |
+
b_imgs = [rgb_images_all[t["img_idx"]] for t in chunk]
|
| 1192 |
+
b_prompt_objs = [t["prompt_objs"] for t in chunk]
|
| 1193 |
+
prepared = self._prepare_batch(
|
| 1194 |
+
batch_images=b_imgs,
|
| 1195 |
+
batch_prompt_objs=b_prompt_objs,
|
| 1196 |
+
processor=proc,
|
| 1197 |
+
lock=lck,
|
| 1198 |
+
needs_mm_token_type_ids=needs_mm,
|
| 1199 |
+
image_token_id=image_token_id,
|
| 1200 |
+
)
|
| 1201 |
+
prep_ms = (time.perf_counter() - tp0) * 1000.0
|
| 1202 |
+
return chunk, prepared, prep_ms
|
| 1203 |
+
|
| 1204 |
+
def _iter_batches() -> Iterator[Tuple[List[Dict[str, Any]], Dict[str, Any], float]]:
|
| 1205 |
+
if not use_threads:
|
| 1206 |
+
for c in task_chunks:
|
| 1207 |
+
yield _prep_chunk(c)
|
| 1208 |
+
return
|
| 1209 |
+
|
| 1210 |
+
max_inflight = max(num_workers * 2, 2)
|
| 1211 |
+
inflight: deque[Future] = deque()
|
| 1212 |
+
chunk_it = iter(task_chunks)
|
| 1213 |
+
exhausted = False
|
| 1214 |
+
|
| 1215 |
+
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
| 1216 |
+
def _fill() -> None:
|
| 1217 |
+
nonlocal exhausted
|
| 1218 |
+
while not exhausted and len(inflight) < max_inflight:
|
| 1219 |
+
try:
|
| 1220 |
+
c = next(chunk_it)
|
| 1221 |
+
except StopIteration:
|
| 1222 |
+
exhausted = True
|
| 1223 |
+
break
|
| 1224 |
+
inflight.append(pool.submit(_prep_chunk, c))
|
| 1225 |
+
|
| 1226 |
+
_fill()
|
| 1227 |
+
while inflight:
|
| 1228 |
+
fut = inflight.popleft()
|
| 1229 |
+
_fill()
|
| 1230 |
+
yield fut.result()
|
| 1231 |
+
|
| 1232 |
+
for batch_num, (chunk, prepared, prep_ms) in enumerate(_iter_batches(), start=1):
|
| 1233 |
+
total_prep_ms += prep_ms
|
| 1234 |
+
|
| 1235 |
+
t_fwd0 = time.perf_counter()
|
| 1236 |
+
dev_inputs = {
|
| 1237 |
+
k: (v.to(self.device, non_blocking=is_cuda) if isinstance(v, torch.Tensor) else v)
|
| 1238 |
+
for k, v in prepared["inputs"].items()
|
| 1239 |
+
}
|
| 1240 |
+
|
| 1241 |
+
ev_start = None
|
| 1242 |
+
ev_end = None
|
| 1243 |
+
if is_cuda and self._debug:
|
| 1244 |
+
ev_start = torch.cuda.Event(enable_timing=True)
|
| 1245 |
+
ev_end = torch.cuda.Event(enable_timing=True)
|
| 1246 |
+
ev_start.record()
|
| 1247 |
+
|
| 1248 |
+
outputs = self.model(
|
| 1249 |
+
input_ids=dev_inputs["input_ids"],
|
| 1250 |
+
attention_mask=dev_inputs["attention_mask"],
|
| 1251 |
+
pixel_values=dev_inputs["pixel_values"],
|
| 1252 |
+
image_grid_thw=dev_inputs["image_grid_thw"],
|
| 1253 |
+
target_sizes=prepared["orig_sizes"],
|
| 1254 |
+
mm_token_type_ids=dev_inputs.get("mm_token_type_ids"),
|
| 1255 |
+
grid_thw_list=prepared["grid_thw_list"],
|
| 1256 |
+
input_ids_cpu=prepared["input_ids_cpu"],
|
| 1257 |
+
)
|
| 1258 |
+
|
| 1259 |
+
if ev_end is not None and ev_start is not None:
|
| 1260 |
+
ev_end.record()
|
| 1261 |
+
torch.cuda.synchronize(self.device)
|
| 1262 |
+
batch_gpu_ms = float(ev_start.elapsed_time(ev_end))
|
| 1263 |
+
total_gpu_ms += batch_gpu_ms
|
| 1264 |
+
else:
|
| 1265 |
+
batch_gpu_ms = 0.0
|
| 1266 |
+
|
| 1267 |
+
batch_fwd_ms = (time.perf_counter() - t_fwd0) * 1000.0
|
| 1268 |
+
total_forward_ms += batch_fwd_ms
|
| 1269 |
+
|
| 1270 |
+
t_post0 = time.perf_counter()
|
| 1271 |
+
for b_pos, task in enumerate(chunk):
|
| 1272 |
+
_record_task_output(
|
| 1273 |
+
task=task,
|
| 1274 |
+
probs_b=outputs["mask_probs"][b_pos],
|
| 1275 |
+
scores_b=outputs["pred_scores"][b_pos],
|
| 1276 |
+
)
|
| 1277 |
+
|
| 1278 |
+
batch_post_ms = (time.perf_counter() - t_post0) * 1000.0
|
| 1279 |
+
total_post_ms += batch_post_ms
|
| 1280 |
+
|
| 1281 |
+
if self._debug:
|
| 1282 |
+
gpu_info = f", gpu_forward={batch_gpu_ms:.1f} ms" if is_cuda else ""
|
| 1283 |
+
self._log(
|
| 1284 |
+
f"Batch {batch_num}/{len(task_chunks)} ({len(chunk)} prompt(s)): "
|
| 1285 |
+
f"prep={prep_ms:.1f} ms, forward={batch_fwd_ms:.1f} ms{gpu_info}, "
|
| 1286 |
+
f"post={batch_post_ms:.1f} ms"
|
| 1287 |
+
)
|
| 1288 |
+
|
| 1289 |
+
total_ms = (time.perf_counter() - t_start) * 1000.0
|
| 1290 |
+
if self._debug:
|
| 1291 |
+
per_img_ms = total_ms / max(len(valid_img_indices), 1)
|
| 1292 |
+
gpu_summary = (
|
| 1293 |
+
f", total_gpu={total_gpu_ms:.1f} ms ({total_gpu_ms / max(len(valid_img_indices), 1):.1f} ms/img)"
|
| 1294 |
+
if is_cuda
|
| 1295 |
+
else ""
|
| 1296 |
+
)
|
| 1297 |
+
self._log(
|
| 1298 |
+
f"Inference finished in {total_ms:.1f} ms ({per_img_ms:.1f} ms/img{gpu_summary}) | "
|
| 1299 |
+
f"prep={total_prep_ms:.1f} ms, forward={total_forward_ms:.1f} ms, post={total_post_ms:.1f} ms."
|
| 1300 |
+
)
|
| 1301 |
+
|
| 1302 |
+
return results
|
| 1303 |
+
|
| 1304 |
+
def render_image(
|
| 1305 |
+
self,
|
| 1306 |
+
images: Union[Image.Image, Sequence[Image.Image]],
|
| 1307 |
+
outputs: Union[Dict[str, Any], Sequence[Dict[str, Any]]],
|
| 1308 |
+
draw_masks: bool = True,
|
| 1309 |
+
draw_boxes: bool = True,
|
| 1310 |
+
draw_labels: bool = True,
|
| 1311 |
+
draw_scores: bool = True,
|
| 1312 |
+
draw_contours: bool = True,
|
| 1313 |
+
mask_alpha: float = 0.45,
|
| 1314 |
+
palette: Sequence[Tuple[int, int, int]] = DEFAULT_PALETTE,
|
| 1315 |
+
) -> List[Image.Image]:
|
| 1316 |
+
"""
|
| 1317 |
+
Renders predicted segmentation masks, bounding boxes, and labels onto PIL images.
|
| 1318 |
+
|
| 1319 |
+
Args:
|
| 1320 |
+
images: List of PIL.Image.Image (or a single PIL.Image.Image).
|
| 1321 |
+
outputs: Output list of dicts returned by `run_model(...)` (or a single output dict).
|
| 1322 |
+
draw_masks: Whether to overlay semi-transparent segmentation masks.
|
| 1323 |
+
draw_boxes: Whether to draw 2D bounding boxes around segmented objects.
|
| 1324 |
+
draw_labels: Whether to draw object description text labels.
|
| 1325 |
+
draw_scores: Whether to include confidence scores in the text labels (when `draw_labels=True`).
|
| 1326 |
+
draw_contours: Whether to draw crisp boundary contours around masks (when `draw_masks=True`).
|
| 1327 |
+
mask_alpha: Opacity of the mask overlay in [0, 1].
|
| 1328 |
+
palette: Sequence of (R, G, B) color tuples cycled across objects.
|
| 1329 |
+
|
| 1330 |
+
Returns:
|
| 1331 |
+
List of rendered PIL.Image.Image objects (RGB).
|
| 1332 |
+
"""
|
| 1333 |
+
t0 = time.perf_counter()
|
| 1334 |
+
|
| 1335 |
+
img_list: List[Image.Image] = (
|
| 1336 |
+
[images] if isinstance(images, Image.Image) else list(images)
|
| 1337 |
+
)
|
| 1338 |
+
out_list: List[Dict[str, Any]] = (
|
| 1339 |
+
[outputs] if isinstance(outputs, dict) else list(outputs)
|
| 1340 |
+
)
|
| 1341 |
+
|
| 1342 |
+
if len(img_list) != len(out_list):
|
| 1343 |
+
raise ValueError(
|
| 1344 |
+
f"Length mismatch in `render_image`: got {len(img_list)} images and {len(out_list)} outputs."
|
| 1345 |
+
)
|
| 1346 |
+
|
| 1347 |
+
rendered_images: List[Image.Image] = []
|
| 1348 |
+
alpha = float(np.clip(mask_alpha, 0.0, 1.0))
|
| 1349 |
+
|
| 1350 |
+
try:
|
| 1351 |
+
font = ImageFont.load_default()
|
| 1352 |
+
except Exception:
|
| 1353 |
+
font = None
|
| 1354 |
+
|
| 1355 |
+
for img, out_dict in zip(img_list, out_list):
|
| 1356 |
+
canvas_np = np.array(img.convert("RGB"), dtype=np.uint8, copy=True)
|
| 1357 |
+
h, w = canvas_np.shape[:2]
|
| 1358 |
+
objects = out_dict.get("objects", [])
|
| 1359 |
+
|
| 1360 |
+
# 1. Render masks and contours on numpy canvas
|
| 1361 |
+
if draw_masks:
|
| 1362 |
+
for obj_idx, obj in enumerate(objects):
|
| 1363 |
+
mask = obj.get("mask")
|
| 1364 |
+
if mask is None:
|
| 1365 |
+
continue
|
| 1366 |
+
mask_u8 = np.asarray(mask, dtype=np.uint8)
|
| 1367 |
+
if mask_u8.ndim > 2:
|
| 1368 |
+
mask_u8 = mask_u8.squeeze()
|
| 1369 |
+
if mask_u8.shape != (h, w):
|
| 1370 |
+
mask_u8 = cv2.resize(mask_u8, (w, h), interpolation=cv2.INTER_NEAREST)
|
| 1371 |
+
|
| 1372 |
+
fg = mask_u8 > 0
|
| 1373 |
+
if not np.any(fg):
|
| 1374 |
+
continue
|
| 1375 |
+
|
| 1376 |
+
color = palette[obj_idx % len(palette)]
|
| 1377 |
+
color_arr = np.array(color, dtype=np.float32)
|
| 1378 |
+
canvas_np[fg] = np.clip(
|
| 1379 |
+
(1.0 - alpha) * canvas_np[fg].astype(np.float32) + alpha * color_arr,
|
| 1380 |
+
0,
|
| 1381 |
+
255,
|
| 1382 |
+
).astype(np.uint8)
|
| 1383 |
+
|
| 1384 |
+
if draw_contours:
|
| 1385 |
+
contours, _ = cv2.findContours(
|
| 1386 |
+
mask_u8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
|
| 1387 |
+
)
|
| 1388 |
+
cv2.drawContours(canvas_np, contours, -1, color, thickness=2)
|
| 1389 |
+
|
| 1390 |
+
pil_canvas = Image.fromarray(canvas_np)
|
| 1391 |
+
draw = ImageDraw.Draw(pil_canvas)
|
| 1392 |
+
|
| 1393 |
+
# 2. Render bounding boxes first, then non-overlapping labels on top
|
| 1394 |
+
if draw_boxes or draw_labels:
|
| 1395 |
+
valid_box_entries: List[Tuple[int, Dict[str, Any], float, float, float, float]] = []
|
| 1396 |
+
for obj_idx, obj in enumerate(objects):
|
| 1397 |
+
area = float(obj.get("area", 0.0))
|
| 1398 |
+
bbox = obj.get("bbox", [0.0, 0.0, 0.0, 0.0])
|
| 1399 |
+
if area <= 0 or len(bbox) != 4 or bbox[2] <= 0 or bbox[3] <= 0:
|
| 1400 |
+
continue
|
| 1401 |
+
|
| 1402 |
+
x, y, bw, bh = [float(v) for v in bbox]
|
| 1403 |
+
x1 = max(0.0, min(x, w - 1.0))
|
| 1404 |
+
y1 = max(0.0, min(y, h - 1.0))
|
| 1405 |
+
x2 = max(0.0, min(x + bw, float(w)))
|
| 1406 |
+
y2 = max(0.0, min(y + bh, float(h)))
|
| 1407 |
+
valid_box_entries.append((obj_idx, obj, x1, y1, x2, y2))
|
| 1408 |
+
|
| 1409 |
+
if draw_boxes:
|
| 1410 |
+
for obj_idx, _, x1, y1, x2, y2 in valid_box_entries:
|
| 1411 |
+
color = palette[obj_idx % len(palette)]
|
| 1412 |
+
draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
|
| 1413 |
+
|
| 1414 |
+
if draw_labels:
|
| 1415 |
+
occupied_labels: List[Tuple[float, float, float, float]] = []
|
| 1416 |
+
for obj_idx, obj, x1, y1, _, _ in valid_box_entries:
|
| 1417 |
+
color = palette[obj_idx % len(palette)]
|
| 1418 |
+
desc = str(obj.get("description", f"obj_{obj_idx}")).strip()
|
| 1419 |
+
if len(desc) > 48:
|
| 1420 |
+
desc = desc[:45] + "..."
|
| 1421 |
+
if draw_scores and "score" in obj:
|
| 1422 |
+
label_text = f"{desc} ({float(obj['score']):.2f})"
|
| 1423 |
+
else:
|
| 1424 |
+
label_text = desc
|
| 1425 |
+
|
| 1426 |
+
if hasattr(draw, "textbbox"):
|
| 1427 |
+
tb = draw.textbbox((0, 0), label_text, font=font)
|
| 1428 |
+
tw, th = float(tb[2] - tb[0]), float(tb[3] - tb[1])
|
| 1429 |
+
else:
|
| 1430 |
+
tw, th = float(len(label_text) * 6), 11.0
|
| 1431 |
+
|
| 1432 |
+
pad = 3.0
|
| 1433 |
+
label_w = tw + 2.0 * pad
|
| 1434 |
+
label_h = th + 2.0 * pad
|
| 1435 |
+
tx1, ty1, tx2, ty2 = _place_non_overlapping_label(
|
| 1436 |
+
x1=x1,
|
| 1437 |
+
y1=y1,
|
| 1438 |
+
label_w=label_w,
|
| 1439 |
+
label_h=label_h,
|
| 1440 |
+
img_w=w,
|
| 1441 |
+
img_h=h,
|
| 1442 |
+
occupied=occupied_labels,
|
| 1443 |
+
)
|
| 1444 |
+
occupied_labels.append((tx1, ty1, tx2, ty2))
|
| 1445 |
+
|
| 1446 |
+
draw.rectangle([tx1, ty1, tx2, ty2], fill=color)
|
| 1447 |
+
draw.text(
|
| 1448 |
+
(tx1 + pad, ty1 + pad),
|
| 1449 |
+
label_text,
|
| 1450 |
+
fill=(255, 255, 255),
|
| 1451 |
+
font=font,
|
| 1452 |
+
)
|
| 1453 |
+
|
| 1454 |
+
rendered_images.append(pil_canvas)
|
| 1455 |
+
|
| 1456 |
+
elapsed = (time.perf_counter() - t0) * 1000.0
|
| 1457 |
+
self._log(
|
| 1458 |
+
f"Rendered {len(rendered_images)} image(s) in {elapsed:.1f} ms "
|
| 1459 |
+
f"(draw_masks={draw_masks}, draw_boxes={draw_boxes}, draw_labels={draw_labels})."
|
| 1460 |
+
)
|
| 1461 |
+
return rendered_images
|
logo.png
ADDED
|
result_0_boxes_and_masks.png
ADDED
|
Git LFS Details
|
train.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import pathlib
|
| 3 |
+
from typing import Dict, Tuple
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from torch.optim import AdamW
|
| 7 |
+
from torch.utils.data import DataLoader
|
| 8 |
+
|
| 9 |
+
from LIS1ModelCode import Qwen2_5_VL_SinglePassSegmenter, SegmentationLoss
|
| 10 |
+
from dataset import RefCOCOJsonCollator, RefCOCOJsonDataset
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@torch.no_grad()
|
| 14 |
+
def evaluate_miou(
|
| 15 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 16 |
+
val_loader: DataLoader,
|
| 17 |
+
device: torch.device,
|
| 18 |
+
max_batches: int = 100,
|
| 19 |
+
) -> Dict[str, float]:
|
| 20 |
+
model.eval()
|
| 21 |
+
total_iou = 0.0
|
| 22 |
+
total_inter = 0.0
|
| 23 |
+
total_union = 0.0
|
| 24 |
+
total_objects = 0
|
| 25 |
+
|
| 26 |
+
for step, batch in enumerate(val_loader):
|
| 27 |
+
if max_batches > 0 and step >= max_batches:
|
| 28 |
+
break
|
| 29 |
+
|
| 30 |
+
outputs = model(
|
| 31 |
+
input_ids=batch["input_ids"].to(device),
|
| 32 |
+
attention_mask=batch["attention_mask"].to(device),
|
| 33 |
+
pixel_values=batch["pixel_values"].to(device),
|
| 34 |
+
image_grid_thw=batch["image_grid_thw"].to(device),
|
| 35 |
+
target_sizes=batch["target_sizes"],
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
for probs, gt in zip(outputs["mask_probs"], batch["gt_masks_list"]):
|
| 39 |
+
pred_bin = (probs >= 0.5).to(torch.float32).cpu().flatten(1)
|
| 40 |
+
gt_flat = gt.to(torch.float32).cpu().flatten(1)
|
| 41 |
+
inter = (pred_bin * gt_flat).sum(dim=1)
|
| 42 |
+
union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1)
|
| 43 |
+
|
| 44 |
+
iou = inter / union.clamp(min=1e-6)
|
| 45 |
+
total_iou += float(iou.sum().item())
|
| 46 |
+
total_inter += float(inter.sum().item())
|
| 47 |
+
total_union += float(union.sum().item())
|
| 48 |
+
total_objects += int(gt.shape[0])
|
| 49 |
+
|
| 50 |
+
miou = total_iou / max(total_objects, 1)
|
| 51 |
+
ciou = total_inter / max(total_union, 1e-6)
|
| 52 |
+
return {"mIoU": round(miou, 4), "cIoU": round(ciou, 4), "objects": total_objects}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def save_checkpoint(
|
| 56 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 57 |
+
output_dir: pathlib.Path,
|
| 58 |
+
tag: str,
|
| 59 |
+
) -> pathlib.Path:
|
| 60 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 61 |
+
ckpt_path = output_dir / f"segmenter_{tag}.pt"
|
| 62 |
+
state = {"mask_head": model.mask_head.state_dict()}
|
| 63 |
+
if not model.freeze_backbone and hasattr(model.backbone.language_model, "peft_config"):
|
| 64 |
+
lora_dir = output_dir / f"lora_{tag}"
|
| 65 |
+
model.backbone.language_model.save_pretrained(str(lora_dir))
|
| 66 |
+
state["lora_dir"] = str(lora_dir)
|
| 67 |
+
torch.save(state, ckpt_path)
|
| 68 |
+
return ckpt_path
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def load_checkpoint(
|
| 72 |
+
model: Qwen2_5_VL_SinglePassSegmenter,
|
| 73 |
+
ckpt_path: pathlib.Path,
|
| 74 |
+
device: torch.device,
|
| 75 |
+
) -> None:
|
| 76 |
+
state = torch.load(ckpt_path, map_location=device)
|
| 77 |
+
model.mask_head.load_state_dict(state["mask_head"], strict=False)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def run_training(args: argparse.Namespace) -> None:
|
| 81 |
+
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
|
| 82 |
+
dtype = torch.bfloat16 if (device.type == "cuda" and torch.cuda.is_bf16_supported()) else torch.float32
|
| 83 |
+
|
| 84 |
+
model = Qwen2_5_VL_SinglePassSegmenter(
|
| 85 |
+
qwen_model_name=args.model_path,
|
| 86 |
+
torch_dtype=dtype,
|
| 87 |
+
freeze_backbone=True,
|
| 88 |
+
num_fused_layers=args.fused_layers,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
if args.stage == 2:
|
| 92 |
+
model.enable_lora(
|
| 93 |
+
r=args.lora_r,
|
| 94 |
+
lora_alpha=args.lora_alpha,
|
| 95 |
+
top_k_layers=args.lora_top_layers,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
model.to(device)
|
| 99 |
+
model.mask_head.to(device=device, dtype=torch.float32)
|
| 100 |
+
|
| 101 |
+
if args.resume_ckpt:
|
| 102 |
+
load_checkpoint(model, pathlib.Path(args.resume_ckpt), device)
|
| 103 |
+
|
| 104 |
+
train_ds = RefCOCOJsonDataset(
|
| 105 |
+
data_root=args.data_root,
|
| 106 |
+
dataset_name=args.dataset,
|
| 107 |
+
split_by=args.split_by,
|
| 108 |
+
split="train",
|
| 109 |
+
max_objects_per_image=args.max_objects,
|
| 110 |
+
riddle_prob=args.riddle_prob,
|
| 111 |
+
)
|
| 112 |
+
val_ds = RefCOCOJsonDataset(
|
| 113 |
+
data_root=args.data_root,
|
| 114 |
+
dataset_name="refcoco" if args.dataset == "all" else args.dataset,
|
| 115 |
+
split_by=args.split_by,
|
| 116 |
+
split="val",
|
| 117 |
+
max_objects_per_image=args.max_objects,
|
| 118 |
+
riddle_prob=0.0,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
collator = RefCOCOJsonCollator(model.processor)
|
| 122 |
+
train_loader = DataLoader(
|
| 123 |
+
train_ds,
|
| 124 |
+
batch_size=args.batch_size,
|
| 125 |
+
shuffle=True,
|
| 126 |
+
num_workers=args.num_workers,
|
| 127 |
+
collate_fn=collator,
|
| 128 |
+
)
|
| 129 |
+
val_loader = DataLoader(
|
| 130 |
+
val_ds,
|
| 131 |
+
batch_size=args.batch_size,
|
| 132 |
+
shuffle=False,
|
| 133 |
+
num_workers=args.num_workers,
|
| 134 |
+
collate_fn=collator,
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
param_groups = [{"params": list(model.mask_head.parameters()), "lr": args.lr_head}]
|
| 138 |
+
if args.stage == 2:
|
| 139 |
+
lora_params = [p for p in model.backbone.parameters() if p.requires_grad]
|
| 140 |
+
param_groups.append({"params": lora_params, "lr": args.lr_lora})
|
| 141 |
+
|
| 142 |
+
optimizer = AdamW(param_groups, weight_decay=args.weight_decay)
|
| 143 |
+
criterion = SegmentationLoss()
|
| 144 |
+
output_dir = pathlib.Path(args.output_dir)
|
| 145 |
+
|
| 146 |
+
for epoch in range(1, args.epochs + 1):
|
| 147 |
+
model.train()
|
| 148 |
+
running_loss = 0.0
|
| 149 |
+
for step, batch in enumerate(train_loader, start=1):
|
| 150 |
+
optimizer.zero_grad(set_to_none=True)
|
| 151 |
+
|
| 152 |
+
outputs = model(
|
| 153 |
+
input_ids=batch["input_ids"].to(device),
|
| 154 |
+
attention_mask=batch["attention_mask"].to(device),
|
| 155 |
+
pixel_values=batch["pixel_values"].to(device),
|
| 156 |
+
image_grid_thw=batch["image_grid_thw"].to(device),
|
| 157 |
+
target_sizes=batch["target_sizes"],
|
| 158 |
+
)
|
| 159 |
+
loss_dict = criterion(
|
| 160 |
+
mask_logits_list=outputs["mask_logits"],
|
| 161 |
+
pred_scores_list=outputs["pred_scores"],
|
| 162 |
+
gt_masks_list=batch["gt_masks_list"],
|
| 163 |
+
)
|
| 164 |
+
loss = loss_dict["loss"]
|
| 165 |
+
loss.backward()
|
| 166 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
|
| 167 |
+
optimizer.step()
|
| 168 |
+
|
| 169 |
+
running_loss += float(loss.item())
|
| 170 |
+
if step % args.log_every == 0:
|
| 171 |
+
avg_loss = running_loss / args.log_every
|
| 172 |
+
running_loss = 0.0
|
| 173 |
+
print(
|
| 174 |
+
f"[Epoch {epoch}/{args.epochs} | Step {step}/{len(train_loader)}] "
|
| 175 |
+
f"loss={avg_loss:.4f} dice={loss_dict['dice_loss'].item():.4f} "
|
| 176 |
+
f"focal={loss_dict['focal_loss'].item():.4f}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
metrics = evaluate_miou(model, val_loader, device=device, max_batches=args.val_batches)
|
| 180 |
+
print(f"[Epoch {epoch} Validation] mIoU={metrics['mIoU']} cIoU={metrics['cIoU']}")
|
| 181 |
+
ckpt = save_checkpoint(model, output_dir, tag=f"stage{args.stage}_epoch{epoch}")
|
| 182 |
+
print(f"Saved checkpoint: {ckpt}")
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def parse_args() -> argparse.Namespace:
|
| 186 |
+
parser = argparse.ArgumentParser(description="Single-Pass Qwen2.5-VL JSON Segmenter")
|
| 187 |
+
parser.add_argument("--model-path", default="./Qwen3.5-4B")
|
| 188 |
+
parser.add_argument("--data-root", default="./ref_coco_lichengunc")
|
| 189 |
+
parser.add_argument("--dataset", default="refcoco", choices=["refcoco", "refcoco+", "refcocog", "all"])
|
| 190 |
+
parser.add_argument("--split-by", default="unc")
|
| 191 |
+
parser.add_argument(
|
| 192 |
+
"--riddle-prob",
|
| 193 |
+
type=float,
|
| 194 |
+
default=0.35,
|
| 195 |
+
help="Probability of augmenting training descriptions into functional/riddle prompts",
|
| 196 |
+
)
|
| 197 |
+
parser.add_argument("--output-dir", default="./checkpoints")
|
| 198 |
+
parser.add_argument("--resume-ckpt", default="")
|
| 199 |
+
parser.add_argument("--stage", type=int, choices=[1, 2], default=1)
|
| 200 |
+
parser.add_argument("--epochs", type=int, default=5)
|
| 201 |
+
parser.add_argument("--batch-size", type=int, default=2)
|
| 202 |
+
parser.add_argument("--max-objects", type=int, default=4)
|
| 203 |
+
parser.add_argument("--lr-head", type=float, default=2e-4)
|
| 204 |
+
parser.add_argument("--lr-lora", type=float, default=2e-5)
|
| 205 |
+
parser.add_argument("--weight-decay", type=float, default=1e-4)
|
| 206 |
+
parser.add_argument("--fused-layers", type=int, default=4, help="Number of top LLM layers fused into mask head")
|
| 207 |
+
parser.add_argument("--lora-r", type=int, default=8)
|
| 208 |
+
parser.add_argument("--lora-alpha", type=int, default=16)
|
| 209 |
+
parser.add_argument(
|
| 210 |
+
"--lora-top-layers",
|
| 211 |
+
type=int,
|
| 212 |
+
default=4,
|
| 213 |
+
help="Only apply LoRA to the last N LLM layers so earlier reasoning layers stay 100%% frozen",
|
| 214 |
+
)
|
| 215 |
+
parser.add_argument("--num-workers", type=int, default=2)
|
| 216 |
+
parser.add_argument("--log-every", type=int, default=50)
|
| 217 |
+
parser.add_argument("--val-batches", type=int, default=100)
|
| 218 |
+
parser.add_argument("--device", default="cuda")
|
| 219 |
+
return parser.parse_args()
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
if __name__ == "__main__":
|
| 223 |
+
cli_args = parse_args()
|
| 224 |
+
run_training(cli_args)
|