Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/added_tokens.json +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/lora_adapter/README.md +202 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/lora_adapter/adapter_config.json +45 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/preprocessor_config.json +114 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/processing_prismatic.py +252 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/processor_config.json +6 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/special_tokens_map.json +30 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/tokenizer.json +0 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/tokenizer_config.json +53 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/added_tokens.json +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/lora_adapter/README.md +202 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/lora_adapter/adapter_config.json +45 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/preprocessor_config.json +114 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/processing_prismatic.py +252 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/processor_config.json +6 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/special_tokens_map.json +30 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/tokenizer.json +0 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/tokenizer_config.json +53 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/train_state--45000_checkpoint.pt +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/added_tokens.json +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/lora_adapter/README.md +202 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/lora_adapter/adapter_config.json +45 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/preprocessor_config.json +114 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/processing_prismatic.py +252 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/processor_config.json +6 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/special_tokens_map.json +30 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/tokenizer.json +0 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/tokenizer_config.json +53 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/added_tokens.json +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/lora_adapter/README.md +202 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/lora_adapter/adapter_config.json +45 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/preprocessor_config.json +114 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/processing_prismatic.py +252 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/processor_config.json +6 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/special_tokens_map.json +30 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/tokenizer.json +0 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/tokenizer_config.json +53 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251107-204623/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/added_tokens.json +3 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/dataset_statistics.json +127 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/lora_adapter/README.md +202 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/lora_adapter/adapter_config.json +45 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/preprocessor_config.json +114 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/processing_prismatic.py +252 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/processor_config.json +6 -0
- output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/special_tokens_map.json +30 -0
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<PAD>": 32000
|
| 3 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/lora_adapter/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: pretrained_models/configs-openvla-7b/config.json
|
| 3 |
+
library_name: peft
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Model Card for Model ID
|
| 7 |
+
|
| 8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## Model Details
|
| 13 |
+
|
| 14 |
+
### Model Description
|
| 15 |
+
|
| 16 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
- **Developed by:** [More Information Needed]
|
| 21 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 22 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 23 |
+
- **Model type:** [More Information Needed]
|
| 24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 25 |
+
- **License:** [More Information Needed]
|
| 26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 27 |
+
|
| 28 |
+
### Model Sources [optional]
|
| 29 |
+
|
| 30 |
+
<!-- Provide the basic links for the model. -->
|
| 31 |
+
|
| 32 |
+
- **Repository:** [More Information Needed]
|
| 33 |
+
- **Paper [optional]:** [More Information Needed]
|
| 34 |
+
- **Demo [optional]:** [More Information Needed]
|
| 35 |
+
|
| 36 |
+
## Uses
|
| 37 |
+
|
| 38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 39 |
+
|
| 40 |
+
### Direct Use
|
| 41 |
+
|
| 42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 43 |
+
|
| 44 |
+
[More Information Needed]
|
| 45 |
+
|
| 46 |
+
### Downstream Use [optional]
|
| 47 |
+
|
| 48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 49 |
+
|
| 50 |
+
[More Information Needed]
|
| 51 |
+
|
| 52 |
+
### Out-of-Scope Use
|
| 53 |
+
|
| 54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 55 |
+
|
| 56 |
+
[More Information Needed]
|
| 57 |
+
|
| 58 |
+
## Bias, Risks, and Limitations
|
| 59 |
+
|
| 60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 61 |
+
|
| 62 |
+
[More Information Needed]
|
| 63 |
+
|
| 64 |
+
### Recommendations
|
| 65 |
+
|
| 66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 67 |
+
|
| 68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 69 |
+
|
| 70 |
+
## How to Get Started with the Model
|
| 71 |
+
|
| 72 |
+
Use the code below to get started with the model.
|
| 73 |
+
|
| 74 |
+
[More Information Needed]
|
| 75 |
+
|
| 76 |
+
## Training Details
|
| 77 |
+
|
| 78 |
+
### Training Data
|
| 79 |
+
|
| 80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 81 |
+
|
| 82 |
+
[More Information Needed]
|
| 83 |
+
|
| 84 |
+
### Training Procedure
|
| 85 |
+
|
| 86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 87 |
+
|
| 88 |
+
#### Preprocessing [optional]
|
| 89 |
+
|
| 90 |
+
[More Information Needed]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
#### Training Hyperparameters
|
| 94 |
+
|
| 95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 96 |
+
|
| 97 |
+
#### Speeds, Sizes, Times [optional]
|
| 98 |
+
|
| 99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 100 |
+
|
| 101 |
+
[More Information Needed]
|
| 102 |
+
|
| 103 |
+
## Evaluation
|
| 104 |
+
|
| 105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 106 |
+
|
| 107 |
+
### Testing Data, Factors & Metrics
|
| 108 |
+
|
| 109 |
+
#### Testing Data
|
| 110 |
+
|
| 111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 112 |
+
|
| 113 |
+
[More Information Needed]
|
| 114 |
+
|
| 115 |
+
#### Factors
|
| 116 |
+
|
| 117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 118 |
+
|
| 119 |
+
[More Information Needed]
|
| 120 |
+
|
| 121 |
+
#### Metrics
|
| 122 |
+
|
| 123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 124 |
+
|
| 125 |
+
[More Information Needed]
|
| 126 |
+
|
| 127 |
+
### Results
|
| 128 |
+
|
| 129 |
+
[More Information Needed]
|
| 130 |
+
|
| 131 |
+
#### Summary
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
## Model Examination [optional]
|
| 136 |
+
|
| 137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 138 |
+
|
| 139 |
+
[More Information Needed]
|
| 140 |
+
|
| 141 |
+
## Environmental Impact
|
| 142 |
+
|
| 143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 144 |
+
|
| 145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 146 |
+
|
| 147 |
+
- **Hardware Type:** [More Information Needed]
|
| 148 |
+
- **Hours used:** [More Information Needed]
|
| 149 |
+
- **Cloud Provider:** [More Information Needed]
|
| 150 |
+
- **Compute Region:** [More Information Needed]
|
| 151 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 152 |
+
|
| 153 |
+
## Technical Specifications [optional]
|
| 154 |
+
|
| 155 |
+
### Model Architecture and Objective
|
| 156 |
+
|
| 157 |
+
[More Information Needed]
|
| 158 |
+
|
| 159 |
+
### Compute Infrastructure
|
| 160 |
+
|
| 161 |
+
[More Information Needed]
|
| 162 |
+
|
| 163 |
+
#### Hardware
|
| 164 |
+
|
| 165 |
+
[More Information Needed]
|
| 166 |
+
|
| 167 |
+
#### Software
|
| 168 |
+
|
| 169 |
+
[More Information Needed]
|
| 170 |
+
|
| 171 |
+
## Citation [optional]
|
| 172 |
+
|
| 173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 174 |
+
|
| 175 |
+
**BibTeX:**
|
| 176 |
+
|
| 177 |
+
[More Information Needed]
|
| 178 |
+
|
| 179 |
+
**APA:**
|
| 180 |
+
|
| 181 |
+
[More Information Needed]
|
| 182 |
+
|
| 183 |
+
## Glossary [optional]
|
| 184 |
+
|
| 185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 186 |
+
|
| 187 |
+
[More Information Needed]
|
| 188 |
+
|
| 189 |
+
## More Information [optional]
|
| 190 |
+
|
| 191 |
+
[More Information Needed]
|
| 192 |
+
|
| 193 |
+
## Model Card Authors [optional]
|
| 194 |
+
|
| 195 |
+
[More Information Needed]
|
| 196 |
+
|
| 197 |
+
## Model Card Contact
|
| 198 |
+
|
| 199 |
+
[More Information Needed]
|
| 200 |
+
### Framework versions
|
| 201 |
+
|
| 202 |
+
- PEFT 0.11.1
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/lora_adapter/adapter_config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alpha_pattern": {},
|
| 3 |
+
"auto_mapping": {
|
| 4 |
+
"base_model_class": "OpenVLAForActionPrediction",
|
| 5 |
+
"parent_library": "prismatic.extern.hf.modeling_prismatic"
|
| 6 |
+
},
|
| 7 |
+
"base_model_name_or_path": "pretrained_models/configs-openvla-7b/config.json",
|
| 8 |
+
"bias": "none",
|
| 9 |
+
"fan_in_fan_out": false,
|
| 10 |
+
"inference_mode": true,
|
| 11 |
+
"init_lora_weights": "gaussian",
|
| 12 |
+
"layer_replication": null,
|
| 13 |
+
"layers_pattern": null,
|
| 14 |
+
"layers_to_transform": null,
|
| 15 |
+
"loftq_config": {},
|
| 16 |
+
"lora_alpha": 64,
|
| 17 |
+
"lora_dropout": 0.0,
|
| 18 |
+
"megatron_config": null,
|
| 19 |
+
"megatron_core": "megatron.core",
|
| 20 |
+
"modules_to_save": null,
|
| 21 |
+
"peft_type": "LORA",
|
| 22 |
+
"r": 32,
|
| 23 |
+
"rank_pattern": {},
|
| 24 |
+
"revision": null,
|
| 25 |
+
"target_modules": [
|
| 26 |
+
"o_proj",
|
| 27 |
+
"lm_head",
|
| 28 |
+
"q_proj",
|
| 29 |
+
"v_proj",
|
| 30 |
+
"kv",
|
| 31 |
+
"proj",
|
| 32 |
+
"fc1",
|
| 33 |
+
"q",
|
| 34 |
+
"down_proj",
|
| 35 |
+
"fc2",
|
| 36 |
+
"k_proj",
|
| 37 |
+
"gate_proj",
|
| 38 |
+
"qkv",
|
| 39 |
+
"up_proj",
|
| 40 |
+
"fc3"
|
| 41 |
+
],
|
| 42 |
+
"task_type": null,
|
| 43 |
+
"use_dora": false,
|
| 44 |
+
"use_rslora": false
|
| 45 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/preprocessor_config.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "processing_prismatic.PrismaticImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 5 |
+
},
|
| 6 |
+
"image_processor_type": "PrismaticImageProcessor",
|
| 7 |
+
"image_resize_strategy": "resize-naive",
|
| 8 |
+
"input_sizes": [
|
| 9 |
+
[
|
| 10 |
+
3,
|
| 11 |
+
224,
|
| 12 |
+
224
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
3,
|
| 16 |
+
224,
|
| 17 |
+
224
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"interpolations": [
|
| 21 |
+
"bicubic",
|
| 22 |
+
"bicubic"
|
| 23 |
+
],
|
| 24 |
+
"means": [
|
| 25 |
+
[
|
| 26 |
+
0.485,
|
| 27 |
+
0.456,
|
| 28 |
+
0.406
|
| 29 |
+
],
|
| 30 |
+
[
|
| 31 |
+
0.5,
|
| 32 |
+
0.5,
|
| 33 |
+
0.5
|
| 34 |
+
]
|
| 35 |
+
],
|
| 36 |
+
"processor_class": "PrismaticProcessor",
|
| 37 |
+
"stds": [
|
| 38 |
+
[
|
| 39 |
+
0.229,
|
| 40 |
+
0.224,
|
| 41 |
+
0.225
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
0.5,
|
| 45 |
+
0.5,
|
| 46 |
+
0.5
|
| 47 |
+
]
|
| 48 |
+
],
|
| 49 |
+
"tvf_crop_params": [
|
| 50 |
+
{
|
| 51 |
+
"output_size": [
|
| 52 |
+
224,
|
| 53 |
+
224
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"output_size": [
|
| 58 |
+
224,
|
| 59 |
+
224
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"tvf_do_letterbox": false,
|
| 64 |
+
"tvf_letterbox_fill": null,
|
| 65 |
+
"tvf_normalize_params": [
|
| 66 |
+
{
|
| 67 |
+
"inplace": false,
|
| 68 |
+
"mean": [
|
| 69 |
+
0.484375,
|
| 70 |
+
0.455078125,
|
| 71 |
+
0.40625
|
| 72 |
+
],
|
| 73 |
+
"std": [
|
| 74 |
+
0.228515625,
|
| 75 |
+
0.2236328125,
|
| 76 |
+
0.224609375
|
| 77 |
+
]
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"inplace": false,
|
| 81 |
+
"mean": [
|
| 82 |
+
0.5,
|
| 83 |
+
0.5,
|
| 84 |
+
0.5
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.5,
|
| 88 |
+
0.5,
|
| 89 |
+
0.5
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"tvf_resize_params": [
|
| 94 |
+
{
|
| 95 |
+
"antialias": true,
|
| 96 |
+
"interpolation": 3,
|
| 97 |
+
"max_size": null,
|
| 98 |
+
"size": [
|
| 99 |
+
224,
|
| 100 |
+
224
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"antialias": true,
|
| 105 |
+
"interpolation": 3,
|
| 106 |
+
"max_size": null,
|
| 107 |
+
"size": [
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"use_fused_vision_backbone": true
|
| 114 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/processing_prismatic.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
processing_prismatic.py
|
| 3 |
+
|
| 4 |
+
HuggingFace-style preprocessor definitions for Prismatic VLMs, inheriting from `ProcessorMixin`. Default configuration
|
| 5 |
+
specifies `siglip-224px+7b`.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Any, ClassVar, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import timm.data
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms.functional as TVF
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor
|
| 15 |
+
from transformers import PreTrainedTokenizerBase
|
| 16 |
+
from transformers.image_processing_utils import BatchFeature, ImageProcessingMixin
|
| 17 |
+
from transformers.processing_utils import ProcessorMixin
|
| 18 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| 19 |
+
from transformers.utils import TensorType
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# === Image Processing ===
|
| 23 |
+
def letterbox_pad_transform(image: Image.Image, padding_fill_value: Tuple[int, int, int]) -> Image.Image:
|
| 24 |
+
"""Given a PIL.Image, pad to square by adding a symmetric border around the height/width."""
|
| 25 |
+
(w, h), max_wh = image.size, max(image.size)
|
| 26 |
+
horizontal_pad, vertical_pad = int((max_wh - w) / 2), int((max_wh - h) / 2)
|
| 27 |
+
padding = (horizontal_pad, vertical_pad, horizontal_pad, vertical_pad)
|
| 28 |
+
|
| 29 |
+
return TVF.pad(image, padding, fill=padding_fill_value, padding_mode="constant")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PrismaticImageProcessor(ImageProcessingMixin):
|
| 33 |
+
model_input_names: ClassVar[List[str]] = ["pixel_values"]
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
use_fused_vision_backbone: bool = False,
|
| 38 |
+
image_resize_strategy: str = "letterbox",
|
| 39 |
+
input_sizes: Optional[List[Tuple[int, int, int]]] = None,
|
| 40 |
+
interpolations: Optional[List[str]] = None,
|
| 41 |
+
means: Optional[List[Tuple[float, float, float]]] = None,
|
| 42 |
+
stds: Optional[List[Tuple[float, float, float]]] = None,
|
| 43 |
+
**kwargs: str,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""
|
| 46 |
+
Initialize a PrismaticImageProcessor as a wrapper around a torchvision transform; this transform will be
|
| 47 |
+
created by TIMM, and edited to follow our custom `image_resize_strategy` logic.
|
| 48 |
+
@param use_fused_vision_backbone: Boolean indicating single or fused (dual) vision backbone
|
| 49 |
+
@param image_resize_strategy: Prismatic image resize strategy in < resize-naive | resize-crop | letterbox >
|
| 50 |
+
@param input_size: [TIMM :: `data_cfg`] Input image size as tuple (channels, width, height)
|
| 51 |
+
@param interpolation: [TIMM :: `data_cfg`] Interpolation as string (default: "bicubic")
|
| 52 |
+
@param mean: [TIMM :: `data_cfg`] Normalization mean as float tuple (or two-tuple if `fused_backbone`)
|
| 53 |
+
@param std: [TIMM :: `data_cfg`] Normalization std as float tuple (or two-tuple if `fused_backbone`)
|
| 54 |
+
"""
|
| 55 |
+
self.use_fused_vision_backbone = use_fused_vision_backbone
|
| 56 |
+
self.image_resize_strategy = image_resize_strategy
|
| 57 |
+
|
| 58 |
+
# Handle `None` default values
|
| 59 |
+
input_sizes = [(3, 224, 224)] if input_sizes is None else input_sizes
|
| 60 |
+
means = [(0.5, 0.5, 0.5)] if means is None else means
|
| 61 |
+
stds = [(0.5, 0.5, 0.5)] if stds is None else stds
|
| 62 |
+
|
| 63 |
+
# TIMM `data_cfg` Parameters
|
| 64 |
+
self.input_sizes, self.interpolations, self.means, self.stds = input_sizes, interpolations, means, stds
|
| 65 |
+
|
| 66 |
+
# Grab torchvision transforms via TIMM =>> need to parse for specific "functional" transform values!
|
| 67 |
+
self.tvf_resize_params, self.tvf_crop_params, self.tvf_normalize_params = [], [], []
|
| 68 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 69 |
+
|
| 70 |
+
for idx in range(len(input_sizes)):
|
| 71 |
+
transform = timm.data.create_transform(
|
| 72 |
+
input_size=self.input_sizes[idx],
|
| 73 |
+
interpolation=self.interpolations[idx],
|
| 74 |
+
mean=self.means[idx],
|
| 75 |
+
std=self.stds[idx],
|
| 76 |
+
crop_pct=1.0, # Set to 1.0 to ignore cropping (initial Resize sets `input_size`)
|
| 77 |
+
crop_mode="center", # Default crop mode -- no-op when `crop_pct == 1.0`
|
| 78 |
+
is_training=False, # No image augmentations when loading the transform!
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# [Validation] Ensure appropriate transform structure, expected sizes
|
| 82 |
+
if not (
|
| 83 |
+
isinstance(transform, Compose)
|
| 84 |
+
and (len(transform.transforms) == 4)
|
| 85 |
+
and isinstance(transform.transforms[0], Resize)
|
| 86 |
+
and isinstance(transform.transforms[1], CenterCrop)
|
| 87 |
+
and isinstance(transform.transforms[2], ToTensor)
|
| 88 |
+
and isinstance(transform.transforms[3], Normalize)
|
| 89 |
+
and (transform.transforms[0].size == self.input_sizes[idx][-1])
|
| 90 |
+
and (transform.transforms[1].size == self.input_sizes[idx][-2:])
|
| 91 |
+
):
|
| 92 |
+
raise ValueError(f"Unexpected TIMM image transformation structure/sizes: `{transform}`")
|
| 93 |
+
|
| 94 |
+
# HF Image Processors *must* be JSON-serializable; as such, cannot have torchvision. as an attribute.
|
| 95 |
+
# => Instead, we're going to parse the transform and call "torchvision.transforms.functional" (`tvf`)
|
| 96 |
+
resize_t, crop_t, norm_t = transform.transforms[0], transform.transforms[1], transform.transforms[3]
|
| 97 |
+
self.tvf_resize_params.append(
|
| 98 |
+
{
|
| 99 |
+
"size": resize_t.size,
|
| 100 |
+
"interpolation": TVF.pil_modes_mapping[resize_t.interpolation],
|
| 101 |
+
"max_size": None,
|
| 102 |
+
"antialias": True,
|
| 103 |
+
}
|
| 104 |
+
)
|
| 105 |
+
self.tvf_crop_params.append({"output_size": crop_t.size})
|
| 106 |
+
self.tvf_normalize_params.append(
|
| 107 |
+
{
|
| 108 |
+
"mean": norm_t.mean.float().numpy().tolist(),
|
| 109 |
+
"std": norm_t.std.float().numpy().tolist(),
|
| 110 |
+
"inplace": False,
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 114 |
+
|
| 115 |
+
# Handle Prismatic `image_resize_strategy`
|
| 116 |
+
if self.image_resize_strategy == "resize-naive":
|
| 117 |
+
self.tvf_resize_params[idx]["size"] = (resize_t.size, resize_t.size)
|
| 118 |
+
elif self.image_resize_strategy == "letterbox":
|
| 119 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = True, tuple([int(x * 255) for x in self.means[idx]])
|
| 120 |
+
elif self.image_resize_strategy == "resize-crop":
|
| 121 |
+
pass
|
| 122 |
+
else:
|
| 123 |
+
raise ValueError(f"Image resize strategy `{self.image_resize_strategy}` is not supported!")
|
| 124 |
+
|
| 125 |
+
# Dispatch **kwargs to super()
|
| 126 |
+
super().__init__(**kwargs)
|
| 127 |
+
|
| 128 |
+
def apply_transform(self, img: Image.Image) -> torch.Tensor:
|
| 129 |
+
"""Apply `functional` variant of TIMM's Transform = Compose([Resize -> CenterCrop -> ToTensor -> Normalize])"""
|
| 130 |
+
if self.tvf_do_letterbox:
|
| 131 |
+
img = letterbox_pad_transform(img, self.tvf_letterbox_fill)
|
| 132 |
+
|
| 133 |
+
# [Contract] Fused Backbones expect "channel-stacked" inputs; we'll unpack on the model side!
|
| 134 |
+
imgs_t = []
|
| 135 |
+
for idx in range(len(self.input_sizes)):
|
| 136 |
+
img_idx = TVF.resize(img, **self.tvf_resize_params[idx])
|
| 137 |
+
img_idx = TVF.center_crop(img_idx, **self.tvf_crop_params[idx])
|
| 138 |
+
img_idx_t = TVF.to_tensor(img_idx)
|
| 139 |
+
img_idx_t = TVF.normalize(img_idx_t, **self.tvf_normalize_params[idx])
|
| 140 |
+
imgs_t.append(img_idx_t)
|
| 141 |
+
|
| 142 |
+
# [Contract] `imgs_t` is a list of Tensors of shape [3, input_size, input_size]; stack along dim = 0
|
| 143 |
+
img_t = torch.vstack(imgs_t)
|
| 144 |
+
|
| 145 |
+
return img_t
|
| 146 |
+
|
| 147 |
+
def preprocess(
|
| 148 |
+
self,
|
| 149 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 150 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 151 |
+
**_: str,
|
| 152 |
+
) -> BatchFeature:
|
| 153 |
+
"""
|
| 154 |
+
Preprocess an image (or batch of images); note that unlike the `transformers :: BaseImageProcessor` we
|
| 155 |
+
explicitly only handle PIL.Image.Image instances for simplicity.
|
| 156 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 157 |
+
@param return_tensors: BatchFeature default Tensor format (e.g., "pt" for torch); if None, returns np.ndarray
|
| 158 |
+
@return: Instance of `transformers :: BatchFeature` with a single key "pixel_values"
|
| 159 |
+
"""
|
| 160 |
+
if not isinstance(images, list):
|
| 161 |
+
images = [images]
|
| 162 |
+
|
| 163 |
+
# Apply `self.img_transform` to each image (will return list of torch.Tensors); stack into "batched" Tensor
|
| 164 |
+
pixel_values = torch.stack([self.apply_transform(img.convert("RGB")) for img in images])
|
| 165 |
+
|
| 166 |
+
# Return BatchFeature =>> note that for compatibility, constructor expects Dict[str, np.ndarray], so we convert
|
| 167 |
+
return BatchFeature(data={"pixel_values": pixel_values.float().numpy()}, tensor_type=return_tensors)
|
| 168 |
+
|
| 169 |
+
def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> BatchFeature:
|
| 170 |
+
return self.preprocess(images, **kwargs)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# === PrismaticProcessor =>> Wraps both ImageProcessor and Tokenizer ===
|
| 174 |
+
# =>> https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/processing_llava.py
|
| 175 |
+
class PrismaticProcessor(ProcessorMixin):
|
| 176 |
+
attributes: ClassVar[List[str]] = ["image_processor", "tokenizer"]
|
| 177 |
+
image_processor_class: str = "AutoImageProcessor"
|
| 178 |
+
tokenizer_class: str = "AutoTokenizer"
|
| 179 |
+
|
| 180 |
+
def __init__(
|
| 181 |
+
self,
|
| 182 |
+
image_processor: Optional[ImageProcessingMixin] = None,
|
| 183 |
+
tokenizer: Optional[PreTrainedTokenizerBase] = None,
|
| 184 |
+
) -> None:
|
| 185 |
+
super().__init__(image_processor, tokenizer)
|
| 186 |
+
|
| 187 |
+
def __call__(
|
| 188 |
+
self,
|
| 189 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
|
| 190 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 191 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 192 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 193 |
+
max_length: Optional[int] = None,
|
| 194 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 195 |
+
) -> BatchFeature:
|
| 196 |
+
"""
|
| 197 |
+
Preprocess a given (batch) of text/images for a Prismatic VLM; forwards text to the underlying LLM's tokenizer,
|
| 198 |
+
forwards images to PrismaticImageProcessor.
|
| 199 |
+
@param text: The (batch) of text to encode; must be a string or list of strings.
|
| 200 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 201 |
+
@param padding: Sequence padding strategy (if multiple specified) in < True = "longest" | "max_length" | False >
|
| 202 |
+
@param truncation: Truncation strategy for the output sequences; requires `max_length` to be specified
|
| 203 |
+
@param max_length: Maximum length (in tokens) to truncate
|
| 204 |
+
@param return_tensors: Type of return tensors (usually "pt" or TensorType.PYTORCH)
|
| 205 |
+
@return: BatchFeature with keys for `input_ids`, `attention_mask` and `pixel_values`.
|
| 206 |
+
"""
|
| 207 |
+
pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"]
|
| 208 |
+
text_inputs = self.tokenizer(
|
| 209 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# [Validate] Need same number of images and text inputs!
|
| 213 |
+
if pixel_values.shape[0] != text_inputs.input_ids.shape[0]:
|
| 214 |
+
raise ValueError("Batch is malformed; expected same number of images and text inputs!")
|
| 215 |
+
|
| 216 |
+
return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
|
| 217 |
+
|
| 218 |
+
# === Tokenizer Dispatch Utilities =>> check `PreTrainedTokenizerBase` for documentation ===
|
| 219 |
+
def batch_decode(
|
| 220 |
+
self,
|
| 221 |
+
sequences: Union[List[int], List[List[int]], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 222 |
+
skip_special_tokens: bool = False,
|
| 223 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 224 |
+
**kwargs: str,
|
| 225 |
+
) -> List[str]:
|
| 226 |
+
return self.tokenizer.batch_decode(
|
| 227 |
+
sequences=sequences,
|
| 228 |
+
skip_special_tokens=skip_special_tokens,
|
| 229 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 230 |
+
**kwargs,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def decode(
|
| 234 |
+
self,
|
| 235 |
+
token_ids: Union[int, List[int], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 236 |
+
skip_special_tokens: bool = False,
|
| 237 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 238 |
+
**kwargs: str,
|
| 239 |
+
) -> str:
|
| 240 |
+
return self.tokenizer.decode(
|
| 241 |
+
token_ids=token_ids,
|
| 242 |
+
skip_special_tokens=skip_special_tokens,
|
| 243 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 244 |
+
**kwargs,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def model_input_names(self) -> List[str]:
|
| 249 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 250 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 251 |
+
|
| 252 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PrismaticProcessor"
|
| 6 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<PAD>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-074103--90000_chkpt/tokenizer_config.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"0": {
|
| 6 |
+
"content": "<unk>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"1": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"2": {
|
| 22 |
+
"content": "</s>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"32000": {
|
| 30 |
+
"content": "<PAD>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
"auto_map": {
|
| 39 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 40 |
+
},
|
| 41 |
+
"bos_token": "<s>",
|
| 42 |
+
"clean_up_tokenization_spaces": false,
|
| 43 |
+
"eos_token": "</s>",
|
| 44 |
+
"legacy": false,
|
| 45 |
+
"model_max_length": 2048,
|
| 46 |
+
"pad_token": "<PAD>",
|
| 47 |
+
"padding_side": "right",
|
| 48 |
+
"processor_class": "PrismaticProcessor",
|
| 49 |
+
"sp_model_kwargs": {},
|
| 50 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 51 |
+
"unk_token": "<unk>",
|
| 52 |
+
"use_default_system_prompt": false
|
| 53 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<PAD>": 32000
|
| 3 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/lora_adapter/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: pretrained_models/configs-openvla-7b/config.json
|
| 3 |
+
library_name: peft
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Model Card for Model ID
|
| 7 |
+
|
| 8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## Model Details
|
| 13 |
+
|
| 14 |
+
### Model Description
|
| 15 |
+
|
| 16 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
- **Developed by:** [More Information Needed]
|
| 21 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 22 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 23 |
+
- **Model type:** [More Information Needed]
|
| 24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 25 |
+
- **License:** [More Information Needed]
|
| 26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 27 |
+
|
| 28 |
+
### Model Sources [optional]
|
| 29 |
+
|
| 30 |
+
<!-- Provide the basic links for the model. -->
|
| 31 |
+
|
| 32 |
+
- **Repository:** [More Information Needed]
|
| 33 |
+
- **Paper [optional]:** [More Information Needed]
|
| 34 |
+
- **Demo [optional]:** [More Information Needed]
|
| 35 |
+
|
| 36 |
+
## Uses
|
| 37 |
+
|
| 38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 39 |
+
|
| 40 |
+
### Direct Use
|
| 41 |
+
|
| 42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 43 |
+
|
| 44 |
+
[More Information Needed]
|
| 45 |
+
|
| 46 |
+
### Downstream Use [optional]
|
| 47 |
+
|
| 48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 49 |
+
|
| 50 |
+
[More Information Needed]
|
| 51 |
+
|
| 52 |
+
### Out-of-Scope Use
|
| 53 |
+
|
| 54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 55 |
+
|
| 56 |
+
[More Information Needed]
|
| 57 |
+
|
| 58 |
+
## Bias, Risks, and Limitations
|
| 59 |
+
|
| 60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 61 |
+
|
| 62 |
+
[More Information Needed]
|
| 63 |
+
|
| 64 |
+
### Recommendations
|
| 65 |
+
|
| 66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 67 |
+
|
| 68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 69 |
+
|
| 70 |
+
## How to Get Started with the Model
|
| 71 |
+
|
| 72 |
+
Use the code below to get started with the model.
|
| 73 |
+
|
| 74 |
+
[More Information Needed]
|
| 75 |
+
|
| 76 |
+
## Training Details
|
| 77 |
+
|
| 78 |
+
### Training Data
|
| 79 |
+
|
| 80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 81 |
+
|
| 82 |
+
[More Information Needed]
|
| 83 |
+
|
| 84 |
+
### Training Procedure
|
| 85 |
+
|
| 86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 87 |
+
|
| 88 |
+
#### Preprocessing [optional]
|
| 89 |
+
|
| 90 |
+
[More Information Needed]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
#### Training Hyperparameters
|
| 94 |
+
|
| 95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 96 |
+
|
| 97 |
+
#### Speeds, Sizes, Times [optional]
|
| 98 |
+
|
| 99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 100 |
+
|
| 101 |
+
[More Information Needed]
|
| 102 |
+
|
| 103 |
+
## Evaluation
|
| 104 |
+
|
| 105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 106 |
+
|
| 107 |
+
### Testing Data, Factors & Metrics
|
| 108 |
+
|
| 109 |
+
#### Testing Data
|
| 110 |
+
|
| 111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 112 |
+
|
| 113 |
+
[More Information Needed]
|
| 114 |
+
|
| 115 |
+
#### Factors
|
| 116 |
+
|
| 117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 118 |
+
|
| 119 |
+
[More Information Needed]
|
| 120 |
+
|
| 121 |
+
#### Metrics
|
| 122 |
+
|
| 123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 124 |
+
|
| 125 |
+
[More Information Needed]
|
| 126 |
+
|
| 127 |
+
### Results
|
| 128 |
+
|
| 129 |
+
[More Information Needed]
|
| 130 |
+
|
| 131 |
+
#### Summary
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
## Model Examination [optional]
|
| 136 |
+
|
| 137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 138 |
+
|
| 139 |
+
[More Information Needed]
|
| 140 |
+
|
| 141 |
+
## Environmental Impact
|
| 142 |
+
|
| 143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 144 |
+
|
| 145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 146 |
+
|
| 147 |
+
- **Hardware Type:** [More Information Needed]
|
| 148 |
+
- **Hours used:** [More Information Needed]
|
| 149 |
+
- **Cloud Provider:** [More Information Needed]
|
| 150 |
+
- **Compute Region:** [More Information Needed]
|
| 151 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 152 |
+
|
| 153 |
+
## Technical Specifications [optional]
|
| 154 |
+
|
| 155 |
+
### Model Architecture and Objective
|
| 156 |
+
|
| 157 |
+
[More Information Needed]
|
| 158 |
+
|
| 159 |
+
### Compute Infrastructure
|
| 160 |
+
|
| 161 |
+
[More Information Needed]
|
| 162 |
+
|
| 163 |
+
#### Hardware
|
| 164 |
+
|
| 165 |
+
[More Information Needed]
|
| 166 |
+
|
| 167 |
+
#### Software
|
| 168 |
+
|
| 169 |
+
[More Information Needed]
|
| 170 |
+
|
| 171 |
+
## Citation [optional]
|
| 172 |
+
|
| 173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 174 |
+
|
| 175 |
+
**BibTeX:**
|
| 176 |
+
|
| 177 |
+
[More Information Needed]
|
| 178 |
+
|
| 179 |
+
**APA:**
|
| 180 |
+
|
| 181 |
+
[More Information Needed]
|
| 182 |
+
|
| 183 |
+
## Glossary [optional]
|
| 184 |
+
|
| 185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 186 |
+
|
| 187 |
+
[More Information Needed]
|
| 188 |
+
|
| 189 |
+
## More Information [optional]
|
| 190 |
+
|
| 191 |
+
[More Information Needed]
|
| 192 |
+
|
| 193 |
+
## Model Card Authors [optional]
|
| 194 |
+
|
| 195 |
+
[More Information Needed]
|
| 196 |
+
|
| 197 |
+
## Model Card Contact
|
| 198 |
+
|
| 199 |
+
[More Information Needed]
|
| 200 |
+
### Framework versions
|
| 201 |
+
|
| 202 |
+
- PEFT 0.11.1
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/lora_adapter/adapter_config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alpha_pattern": {},
|
| 3 |
+
"auto_mapping": {
|
| 4 |
+
"base_model_class": "OpenVLAForActionPrediction",
|
| 5 |
+
"parent_library": "prismatic.extern.hf.modeling_prismatic"
|
| 6 |
+
},
|
| 7 |
+
"base_model_name_or_path": "pretrained_models/configs-openvla-7b/config.json",
|
| 8 |
+
"bias": "none",
|
| 9 |
+
"fan_in_fan_out": false,
|
| 10 |
+
"inference_mode": true,
|
| 11 |
+
"init_lora_weights": "gaussian",
|
| 12 |
+
"layer_replication": null,
|
| 13 |
+
"layers_pattern": null,
|
| 14 |
+
"layers_to_transform": null,
|
| 15 |
+
"loftq_config": {},
|
| 16 |
+
"lora_alpha": 64,
|
| 17 |
+
"lora_dropout": 0.0,
|
| 18 |
+
"megatron_config": null,
|
| 19 |
+
"megatron_core": "megatron.core",
|
| 20 |
+
"modules_to_save": null,
|
| 21 |
+
"peft_type": "LORA",
|
| 22 |
+
"r": 32,
|
| 23 |
+
"rank_pattern": {},
|
| 24 |
+
"revision": null,
|
| 25 |
+
"target_modules": [
|
| 26 |
+
"q_proj",
|
| 27 |
+
"o_proj",
|
| 28 |
+
"proj",
|
| 29 |
+
"qkv",
|
| 30 |
+
"kv",
|
| 31 |
+
"q",
|
| 32 |
+
"v_proj",
|
| 33 |
+
"fc3",
|
| 34 |
+
"down_proj",
|
| 35 |
+
"gate_proj",
|
| 36 |
+
"up_proj",
|
| 37 |
+
"lm_head",
|
| 38 |
+
"k_proj",
|
| 39 |
+
"fc2",
|
| 40 |
+
"fc1"
|
| 41 |
+
],
|
| 42 |
+
"task_type": null,
|
| 43 |
+
"use_dora": false,
|
| 44 |
+
"use_rslora": false
|
| 45 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/preprocessor_config.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "processing_prismatic.PrismaticImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 5 |
+
},
|
| 6 |
+
"image_processor_type": "PrismaticImageProcessor",
|
| 7 |
+
"image_resize_strategy": "resize-naive",
|
| 8 |
+
"input_sizes": [
|
| 9 |
+
[
|
| 10 |
+
3,
|
| 11 |
+
224,
|
| 12 |
+
224
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
3,
|
| 16 |
+
224,
|
| 17 |
+
224
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"interpolations": [
|
| 21 |
+
"bicubic",
|
| 22 |
+
"bicubic"
|
| 23 |
+
],
|
| 24 |
+
"means": [
|
| 25 |
+
[
|
| 26 |
+
0.485,
|
| 27 |
+
0.456,
|
| 28 |
+
0.406
|
| 29 |
+
],
|
| 30 |
+
[
|
| 31 |
+
0.5,
|
| 32 |
+
0.5,
|
| 33 |
+
0.5
|
| 34 |
+
]
|
| 35 |
+
],
|
| 36 |
+
"processor_class": "PrismaticProcessor",
|
| 37 |
+
"stds": [
|
| 38 |
+
[
|
| 39 |
+
0.229,
|
| 40 |
+
0.224,
|
| 41 |
+
0.225
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
0.5,
|
| 45 |
+
0.5,
|
| 46 |
+
0.5
|
| 47 |
+
]
|
| 48 |
+
],
|
| 49 |
+
"tvf_crop_params": [
|
| 50 |
+
{
|
| 51 |
+
"output_size": [
|
| 52 |
+
224,
|
| 53 |
+
224
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"output_size": [
|
| 58 |
+
224,
|
| 59 |
+
224
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"tvf_do_letterbox": false,
|
| 64 |
+
"tvf_letterbox_fill": null,
|
| 65 |
+
"tvf_normalize_params": [
|
| 66 |
+
{
|
| 67 |
+
"inplace": false,
|
| 68 |
+
"mean": [
|
| 69 |
+
0.484375,
|
| 70 |
+
0.455078125,
|
| 71 |
+
0.40625
|
| 72 |
+
],
|
| 73 |
+
"std": [
|
| 74 |
+
0.228515625,
|
| 75 |
+
0.2236328125,
|
| 76 |
+
0.224609375
|
| 77 |
+
]
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"inplace": false,
|
| 81 |
+
"mean": [
|
| 82 |
+
0.5,
|
| 83 |
+
0.5,
|
| 84 |
+
0.5
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.5,
|
| 88 |
+
0.5,
|
| 89 |
+
0.5
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"tvf_resize_params": [
|
| 94 |
+
{
|
| 95 |
+
"antialias": true,
|
| 96 |
+
"interpolation": 3,
|
| 97 |
+
"max_size": null,
|
| 98 |
+
"size": [
|
| 99 |
+
224,
|
| 100 |
+
224
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"antialias": true,
|
| 105 |
+
"interpolation": 3,
|
| 106 |
+
"max_size": null,
|
| 107 |
+
"size": [
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"use_fused_vision_backbone": true
|
| 114 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/processing_prismatic.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
processing_prismatic.py
|
| 3 |
+
|
| 4 |
+
HuggingFace-style preprocessor definitions for Prismatic VLMs, inheriting from `ProcessorMixin`. Default configuration
|
| 5 |
+
specifies `siglip-224px+7b`.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Any, ClassVar, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import timm.data
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms.functional as TVF
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor
|
| 15 |
+
from transformers import PreTrainedTokenizerBase
|
| 16 |
+
from transformers.image_processing_utils import BatchFeature, ImageProcessingMixin
|
| 17 |
+
from transformers.processing_utils import ProcessorMixin
|
| 18 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| 19 |
+
from transformers.utils import TensorType
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# === Image Processing ===
|
| 23 |
+
def letterbox_pad_transform(image: Image.Image, padding_fill_value: Tuple[int, int, int]) -> Image.Image:
|
| 24 |
+
"""Given a PIL.Image, pad to square by adding a symmetric border around the height/width."""
|
| 25 |
+
(w, h), max_wh = image.size, max(image.size)
|
| 26 |
+
horizontal_pad, vertical_pad = int((max_wh - w) / 2), int((max_wh - h) / 2)
|
| 27 |
+
padding = (horizontal_pad, vertical_pad, horizontal_pad, vertical_pad)
|
| 28 |
+
|
| 29 |
+
return TVF.pad(image, padding, fill=padding_fill_value, padding_mode="constant")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PrismaticImageProcessor(ImageProcessingMixin):
|
| 33 |
+
model_input_names: ClassVar[List[str]] = ["pixel_values"]
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
use_fused_vision_backbone: bool = False,
|
| 38 |
+
image_resize_strategy: str = "letterbox",
|
| 39 |
+
input_sizes: Optional[List[Tuple[int, int, int]]] = None,
|
| 40 |
+
interpolations: Optional[List[str]] = None,
|
| 41 |
+
means: Optional[List[Tuple[float, float, float]]] = None,
|
| 42 |
+
stds: Optional[List[Tuple[float, float, float]]] = None,
|
| 43 |
+
**kwargs: str,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""
|
| 46 |
+
Initialize a PrismaticImageProcessor as a wrapper around a torchvision transform; this transform will be
|
| 47 |
+
created by TIMM, and edited to follow our custom `image_resize_strategy` logic.
|
| 48 |
+
@param use_fused_vision_backbone: Boolean indicating single or fused (dual) vision backbone
|
| 49 |
+
@param image_resize_strategy: Prismatic image resize strategy in < resize-naive | resize-crop | letterbox >
|
| 50 |
+
@param input_size: [TIMM :: `data_cfg`] Input image size as tuple (channels, width, height)
|
| 51 |
+
@param interpolation: [TIMM :: `data_cfg`] Interpolation as string (default: "bicubic")
|
| 52 |
+
@param mean: [TIMM :: `data_cfg`] Normalization mean as float tuple (or two-tuple if `fused_backbone`)
|
| 53 |
+
@param std: [TIMM :: `data_cfg`] Normalization std as float tuple (or two-tuple if `fused_backbone`)
|
| 54 |
+
"""
|
| 55 |
+
self.use_fused_vision_backbone = use_fused_vision_backbone
|
| 56 |
+
self.image_resize_strategy = image_resize_strategy
|
| 57 |
+
|
| 58 |
+
# Handle `None` default values
|
| 59 |
+
input_sizes = [(3, 224, 224)] if input_sizes is None else input_sizes
|
| 60 |
+
means = [(0.5, 0.5, 0.5)] if means is None else means
|
| 61 |
+
stds = [(0.5, 0.5, 0.5)] if stds is None else stds
|
| 62 |
+
|
| 63 |
+
# TIMM `data_cfg` Parameters
|
| 64 |
+
self.input_sizes, self.interpolations, self.means, self.stds = input_sizes, interpolations, means, stds
|
| 65 |
+
|
| 66 |
+
# Grab torchvision transforms via TIMM =>> need to parse for specific "functional" transform values!
|
| 67 |
+
self.tvf_resize_params, self.tvf_crop_params, self.tvf_normalize_params = [], [], []
|
| 68 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 69 |
+
|
| 70 |
+
for idx in range(len(input_sizes)):
|
| 71 |
+
transform = timm.data.create_transform(
|
| 72 |
+
input_size=self.input_sizes[idx],
|
| 73 |
+
interpolation=self.interpolations[idx],
|
| 74 |
+
mean=self.means[idx],
|
| 75 |
+
std=self.stds[idx],
|
| 76 |
+
crop_pct=1.0, # Set to 1.0 to ignore cropping (initial Resize sets `input_size`)
|
| 77 |
+
crop_mode="center", # Default crop mode -- no-op when `crop_pct == 1.0`
|
| 78 |
+
is_training=False, # No image augmentations when loading the transform!
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# [Validation] Ensure appropriate transform structure, expected sizes
|
| 82 |
+
if not (
|
| 83 |
+
isinstance(transform, Compose)
|
| 84 |
+
and (len(transform.transforms) == 4)
|
| 85 |
+
and isinstance(transform.transforms[0], Resize)
|
| 86 |
+
and isinstance(transform.transforms[1], CenterCrop)
|
| 87 |
+
and isinstance(transform.transforms[2], ToTensor)
|
| 88 |
+
and isinstance(transform.transforms[3], Normalize)
|
| 89 |
+
and (transform.transforms[0].size == self.input_sizes[idx][-1])
|
| 90 |
+
and (transform.transforms[1].size == self.input_sizes[idx][-2:])
|
| 91 |
+
):
|
| 92 |
+
raise ValueError(f"Unexpected TIMM image transformation structure/sizes: `{transform}`")
|
| 93 |
+
|
| 94 |
+
# HF Image Processors *must* be JSON-serializable; as such, cannot have torchvision. as an attribute.
|
| 95 |
+
# => Instead, we're going to parse the transform and call "torchvision.transforms.functional" (`tvf`)
|
| 96 |
+
resize_t, crop_t, norm_t = transform.transforms[0], transform.transforms[1], transform.transforms[3]
|
| 97 |
+
self.tvf_resize_params.append(
|
| 98 |
+
{
|
| 99 |
+
"size": resize_t.size,
|
| 100 |
+
"interpolation": TVF.pil_modes_mapping[resize_t.interpolation],
|
| 101 |
+
"max_size": None,
|
| 102 |
+
"antialias": True,
|
| 103 |
+
}
|
| 104 |
+
)
|
| 105 |
+
self.tvf_crop_params.append({"output_size": crop_t.size})
|
| 106 |
+
self.tvf_normalize_params.append(
|
| 107 |
+
{
|
| 108 |
+
"mean": norm_t.mean.float().numpy().tolist(),
|
| 109 |
+
"std": norm_t.std.float().numpy().tolist(),
|
| 110 |
+
"inplace": False,
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 114 |
+
|
| 115 |
+
# Handle Prismatic `image_resize_strategy`
|
| 116 |
+
if self.image_resize_strategy == "resize-naive":
|
| 117 |
+
self.tvf_resize_params[idx]["size"] = (resize_t.size, resize_t.size)
|
| 118 |
+
elif self.image_resize_strategy == "letterbox":
|
| 119 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = True, tuple([int(x * 255) for x in self.means[idx]])
|
| 120 |
+
elif self.image_resize_strategy == "resize-crop":
|
| 121 |
+
pass
|
| 122 |
+
else:
|
| 123 |
+
raise ValueError(f"Image resize strategy `{self.image_resize_strategy}` is not supported!")
|
| 124 |
+
|
| 125 |
+
# Dispatch **kwargs to super()
|
| 126 |
+
super().__init__(**kwargs)
|
| 127 |
+
|
| 128 |
+
def apply_transform(self, img: Image.Image) -> torch.Tensor:
|
| 129 |
+
"""Apply `functional` variant of TIMM's Transform = Compose([Resize -> CenterCrop -> ToTensor -> Normalize])"""
|
| 130 |
+
if self.tvf_do_letterbox:
|
| 131 |
+
img = letterbox_pad_transform(img, self.tvf_letterbox_fill)
|
| 132 |
+
|
| 133 |
+
# [Contract] Fused Backbones expect "channel-stacked" inputs; we'll unpack on the model side!
|
| 134 |
+
imgs_t = []
|
| 135 |
+
for idx in range(len(self.input_sizes)):
|
| 136 |
+
img_idx = TVF.resize(img, **self.tvf_resize_params[idx])
|
| 137 |
+
img_idx = TVF.center_crop(img_idx, **self.tvf_crop_params[idx])
|
| 138 |
+
img_idx_t = TVF.to_tensor(img_idx)
|
| 139 |
+
img_idx_t = TVF.normalize(img_idx_t, **self.tvf_normalize_params[idx])
|
| 140 |
+
imgs_t.append(img_idx_t)
|
| 141 |
+
|
| 142 |
+
# [Contract] `imgs_t` is a list of Tensors of shape [3, input_size, input_size]; stack along dim = 0
|
| 143 |
+
img_t = torch.vstack(imgs_t)
|
| 144 |
+
|
| 145 |
+
return img_t
|
| 146 |
+
|
| 147 |
+
def preprocess(
|
| 148 |
+
self,
|
| 149 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 150 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 151 |
+
**_: str,
|
| 152 |
+
) -> BatchFeature:
|
| 153 |
+
"""
|
| 154 |
+
Preprocess an image (or batch of images); note that unlike the `transformers :: BaseImageProcessor` we
|
| 155 |
+
explicitly only handle PIL.Image.Image instances for simplicity.
|
| 156 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 157 |
+
@param return_tensors: BatchFeature default Tensor format (e.g., "pt" for torch); if None, returns np.ndarray
|
| 158 |
+
@return: Instance of `transformers :: BatchFeature` with a single key "pixel_values"
|
| 159 |
+
"""
|
| 160 |
+
if not isinstance(images, list):
|
| 161 |
+
images = [images]
|
| 162 |
+
|
| 163 |
+
# Apply `self.img_transform` to each image (will return list of torch.Tensors); stack into "batched" Tensor
|
| 164 |
+
pixel_values = torch.stack([self.apply_transform(img.convert("RGB")) for img in images])
|
| 165 |
+
|
| 166 |
+
# Return BatchFeature =>> note that for compatibility, constructor expects Dict[str, np.ndarray], so we convert
|
| 167 |
+
return BatchFeature(data={"pixel_values": pixel_values.float().numpy()}, tensor_type=return_tensors)
|
| 168 |
+
|
| 169 |
+
def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> BatchFeature:
|
| 170 |
+
return self.preprocess(images, **kwargs)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# === PrismaticProcessor =>> Wraps both ImageProcessor and Tokenizer ===
|
| 174 |
+
# =>> https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/processing_llava.py
|
| 175 |
+
class PrismaticProcessor(ProcessorMixin):
|
| 176 |
+
attributes: ClassVar[List[str]] = ["image_processor", "tokenizer"]
|
| 177 |
+
image_processor_class: str = "AutoImageProcessor"
|
| 178 |
+
tokenizer_class: str = "AutoTokenizer"
|
| 179 |
+
|
| 180 |
+
def __init__(
|
| 181 |
+
self,
|
| 182 |
+
image_processor: Optional[ImageProcessingMixin] = None,
|
| 183 |
+
tokenizer: Optional[PreTrainedTokenizerBase] = None,
|
| 184 |
+
) -> None:
|
| 185 |
+
super().__init__(image_processor, tokenizer)
|
| 186 |
+
|
| 187 |
+
def __call__(
|
| 188 |
+
self,
|
| 189 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
|
| 190 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 191 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 192 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 193 |
+
max_length: Optional[int] = None,
|
| 194 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 195 |
+
) -> BatchFeature:
|
| 196 |
+
"""
|
| 197 |
+
Preprocess a given (batch) of text/images for a Prismatic VLM; forwards text to the underlying LLM's tokenizer,
|
| 198 |
+
forwards images to PrismaticImageProcessor.
|
| 199 |
+
@param text: The (batch) of text to encode; must be a string or list of strings.
|
| 200 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 201 |
+
@param padding: Sequence padding strategy (if multiple specified) in < True = "longest" | "max_length" | False >
|
| 202 |
+
@param truncation: Truncation strategy for the output sequences; requires `max_length` to be specified
|
| 203 |
+
@param max_length: Maximum length (in tokens) to truncate
|
| 204 |
+
@param return_tensors: Type of return tensors (usually "pt" or TensorType.PYTORCH)
|
| 205 |
+
@return: BatchFeature with keys for `input_ids`, `attention_mask` and `pixel_values`.
|
| 206 |
+
"""
|
| 207 |
+
pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"]
|
| 208 |
+
text_inputs = self.tokenizer(
|
| 209 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# [Validate] Need same number of images and text inputs!
|
| 213 |
+
if pixel_values.shape[0] != text_inputs.input_ids.shape[0]:
|
| 214 |
+
raise ValueError("Batch is malformed; expected same number of images and text inputs!")
|
| 215 |
+
|
| 216 |
+
return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
|
| 217 |
+
|
| 218 |
+
# === Tokenizer Dispatch Utilities =>> check `PreTrainedTokenizerBase` for documentation ===
|
| 219 |
+
def batch_decode(
|
| 220 |
+
self,
|
| 221 |
+
sequences: Union[List[int], List[List[int]], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 222 |
+
skip_special_tokens: bool = False,
|
| 223 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 224 |
+
**kwargs: str,
|
| 225 |
+
) -> List[str]:
|
| 226 |
+
return self.tokenizer.batch_decode(
|
| 227 |
+
sequences=sequences,
|
| 228 |
+
skip_special_tokens=skip_special_tokens,
|
| 229 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 230 |
+
**kwargs,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def decode(
|
| 234 |
+
self,
|
| 235 |
+
token_ids: Union[int, List[int], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 236 |
+
skip_special_tokens: bool = False,
|
| 237 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 238 |
+
**kwargs: str,
|
| 239 |
+
) -> str:
|
| 240 |
+
return self.tokenizer.decode(
|
| 241 |
+
token_ids=token_ids,
|
| 242 |
+
skip_special_tokens=skip_special_tokens,
|
| 243 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 244 |
+
**kwargs,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def model_input_names(self) -> List[str]:
|
| 249 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 250 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 251 |
+
|
| 252 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PrismaticProcessor"
|
| 6 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<PAD>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/tokenizer_config.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"0": {
|
| 6 |
+
"content": "<unk>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"1": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"2": {
|
| 22 |
+
"content": "</s>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"32000": {
|
| 30 |
+
"content": "<PAD>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
"auto_map": {
|
| 39 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 40 |
+
},
|
| 41 |
+
"bos_token": "<s>",
|
| 42 |
+
"clean_up_tokenization_spaces": false,
|
| 43 |
+
"eos_token": "</s>",
|
| 44 |
+
"legacy": false,
|
| 45 |
+
"model_max_length": 2048,
|
| 46 |
+
"pad_token": "<PAD>",
|
| 47 |
+
"padding_side": "right",
|
| 48 |
+
"processor_class": "PrismaticProcessor",
|
| 49 |
+
"sp_model_kwargs": {},
|
| 50 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 51 |
+
"unk_token": "<unk>",
|
| 52 |
+
"use_default_system_prompt": false
|
| 53 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--45000_chkpt/train_state--45000_checkpoint.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c117fa9c17063bf29012c17389ef18701a47a580cdaf45075f29ed4d5f5e7f83
|
| 3 |
+
size 2079629718
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<PAD>": 32000
|
| 3 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/lora_adapter/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: pretrained_models/configs-openvla-7b/config.json
|
| 3 |
+
library_name: peft
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Model Card for Model ID
|
| 7 |
+
|
| 8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## Model Details
|
| 13 |
+
|
| 14 |
+
### Model Description
|
| 15 |
+
|
| 16 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
- **Developed by:** [More Information Needed]
|
| 21 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 22 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 23 |
+
- **Model type:** [More Information Needed]
|
| 24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 25 |
+
- **License:** [More Information Needed]
|
| 26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 27 |
+
|
| 28 |
+
### Model Sources [optional]
|
| 29 |
+
|
| 30 |
+
<!-- Provide the basic links for the model. -->
|
| 31 |
+
|
| 32 |
+
- **Repository:** [More Information Needed]
|
| 33 |
+
- **Paper [optional]:** [More Information Needed]
|
| 34 |
+
- **Demo [optional]:** [More Information Needed]
|
| 35 |
+
|
| 36 |
+
## Uses
|
| 37 |
+
|
| 38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 39 |
+
|
| 40 |
+
### Direct Use
|
| 41 |
+
|
| 42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 43 |
+
|
| 44 |
+
[More Information Needed]
|
| 45 |
+
|
| 46 |
+
### Downstream Use [optional]
|
| 47 |
+
|
| 48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 49 |
+
|
| 50 |
+
[More Information Needed]
|
| 51 |
+
|
| 52 |
+
### Out-of-Scope Use
|
| 53 |
+
|
| 54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 55 |
+
|
| 56 |
+
[More Information Needed]
|
| 57 |
+
|
| 58 |
+
## Bias, Risks, and Limitations
|
| 59 |
+
|
| 60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 61 |
+
|
| 62 |
+
[More Information Needed]
|
| 63 |
+
|
| 64 |
+
### Recommendations
|
| 65 |
+
|
| 66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 67 |
+
|
| 68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 69 |
+
|
| 70 |
+
## How to Get Started with the Model
|
| 71 |
+
|
| 72 |
+
Use the code below to get started with the model.
|
| 73 |
+
|
| 74 |
+
[More Information Needed]
|
| 75 |
+
|
| 76 |
+
## Training Details
|
| 77 |
+
|
| 78 |
+
### Training Data
|
| 79 |
+
|
| 80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 81 |
+
|
| 82 |
+
[More Information Needed]
|
| 83 |
+
|
| 84 |
+
### Training Procedure
|
| 85 |
+
|
| 86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 87 |
+
|
| 88 |
+
#### Preprocessing [optional]
|
| 89 |
+
|
| 90 |
+
[More Information Needed]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
#### Training Hyperparameters
|
| 94 |
+
|
| 95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 96 |
+
|
| 97 |
+
#### Speeds, Sizes, Times [optional]
|
| 98 |
+
|
| 99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 100 |
+
|
| 101 |
+
[More Information Needed]
|
| 102 |
+
|
| 103 |
+
## Evaluation
|
| 104 |
+
|
| 105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 106 |
+
|
| 107 |
+
### Testing Data, Factors & Metrics
|
| 108 |
+
|
| 109 |
+
#### Testing Data
|
| 110 |
+
|
| 111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 112 |
+
|
| 113 |
+
[More Information Needed]
|
| 114 |
+
|
| 115 |
+
#### Factors
|
| 116 |
+
|
| 117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 118 |
+
|
| 119 |
+
[More Information Needed]
|
| 120 |
+
|
| 121 |
+
#### Metrics
|
| 122 |
+
|
| 123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 124 |
+
|
| 125 |
+
[More Information Needed]
|
| 126 |
+
|
| 127 |
+
### Results
|
| 128 |
+
|
| 129 |
+
[More Information Needed]
|
| 130 |
+
|
| 131 |
+
#### Summary
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
## Model Examination [optional]
|
| 136 |
+
|
| 137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 138 |
+
|
| 139 |
+
[More Information Needed]
|
| 140 |
+
|
| 141 |
+
## Environmental Impact
|
| 142 |
+
|
| 143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 144 |
+
|
| 145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 146 |
+
|
| 147 |
+
- **Hardware Type:** [More Information Needed]
|
| 148 |
+
- **Hours used:** [More Information Needed]
|
| 149 |
+
- **Cloud Provider:** [More Information Needed]
|
| 150 |
+
- **Compute Region:** [More Information Needed]
|
| 151 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 152 |
+
|
| 153 |
+
## Technical Specifications [optional]
|
| 154 |
+
|
| 155 |
+
### Model Architecture and Objective
|
| 156 |
+
|
| 157 |
+
[More Information Needed]
|
| 158 |
+
|
| 159 |
+
### Compute Infrastructure
|
| 160 |
+
|
| 161 |
+
[More Information Needed]
|
| 162 |
+
|
| 163 |
+
#### Hardware
|
| 164 |
+
|
| 165 |
+
[More Information Needed]
|
| 166 |
+
|
| 167 |
+
#### Software
|
| 168 |
+
|
| 169 |
+
[More Information Needed]
|
| 170 |
+
|
| 171 |
+
## Citation [optional]
|
| 172 |
+
|
| 173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 174 |
+
|
| 175 |
+
**BibTeX:**
|
| 176 |
+
|
| 177 |
+
[More Information Needed]
|
| 178 |
+
|
| 179 |
+
**APA:**
|
| 180 |
+
|
| 181 |
+
[More Information Needed]
|
| 182 |
+
|
| 183 |
+
## Glossary [optional]
|
| 184 |
+
|
| 185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 186 |
+
|
| 187 |
+
[More Information Needed]
|
| 188 |
+
|
| 189 |
+
## More Information [optional]
|
| 190 |
+
|
| 191 |
+
[More Information Needed]
|
| 192 |
+
|
| 193 |
+
## Model Card Authors [optional]
|
| 194 |
+
|
| 195 |
+
[More Information Needed]
|
| 196 |
+
|
| 197 |
+
## Model Card Contact
|
| 198 |
+
|
| 199 |
+
[More Information Needed]
|
| 200 |
+
### Framework versions
|
| 201 |
+
|
| 202 |
+
- PEFT 0.11.1
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/lora_adapter/adapter_config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alpha_pattern": {},
|
| 3 |
+
"auto_mapping": {
|
| 4 |
+
"base_model_class": "OpenVLAForActionPrediction",
|
| 5 |
+
"parent_library": "prismatic.extern.hf.modeling_prismatic"
|
| 6 |
+
},
|
| 7 |
+
"base_model_name_or_path": "pretrained_models/configs-openvla-7b/config.json",
|
| 8 |
+
"bias": "none",
|
| 9 |
+
"fan_in_fan_out": false,
|
| 10 |
+
"inference_mode": true,
|
| 11 |
+
"init_lora_weights": "gaussian",
|
| 12 |
+
"layer_replication": null,
|
| 13 |
+
"layers_pattern": null,
|
| 14 |
+
"layers_to_transform": null,
|
| 15 |
+
"loftq_config": {},
|
| 16 |
+
"lora_alpha": 64,
|
| 17 |
+
"lora_dropout": 0.0,
|
| 18 |
+
"megatron_config": null,
|
| 19 |
+
"megatron_core": "megatron.core",
|
| 20 |
+
"modules_to_save": null,
|
| 21 |
+
"peft_type": "LORA",
|
| 22 |
+
"r": 32,
|
| 23 |
+
"rank_pattern": {},
|
| 24 |
+
"revision": null,
|
| 25 |
+
"target_modules": [
|
| 26 |
+
"q_proj",
|
| 27 |
+
"o_proj",
|
| 28 |
+
"proj",
|
| 29 |
+
"qkv",
|
| 30 |
+
"kv",
|
| 31 |
+
"q",
|
| 32 |
+
"v_proj",
|
| 33 |
+
"fc3",
|
| 34 |
+
"down_proj",
|
| 35 |
+
"gate_proj",
|
| 36 |
+
"up_proj",
|
| 37 |
+
"lm_head",
|
| 38 |
+
"k_proj",
|
| 39 |
+
"fc2",
|
| 40 |
+
"fc1"
|
| 41 |
+
],
|
| 42 |
+
"task_type": null,
|
| 43 |
+
"use_dora": false,
|
| 44 |
+
"use_rslora": false
|
| 45 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/preprocessor_config.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "processing_prismatic.PrismaticImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 5 |
+
},
|
| 6 |
+
"image_processor_type": "PrismaticImageProcessor",
|
| 7 |
+
"image_resize_strategy": "resize-naive",
|
| 8 |
+
"input_sizes": [
|
| 9 |
+
[
|
| 10 |
+
3,
|
| 11 |
+
224,
|
| 12 |
+
224
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
3,
|
| 16 |
+
224,
|
| 17 |
+
224
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"interpolations": [
|
| 21 |
+
"bicubic",
|
| 22 |
+
"bicubic"
|
| 23 |
+
],
|
| 24 |
+
"means": [
|
| 25 |
+
[
|
| 26 |
+
0.485,
|
| 27 |
+
0.456,
|
| 28 |
+
0.406
|
| 29 |
+
],
|
| 30 |
+
[
|
| 31 |
+
0.5,
|
| 32 |
+
0.5,
|
| 33 |
+
0.5
|
| 34 |
+
]
|
| 35 |
+
],
|
| 36 |
+
"processor_class": "PrismaticProcessor",
|
| 37 |
+
"stds": [
|
| 38 |
+
[
|
| 39 |
+
0.229,
|
| 40 |
+
0.224,
|
| 41 |
+
0.225
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
0.5,
|
| 45 |
+
0.5,
|
| 46 |
+
0.5
|
| 47 |
+
]
|
| 48 |
+
],
|
| 49 |
+
"tvf_crop_params": [
|
| 50 |
+
{
|
| 51 |
+
"output_size": [
|
| 52 |
+
224,
|
| 53 |
+
224
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"output_size": [
|
| 58 |
+
224,
|
| 59 |
+
224
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"tvf_do_letterbox": false,
|
| 64 |
+
"tvf_letterbox_fill": null,
|
| 65 |
+
"tvf_normalize_params": [
|
| 66 |
+
{
|
| 67 |
+
"inplace": false,
|
| 68 |
+
"mean": [
|
| 69 |
+
0.484375,
|
| 70 |
+
0.455078125,
|
| 71 |
+
0.40625
|
| 72 |
+
],
|
| 73 |
+
"std": [
|
| 74 |
+
0.228515625,
|
| 75 |
+
0.2236328125,
|
| 76 |
+
0.224609375
|
| 77 |
+
]
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"inplace": false,
|
| 81 |
+
"mean": [
|
| 82 |
+
0.5,
|
| 83 |
+
0.5,
|
| 84 |
+
0.5
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.5,
|
| 88 |
+
0.5,
|
| 89 |
+
0.5
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"tvf_resize_params": [
|
| 94 |
+
{
|
| 95 |
+
"antialias": true,
|
| 96 |
+
"interpolation": 3,
|
| 97 |
+
"max_size": null,
|
| 98 |
+
"size": [
|
| 99 |
+
224,
|
| 100 |
+
224
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"antialias": true,
|
| 105 |
+
"interpolation": 3,
|
| 106 |
+
"max_size": null,
|
| 107 |
+
"size": [
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"use_fused_vision_backbone": true
|
| 114 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/processing_prismatic.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
processing_prismatic.py
|
| 3 |
+
|
| 4 |
+
HuggingFace-style preprocessor definitions for Prismatic VLMs, inheriting from `ProcessorMixin`. Default configuration
|
| 5 |
+
specifies `siglip-224px+7b`.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Any, ClassVar, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import timm.data
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms.functional as TVF
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor
|
| 15 |
+
from transformers import PreTrainedTokenizerBase
|
| 16 |
+
from transformers.image_processing_utils import BatchFeature, ImageProcessingMixin
|
| 17 |
+
from transformers.processing_utils import ProcessorMixin
|
| 18 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| 19 |
+
from transformers.utils import TensorType
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# === Image Processing ===
|
| 23 |
+
def letterbox_pad_transform(image: Image.Image, padding_fill_value: Tuple[int, int, int]) -> Image.Image:
|
| 24 |
+
"""Given a PIL.Image, pad to square by adding a symmetric border around the height/width."""
|
| 25 |
+
(w, h), max_wh = image.size, max(image.size)
|
| 26 |
+
horizontal_pad, vertical_pad = int((max_wh - w) / 2), int((max_wh - h) / 2)
|
| 27 |
+
padding = (horizontal_pad, vertical_pad, horizontal_pad, vertical_pad)
|
| 28 |
+
|
| 29 |
+
return TVF.pad(image, padding, fill=padding_fill_value, padding_mode="constant")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PrismaticImageProcessor(ImageProcessingMixin):
|
| 33 |
+
model_input_names: ClassVar[List[str]] = ["pixel_values"]
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
use_fused_vision_backbone: bool = False,
|
| 38 |
+
image_resize_strategy: str = "letterbox",
|
| 39 |
+
input_sizes: Optional[List[Tuple[int, int, int]]] = None,
|
| 40 |
+
interpolations: Optional[List[str]] = None,
|
| 41 |
+
means: Optional[List[Tuple[float, float, float]]] = None,
|
| 42 |
+
stds: Optional[List[Tuple[float, float, float]]] = None,
|
| 43 |
+
**kwargs: str,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""
|
| 46 |
+
Initialize a PrismaticImageProcessor as a wrapper around a torchvision transform; this transform will be
|
| 47 |
+
created by TIMM, and edited to follow our custom `image_resize_strategy` logic.
|
| 48 |
+
@param use_fused_vision_backbone: Boolean indicating single or fused (dual) vision backbone
|
| 49 |
+
@param image_resize_strategy: Prismatic image resize strategy in < resize-naive | resize-crop | letterbox >
|
| 50 |
+
@param input_size: [TIMM :: `data_cfg`] Input image size as tuple (channels, width, height)
|
| 51 |
+
@param interpolation: [TIMM :: `data_cfg`] Interpolation as string (default: "bicubic")
|
| 52 |
+
@param mean: [TIMM :: `data_cfg`] Normalization mean as float tuple (or two-tuple if `fused_backbone`)
|
| 53 |
+
@param std: [TIMM :: `data_cfg`] Normalization std as float tuple (or two-tuple if `fused_backbone`)
|
| 54 |
+
"""
|
| 55 |
+
self.use_fused_vision_backbone = use_fused_vision_backbone
|
| 56 |
+
self.image_resize_strategy = image_resize_strategy
|
| 57 |
+
|
| 58 |
+
# Handle `None` default values
|
| 59 |
+
input_sizes = [(3, 224, 224)] if input_sizes is None else input_sizes
|
| 60 |
+
means = [(0.5, 0.5, 0.5)] if means is None else means
|
| 61 |
+
stds = [(0.5, 0.5, 0.5)] if stds is None else stds
|
| 62 |
+
|
| 63 |
+
# TIMM `data_cfg` Parameters
|
| 64 |
+
self.input_sizes, self.interpolations, self.means, self.stds = input_sizes, interpolations, means, stds
|
| 65 |
+
|
| 66 |
+
# Grab torchvision transforms via TIMM =>> need to parse for specific "functional" transform values!
|
| 67 |
+
self.tvf_resize_params, self.tvf_crop_params, self.tvf_normalize_params = [], [], []
|
| 68 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 69 |
+
|
| 70 |
+
for idx in range(len(input_sizes)):
|
| 71 |
+
transform = timm.data.create_transform(
|
| 72 |
+
input_size=self.input_sizes[idx],
|
| 73 |
+
interpolation=self.interpolations[idx],
|
| 74 |
+
mean=self.means[idx],
|
| 75 |
+
std=self.stds[idx],
|
| 76 |
+
crop_pct=1.0, # Set to 1.0 to ignore cropping (initial Resize sets `input_size`)
|
| 77 |
+
crop_mode="center", # Default crop mode -- no-op when `crop_pct == 1.0`
|
| 78 |
+
is_training=False, # No image augmentations when loading the transform!
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# [Validation] Ensure appropriate transform structure, expected sizes
|
| 82 |
+
if not (
|
| 83 |
+
isinstance(transform, Compose)
|
| 84 |
+
and (len(transform.transforms) == 4)
|
| 85 |
+
and isinstance(transform.transforms[0], Resize)
|
| 86 |
+
and isinstance(transform.transforms[1], CenterCrop)
|
| 87 |
+
and isinstance(transform.transforms[2], ToTensor)
|
| 88 |
+
and isinstance(transform.transforms[3], Normalize)
|
| 89 |
+
and (transform.transforms[0].size == self.input_sizes[idx][-1])
|
| 90 |
+
and (transform.transforms[1].size == self.input_sizes[idx][-2:])
|
| 91 |
+
):
|
| 92 |
+
raise ValueError(f"Unexpected TIMM image transformation structure/sizes: `{transform}`")
|
| 93 |
+
|
| 94 |
+
# HF Image Processors *must* be JSON-serializable; as such, cannot have torchvision. as an attribute.
|
| 95 |
+
# => Instead, we're going to parse the transform and call "torchvision.transforms.functional" (`tvf`)
|
| 96 |
+
resize_t, crop_t, norm_t = transform.transforms[0], transform.transforms[1], transform.transforms[3]
|
| 97 |
+
self.tvf_resize_params.append(
|
| 98 |
+
{
|
| 99 |
+
"size": resize_t.size,
|
| 100 |
+
"interpolation": TVF.pil_modes_mapping[resize_t.interpolation],
|
| 101 |
+
"max_size": None,
|
| 102 |
+
"antialias": True,
|
| 103 |
+
}
|
| 104 |
+
)
|
| 105 |
+
self.tvf_crop_params.append({"output_size": crop_t.size})
|
| 106 |
+
self.tvf_normalize_params.append(
|
| 107 |
+
{
|
| 108 |
+
"mean": norm_t.mean.float().numpy().tolist(),
|
| 109 |
+
"std": norm_t.std.float().numpy().tolist(),
|
| 110 |
+
"inplace": False,
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 114 |
+
|
| 115 |
+
# Handle Prismatic `image_resize_strategy`
|
| 116 |
+
if self.image_resize_strategy == "resize-naive":
|
| 117 |
+
self.tvf_resize_params[idx]["size"] = (resize_t.size, resize_t.size)
|
| 118 |
+
elif self.image_resize_strategy == "letterbox":
|
| 119 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = True, tuple([int(x * 255) for x in self.means[idx]])
|
| 120 |
+
elif self.image_resize_strategy == "resize-crop":
|
| 121 |
+
pass
|
| 122 |
+
else:
|
| 123 |
+
raise ValueError(f"Image resize strategy `{self.image_resize_strategy}` is not supported!")
|
| 124 |
+
|
| 125 |
+
# Dispatch **kwargs to super()
|
| 126 |
+
super().__init__(**kwargs)
|
| 127 |
+
|
| 128 |
+
def apply_transform(self, img: Image.Image) -> torch.Tensor:
|
| 129 |
+
"""Apply `functional` variant of TIMM's Transform = Compose([Resize -> CenterCrop -> ToTensor -> Normalize])"""
|
| 130 |
+
if self.tvf_do_letterbox:
|
| 131 |
+
img = letterbox_pad_transform(img, self.tvf_letterbox_fill)
|
| 132 |
+
|
| 133 |
+
# [Contract] Fused Backbones expect "channel-stacked" inputs; we'll unpack on the model side!
|
| 134 |
+
imgs_t = []
|
| 135 |
+
for idx in range(len(self.input_sizes)):
|
| 136 |
+
img_idx = TVF.resize(img, **self.tvf_resize_params[idx])
|
| 137 |
+
img_idx = TVF.center_crop(img_idx, **self.tvf_crop_params[idx])
|
| 138 |
+
img_idx_t = TVF.to_tensor(img_idx)
|
| 139 |
+
img_idx_t = TVF.normalize(img_idx_t, **self.tvf_normalize_params[idx])
|
| 140 |
+
imgs_t.append(img_idx_t)
|
| 141 |
+
|
| 142 |
+
# [Contract] `imgs_t` is a list of Tensors of shape [3, input_size, input_size]; stack along dim = 0
|
| 143 |
+
img_t = torch.vstack(imgs_t)
|
| 144 |
+
|
| 145 |
+
return img_t
|
| 146 |
+
|
| 147 |
+
def preprocess(
|
| 148 |
+
self,
|
| 149 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 150 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 151 |
+
**_: str,
|
| 152 |
+
) -> BatchFeature:
|
| 153 |
+
"""
|
| 154 |
+
Preprocess an image (or batch of images); note that unlike the `transformers :: BaseImageProcessor` we
|
| 155 |
+
explicitly only handle PIL.Image.Image instances for simplicity.
|
| 156 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 157 |
+
@param return_tensors: BatchFeature default Tensor format (e.g., "pt" for torch); if None, returns np.ndarray
|
| 158 |
+
@return: Instance of `transformers :: BatchFeature` with a single key "pixel_values"
|
| 159 |
+
"""
|
| 160 |
+
if not isinstance(images, list):
|
| 161 |
+
images = [images]
|
| 162 |
+
|
| 163 |
+
# Apply `self.img_transform` to each image (will return list of torch.Tensors); stack into "batched" Tensor
|
| 164 |
+
pixel_values = torch.stack([self.apply_transform(img.convert("RGB")) for img in images])
|
| 165 |
+
|
| 166 |
+
# Return BatchFeature =>> note that for compatibility, constructor expects Dict[str, np.ndarray], so we convert
|
| 167 |
+
return BatchFeature(data={"pixel_values": pixel_values.float().numpy()}, tensor_type=return_tensors)
|
| 168 |
+
|
| 169 |
+
def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> BatchFeature:
|
| 170 |
+
return self.preprocess(images, **kwargs)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# === PrismaticProcessor =>> Wraps both ImageProcessor and Tokenizer ===
|
| 174 |
+
# =>> https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/processing_llava.py
|
| 175 |
+
class PrismaticProcessor(ProcessorMixin):
|
| 176 |
+
attributes: ClassVar[List[str]] = ["image_processor", "tokenizer"]
|
| 177 |
+
image_processor_class: str = "AutoImageProcessor"
|
| 178 |
+
tokenizer_class: str = "AutoTokenizer"
|
| 179 |
+
|
| 180 |
+
def __init__(
|
| 181 |
+
self,
|
| 182 |
+
image_processor: Optional[ImageProcessingMixin] = None,
|
| 183 |
+
tokenizer: Optional[PreTrainedTokenizerBase] = None,
|
| 184 |
+
) -> None:
|
| 185 |
+
super().__init__(image_processor, tokenizer)
|
| 186 |
+
|
| 187 |
+
def __call__(
|
| 188 |
+
self,
|
| 189 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
|
| 190 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 191 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 192 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 193 |
+
max_length: Optional[int] = None,
|
| 194 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 195 |
+
) -> BatchFeature:
|
| 196 |
+
"""
|
| 197 |
+
Preprocess a given (batch) of text/images for a Prismatic VLM; forwards text to the underlying LLM's tokenizer,
|
| 198 |
+
forwards images to PrismaticImageProcessor.
|
| 199 |
+
@param text: The (batch) of text to encode; must be a string or list of strings.
|
| 200 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 201 |
+
@param padding: Sequence padding strategy (if multiple specified) in < True = "longest" | "max_length" | False >
|
| 202 |
+
@param truncation: Truncation strategy for the output sequences; requires `max_length` to be specified
|
| 203 |
+
@param max_length: Maximum length (in tokens) to truncate
|
| 204 |
+
@param return_tensors: Type of return tensors (usually "pt" or TensorType.PYTORCH)
|
| 205 |
+
@return: BatchFeature with keys for `input_ids`, `attention_mask` and `pixel_values`.
|
| 206 |
+
"""
|
| 207 |
+
pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"]
|
| 208 |
+
text_inputs = self.tokenizer(
|
| 209 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# [Validate] Need same number of images and text inputs!
|
| 213 |
+
if pixel_values.shape[0] != text_inputs.input_ids.shape[0]:
|
| 214 |
+
raise ValueError("Batch is malformed; expected same number of images and text inputs!")
|
| 215 |
+
|
| 216 |
+
return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
|
| 217 |
+
|
| 218 |
+
# === Tokenizer Dispatch Utilities =>> check `PreTrainedTokenizerBase` for documentation ===
|
| 219 |
+
def batch_decode(
|
| 220 |
+
self,
|
| 221 |
+
sequences: Union[List[int], List[List[int]], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 222 |
+
skip_special_tokens: bool = False,
|
| 223 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 224 |
+
**kwargs: str,
|
| 225 |
+
) -> List[str]:
|
| 226 |
+
return self.tokenizer.batch_decode(
|
| 227 |
+
sequences=sequences,
|
| 228 |
+
skip_special_tokens=skip_special_tokens,
|
| 229 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 230 |
+
**kwargs,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def decode(
|
| 234 |
+
self,
|
| 235 |
+
token_ids: Union[int, List[int], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 236 |
+
skip_special_tokens: bool = False,
|
| 237 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 238 |
+
**kwargs: str,
|
| 239 |
+
) -> str:
|
| 240 |
+
return self.tokenizer.decode(
|
| 241 |
+
token_ids=token_ids,
|
| 242 |
+
skip_special_tokens=skip_special_tokens,
|
| 243 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 244 |
+
**kwargs,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def model_input_names(self) -> List[str]:
|
| 249 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 250 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 251 |
+
|
| 252 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PrismaticProcessor"
|
| 6 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<PAD>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--55000_chkpt/tokenizer_config.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"0": {
|
| 6 |
+
"content": "<unk>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"1": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"2": {
|
| 22 |
+
"content": "</s>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"32000": {
|
| 30 |
+
"content": "<PAD>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
"auto_map": {
|
| 39 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 40 |
+
},
|
| 41 |
+
"bos_token": "<s>",
|
| 42 |
+
"clean_up_tokenization_spaces": false,
|
| 43 |
+
"eos_token": "</s>",
|
| 44 |
+
"legacy": false,
|
| 45 |
+
"model_max_length": 2048,
|
| 46 |
+
"pad_token": "<PAD>",
|
| 47 |
+
"padding_side": "right",
|
| 48 |
+
"processor_class": "PrismaticProcessor",
|
| 49 |
+
"sp_model_kwargs": {},
|
| 50 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 51 |
+
"unk_token": "<unk>",
|
| 52 |
+
"use_default_system_prompt": false
|
| 53 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<PAD>": 32000
|
| 3 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/lora_adapter/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: pretrained_models/configs-openvla-7b/config.json
|
| 3 |
+
library_name: peft
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Model Card for Model ID
|
| 7 |
+
|
| 8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## Model Details
|
| 13 |
+
|
| 14 |
+
### Model Description
|
| 15 |
+
|
| 16 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
- **Developed by:** [More Information Needed]
|
| 21 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 22 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 23 |
+
- **Model type:** [More Information Needed]
|
| 24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 25 |
+
- **License:** [More Information Needed]
|
| 26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 27 |
+
|
| 28 |
+
### Model Sources [optional]
|
| 29 |
+
|
| 30 |
+
<!-- Provide the basic links for the model. -->
|
| 31 |
+
|
| 32 |
+
- **Repository:** [More Information Needed]
|
| 33 |
+
- **Paper [optional]:** [More Information Needed]
|
| 34 |
+
- **Demo [optional]:** [More Information Needed]
|
| 35 |
+
|
| 36 |
+
## Uses
|
| 37 |
+
|
| 38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 39 |
+
|
| 40 |
+
### Direct Use
|
| 41 |
+
|
| 42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 43 |
+
|
| 44 |
+
[More Information Needed]
|
| 45 |
+
|
| 46 |
+
### Downstream Use [optional]
|
| 47 |
+
|
| 48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 49 |
+
|
| 50 |
+
[More Information Needed]
|
| 51 |
+
|
| 52 |
+
### Out-of-Scope Use
|
| 53 |
+
|
| 54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 55 |
+
|
| 56 |
+
[More Information Needed]
|
| 57 |
+
|
| 58 |
+
## Bias, Risks, and Limitations
|
| 59 |
+
|
| 60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 61 |
+
|
| 62 |
+
[More Information Needed]
|
| 63 |
+
|
| 64 |
+
### Recommendations
|
| 65 |
+
|
| 66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 67 |
+
|
| 68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 69 |
+
|
| 70 |
+
## How to Get Started with the Model
|
| 71 |
+
|
| 72 |
+
Use the code below to get started with the model.
|
| 73 |
+
|
| 74 |
+
[More Information Needed]
|
| 75 |
+
|
| 76 |
+
## Training Details
|
| 77 |
+
|
| 78 |
+
### Training Data
|
| 79 |
+
|
| 80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 81 |
+
|
| 82 |
+
[More Information Needed]
|
| 83 |
+
|
| 84 |
+
### Training Procedure
|
| 85 |
+
|
| 86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 87 |
+
|
| 88 |
+
#### Preprocessing [optional]
|
| 89 |
+
|
| 90 |
+
[More Information Needed]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
#### Training Hyperparameters
|
| 94 |
+
|
| 95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 96 |
+
|
| 97 |
+
#### Speeds, Sizes, Times [optional]
|
| 98 |
+
|
| 99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 100 |
+
|
| 101 |
+
[More Information Needed]
|
| 102 |
+
|
| 103 |
+
## Evaluation
|
| 104 |
+
|
| 105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 106 |
+
|
| 107 |
+
### Testing Data, Factors & Metrics
|
| 108 |
+
|
| 109 |
+
#### Testing Data
|
| 110 |
+
|
| 111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 112 |
+
|
| 113 |
+
[More Information Needed]
|
| 114 |
+
|
| 115 |
+
#### Factors
|
| 116 |
+
|
| 117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 118 |
+
|
| 119 |
+
[More Information Needed]
|
| 120 |
+
|
| 121 |
+
#### Metrics
|
| 122 |
+
|
| 123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 124 |
+
|
| 125 |
+
[More Information Needed]
|
| 126 |
+
|
| 127 |
+
### Results
|
| 128 |
+
|
| 129 |
+
[More Information Needed]
|
| 130 |
+
|
| 131 |
+
#### Summary
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
## Model Examination [optional]
|
| 136 |
+
|
| 137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 138 |
+
|
| 139 |
+
[More Information Needed]
|
| 140 |
+
|
| 141 |
+
## Environmental Impact
|
| 142 |
+
|
| 143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 144 |
+
|
| 145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 146 |
+
|
| 147 |
+
- **Hardware Type:** [More Information Needed]
|
| 148 |
+
- **Hours used:** [More Information Needed]
|
| 149 |
+
- **Cloud Provider:** [More Information Needed]
|
| 150 |
+
- **Compute Region:** [More Information Needed]
|
| 151 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 152 |
+
|
| 153 |
+
## Technical Specifications [optional]
|
| 154 |
+
|
| 155 |
+
### Model Architecture and Objective
|
| 156 |
+
|
| 157 |
+
[More Information Needed]
|
| 158 |
+
|
| 159 |
+
### Compute Infrastructure
|
| 160 |
+
|
| 161 |
+
[More Information Needed]
|
| 162 |
+
|
| 163 |
+
#### Hardware
|
| 164 |
+
|
| 165 |
+
[More Information Needed]
|
| 166 |
+
|
| 167 |
+
#### Software
|
| 168 |
+
|
| 169 |
+
[More Information Needed]
|
| 170 |
+
|
| 171 |
+
## Citation [optional]
|
| 172 |
+
|
| 173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 174 |
+
|
| 175 |
+
**BibTeX:**
|
| 176 |
+
|
| 177 |
+
[More Information Needed]
|
| 178 |
+
|
| 179 |
+
**APA:**
|
| 180 |
+
|
| 181 |
+
[More Information Needed]
|
| 182 |
+
|
| 183 |
+
## Glossary [optional]
|
| 184 |
+
|
| 185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 186 |
+
|
| 187 |
+
[More Information Needed]
|
| 188 |
+
|
| 189 |
+
## More Information [optional]
|
| 190 |
+
|
| 191 |
+
[More Information Needed]
|
| 192 |
+
|
| 193 |
+
## Model Card Authors [optional]
|
| 194 |
+
|
| 195 |
+
[More Information Needed]
|
| 196 |
+
|
| 197 |
+
## Model Card Contact
|
| 198 |
+
|
| 199 |
+
[More Information Needed]
|
| 200 |
+
### Framework versions
|
| 201 |
+
|
| 202 |
+
- PEFT 0.11.1
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/lora_adapter/adapter_config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alpha_pattern": {},
|
| 3 |
+
"auto_mapping": {
|
| 4 |
+
"base_model_class": "OpenVLAForActionPrediction",
|
| 5 |
+
"parent_library": "prismatic.extern.hf.modeling_prismatic"
|
| 6 |
+
},
|
| 7 |
+
"base_model_name_or_path": "pretrained_models/configs-openvla-7b/config.json",
|
| 8 |
+
"bias": "none",
|
| 9 |
+
"fan_in_fan_out": false,
|
| 10 |
+
"inference_mode": true,
|
| 11 |
+
"init_lora_weights": "gaussian",
|
| 12 |
+
"layer_replication": null,
|
| 13 |
+
"layers_pattern": null,
|
| 14 |
+
"layers_to_transform": null,
|
| 15 |
+
"loftq_config": {},
|
| 16 |
+
"lora_alpha": 64,
|
| 17 |
+
"lora_dropout": 0.0,
|
| 18 |
+
"megatron_config": null,
|
| 19 |
+
"megatron_core": "megatron.core",
|
| 20 |
+
"modules_to_save": null,
|
| 21 |
+
"peft_type": "LORA",
|
| 22 |
+
"r": 32,
|
| 23 |
+
"rank_pattern": {},
|
| 24 |
+
"revision": null,
|
| 25 |
+
"target_modules": [
|
| 26 |
+
"q_proj",
|
| 27 |
+
"o_proj",
|
| 28 |
+
"proj",
|
| 29 |
+
"qkv",
|
| 30 |
+
"kv",
|
| 31 |
+
"q",
|
| 32 |
+
"v_proj",
|
| 33 |
+
"fc3",
|
| 34 |
+
"down_proj",
|
| 35 |
+
"gate_proj",
|
| 36 |
+
"up_proj",
|
| 37 |
+
"lm_head",
|
| 38 |
+
"k_proj",
|
| 39 |
+
"fc2",
|
| 40 |
+
"fc1"
|
| 41 |
+
],
|
| 42 |
+
"task_type": null,
|
| 43 |
+
"use_dora": false,
|
| 44 |
+
"use_rslora": false
|
| 45 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/preprocessor_config.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "processing_prismatic.PrismaticImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 5 |
+
},
|
| 6 |
+
"image_processor_type": "PrismaticImageProcessor",
|
| 7 |
+
"image_resize_strategy": "resize-naive",
|
| 8 |
+
"input_sizes": [
|
| 9 |
+
[
|
| 10 |
+
3,
|
| 11 |
+
224,
|
| 12 |
+
224
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
3,
|
| 16 |
+
224,
|
| 17 |
+
224
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"interpolations": [
|
| 21 |
+
"bicubic",
|
| 22 |
+
"bicubic"
|
| 23 |
+
],
|
| 24 |
+
"means": [
|
| 25 |
+
[
|
| 26 |
+
0.485,
|
| 27 |
+
0.456,
|
| 28 |
+
0.406
|
| 29 |
+
],
|
| 30 |
+
[
|
| 31 |
+
0.5,
|
| 32 |
+
0.5,
|
| 33 |
+
0.5
|
| 34 |
+
]
|
| 35 |
+
],
|
| 36 |
+
"processor_class": "PrismaticProcessor",
|
| 37 |
+
"stds": [
|
| 38 |
+
[
|
| 39 |
+
0.229,
|
| 40 |
+
0.224,
|
| 41 |
+
0.225
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
0.5,
|
| 45 |
+
0.5,
|
| 46 |
+
0.5
|
| 47 |
+
]
|
| 48 |
+
],
|
| 49 |
+
"tvf_crop_params": [
|
| 50 |
+
{
|
| 51 |
+
"output_size": [
|
| 52 |
+
224,
|
| 53 |
+
224
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"output_size": [
|
| 58 |
+
224,
|
| 59 |
+
224
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"tvf_do_letterbox": false,
|
| 64 |
+
"tvf_letterbox_fill": null,
|
| 65 |
+
"tvf_normalize_params": [
|
| 66 |
+
{
|
| 67 |
+
"inplace": false,
|
| 68 |
+
"mean": [
|
| 69 |
+
0.484375,
|
| 70 |
+
0.455078125,
|
| 71 |
+
0.40625
|
| 72 |
+
],
|
| 73 |
+
"std": [
|
| 74 |
+
0.228515625,
|
| 75 |
+
0.2236328125,
|
| 76 |
+
0.224609375
|
| 77 |
+
]
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"inplace": false,
|
| 81 |
+
"mean": [
|
| 82 |
+
0.5,
|
| 83 |
+
0.5,
|
| 84 |
+
0.5
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.5,
|
| 88 |
+
0.5,
|
| 89 |
+
0.5
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"tvf_resize_params": [
|
| 94 |
+
{
|
| 95 |
+
"antialias": true,
|
| 96 |
+
"interpolation": 3,
|
| 97 |
+
"max_size": null,
|
| 98 |
+
"size": [
|
| 99 |
+
224,
|
| 100 |
+
224
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"antialias": true,
|
| 105 |
+
"interpolation": 3,
|
| 106 |
+
"max_size": null,
|
| 107 |
+
"size": [
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"use_fused_vision_backbone": true
|
| 114 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/processing_prismatic.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
processing_prismatic.py
|
| 3 |
+
|
| 4 |
+
HuggingFace-style preprocessor definitions for Prismatic VLMs, inheriting from `ProcessorMixin`. Default configuration
|
| 5 |
+
specifies `siglip-224px+7b`.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Any, ClassVar, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import timm.data
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms.functional as TVF
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor
|
| 15 |
+
from transformers import PreTrainedTokenizerBase
|
| 16 |
+
from transformers.image_processing_utils import BatchFeature, ImageProcessingMixin
|
| 17 |
+
from transformers.processing_utils import ProcessorMixin
|
| 18 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| 19 |
+
from transformers.utils import TensorType
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# === Image Processing ===
|
| 23 |
+
def letterbox_pad_transform(image: Image.Image, padding_fill_value: Tuple[int, int, int]) -> Image.Image:
|
| 24 |
+
"""Given a PIL.Image, pad to square by adding a symmetric border around the height/width."""
|
| 25 |
+
(w, h), max_wh = image.size, max(image.size)
|
| 26 |
+
horizontal_pad, vertical_pad = int((max_wh - w) / 2), int((max_wh - h) / 2)
|
| 27 |
+
padding = (horizontal_pad, vertical_pad, horizontal_pad, vertical_pad)
|
| 28 |
+
|
| 29 |
+
return TVF.pad(image, padding, fill=padding_fill_value, padding_mode="constant")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PrismaticImageProcessor(ImageProcessingMixin):
|
| 33 |
+
model_input_names: ClassVar[List[str]] = ["pixel_values"]
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
use_fused_vision_backbone: bool = False,
|
| 38 |
+
image_resize_strategy: str = "letterbox",
|
| 39 |
+
input_sizes: Optional[List[Tuple[int, int, int]]] = None,
|
| 40 |
+
interpolations: Optional[List[str]] = None,
|
| 41 |
+
means: Optional[List[Tuple[float, float, float]]] = None,
|
| 42 |
+
stds: Optional[List[Tuple[float, float, float]]] = None,
|
| 43 |
+
**kwargs: str,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""
|
| 46 |
+
Initialize a PrismaticImageProcessor as a wrapper around a torchvision transform; this transform will be
|
| 47 |
+
created by TIMM, and edited to follow our custom `image_resize_strategy` logic.
|
| 48 |
+
@param use_fused_vision_backbone: Boolean indicating single or fused (dual) vision backbone
|
| 49 |
+
@param image_resize_strategy: Prismatic image resize strategy in < resize-naive | resize-crop | letterbox >
|
| 50 |
+
@param input_size: [TIMM :: `data_cfg`] Input image size as tuple (channels, width, height)
|
| 51 |
+
@param interpolation: [TIMM :: `data_cfg`] Interpolation as string (default: "bicubic")
|
| 52 |
+
@param mean: [TIMM :: `data_cfg`] Normalization mean as float tuple (or two-tuple if `fused_backbone`)
|
| 53 |
+
@param std: [TIMM :: `data_cfg`] Normalization std as float tuple (or two-tuple if `fused_backbone`)
|
| 54 |
+
"""
|
| 55 |
+
self.use_fused_vision_backbone = use_fused_vision_backbone
|
| 56 |
+
self.image_resize_strategy = image_resize_strategy
|
| 57 |
+
|
| 58 |
+
# Handle `None` default values
|
| 59 |
+
input_sizes = [(3, 224, 224)] if input_sizes is None else input_sizes
|
| 60 |
+
means = [(0.5, 0.5, 0.5)] if means is None else means
|
| 61 |
+
stds = [(0.5, 0.5, 0.5)] if stds is None else stds
|
| 62 |
+
|
| 63 |
+
# TIMM `data_cfg` Parameters
|
| 64 |
+
self.input_sizes, self.interpolations, self.means, self.stds = input_sizes, interpolations, means, stds
|
| 65 |
+
|
| 66 |
+
# Grab torchvision transforms via TIMM =>> need to parse for specific "functional" transform values!
|
| 67 |
+
self.tvf_resize_params, self.tvf_crop_params, self.tvf_normalize_params = [], [], []
|
| 68 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 69 |
+
|
| 70 |
+
for idx in range(len(input_sizes)):
|
| 71 |
+
transform = timm.data.create_transform(
|
| 72 |
+
input_size=self.input_sizes[idx],
|
| 73 |
+
interpolation=self.interpolations[idx],
|
| 74 |
+
mean=self.means[idx],
|
| 75 |
+
std=self.stds[idx],
|
| 76 |
+
crop_pct=1.0, # Set to 1.0 to ignore cropping (initial Resize sets `input_size`)
|
| 77 |
+
crop_mode="center", # Default crop mode -- no-op when `crop_pct == 1.0`
|
| 78 |
+
is_training=False, # No image augmentations when loading the transform!
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# [Validation] Ensure appropriate transform structure, expected sizes
|
| 82 |
+
if not (
|
| 83 |
+
isinstance(transform, Compose)
|
| 84 |
+
and (len(transform.transforms) == 4)
|
| 85 |
+
and isinstance(transform.transforms[0], Resize)
|
| 86 |
+
and isinstance(transform.transforms[1], CenterCrop)
|
| 87 |
+
and isinstance(transform.transforms[2], ToTensor)
|
| 88 |
+
and isinstance(transform.transforms[3], Normalize)
|
| 89 |
+
and (transform.transforms[0].size == self.input_sizes[idx][-1])
|
| 90 |
+
and (transform.transforms[1].size == self.input_sizes[idx][-2:])
|
| 91 |
+
):
|
| 92 |
+
raise ValueError(f"Unexpected TIMM image transformation structure/sizes: `{transform}`")
|
| 93 |
+
|
| 94 |
+
# HF Image Processors *must* be JSON-serializable; as such, cannot have torchvision. as an attribute.
|
| 95 |
+
# => Instead, we're going to parse the transform and call "torchvision.transforms.functional" (`tvf`)
|
| 96 |
+
resize_t, crop_t, norm_t = transform.transforms[0], transform.transforms[1], transform.transforms[3]
|
| 97 |
+
self.tvf_resize_params.append(
|
| 98 |
+
{
|
| 99 |
+
"size": resize_t.size,
|
| 100 |
+
"interpolation": TVF.pil_modes_mapping[resize_t.interpolation],
|
| 101 |
+
"max_size": None,
|
| 102 |
+
"antialias": True,
|
| 103 |
+
}
|
| 104 |
+
)
|
| 105 |
+
self.tvf_crop_params.append({"output_size": crop_t.size})
|
| 106 |
+
self.tvf_normalize_params.append(
|
| 107 |
+
{
|
| 108 |
+
"mean": norm_t.mean.float().numpy().tolist(),
|
| 109 |
+
"std": norm_t.std.float().numpy().tolist(),
|
| 110 |
+
"inplace": False,
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 114 |
+
|
| 115 |
+
# Handle Prismatic `image_resize_strategy`
|
| 116 |
+
if self.image_resize_strategy == "resize-naive":
|
| 117 |
+
self.tvf_resize_params[idx]["size"] = (resize_t.size, resize_t.size)
|
| 118 |
+
elif self.image_resize_strategy == "letterbox":
|
| 119 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = True, tuple([int(x * 255) for x in self.means[idx]])
|
| 120 |
+
elif self.image_resize_strategy == "resize-crop":
|
| 121 |
+
pass
|
| 122 |
+
else:
|
| 123 |
+
raise ValueError(f"Image resize strategy `{self.image_resize_strategy}` is not supported!")
|
| 124 |
+
|
| 125 |
+
# Dispatch **kwargs to super()
|
| 126 |
+
super().__init__(**kwargs)
|
| 127 |
+
|
| 128 |
+
def apply_transform(self, img: Image.Image) -> torch.Tensor:
|
| 129 |
+
"""Apply `functional` variant of TIMM's Transform = Compose([Resize -> CenterCrop -> ToTensor -> Normalize])"""
|
| 130 |
+
if self.tvf_do_letterbox:
|
| 131 |
+
img = letterbox_pad_transform(img, self.tvf_letterbox_fill)
|
| 132 |
+
|
| 133 |
+
# [Contract] Fused Backbones expect "channel-stacked" inputs; we'll unpack on the model side!
|
| 134 |
+
imgs_t = []
|
| 135 |
+
for idx in range(len(self.input_sizes)):
|
| 136 |
+
img_idx = TVF.resize(img, **self.tvf_resize_params[idx])
|
| 137 |
+
img_idx = TVF.center_crop(img_idx, **self.tvf_crop_params[idx])
|
| 138 |
+
img_idx_t = TVF.to_tensor(img_idx)
|
| 139 |
+
img_idx_t = TVF.normalize(img_idx_t, **self.tvf_normalize_params[idx])
|
| 140 |
+
imgs_t.append(img_idx_t)
|
| 141 |
+
|
| 142 |
+
# [Contract] `imgs_t` is a list of Tensors of shape [3, input_size, input_size]; stack along dim = 0
|
| 143 |
+
img_t = torch.vstack(imgs_t)
|
| 144 |
+
|
| 145 |
+
return img_t
|
| 146 |
+
|
| 147 |
+
def preprocess(
|
| 148 |
+
self,
|
| 149 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 150 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 151 |
+
**_: str,
|
| 152 |
+
) -> BatchFeature:
|
| 153 |
+
"""
|
| 154 |
+
Preprocess an image (or batch of images); note that unlike the `transformers :: BaseImageProcessor` we
|
| 155 |
+
explicitly only handle PIL.Image.Image instances for simplicity.
|
| 156 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 157 |
+
@param return_tensors: BatchFeature default Tensor format (e.g., "pt" for torch); if None, returns np.ndarray
|
| 158 |
+
@return: Instance of `transformers :: BatchFeature` with a single key "pixel_values"
|
| 159 |
+
"""
|
| 160 |
+
if not isinstance(images, list):
|
| 161 |
+
images = [images]
|
| 162 |
+
|
| 163 |
+
# Apply `self.img_transform` to each image (will return list of torch.Tensors); stack into "batched" Tensor
|
| 164 |
+
pixel_values = torch.stack([self.apply_transform(img.convert("RGB")) for img in images])
|
| 165 |
+
|
| 166 |
+
# Return BatchFeature =>> note that for compatibility, constructor expects Dict[str, np.ndarray], so we convert
|
| 167 |
+
return BatchFeature(data={"pixel_values": pixel_values.float().numpy()}, tensor_type=return_tensors)
|
| 168 |
+
|
| 169 |
+
def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> BatchFeature:
|
| 170 |
+
return self.preprocess(images, **kwargs)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# === PrismaticProcessor =>> Wraps both ImageProcessor and Tokenizer ===
|
| 174 |
+
# =>> https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/processing_llava.py
|
| 175 |
+
class PrismaticProcessor(ProcessorMixin):
|
| 176 |
+
attributes: ClassVar[List[str]] = ["image_processor", "tokenizer"]
|
| 177 |
+
image_processor_class: str = "AutoImageProcessor"
|
| 178 |
+
tokenizer_class: str = "AutoTokenizer"
|
| 179 |
+
|
| 180 |
+
def __init__(
|
| 181 |
+
self,
|
| 182 |
+
image_processor: Optional[ImageProcessingMixin] = None,
|
| 183 |
+
tokenizer: Optional[PreTrainedTokenizerBase] = None,
|
| 184 |
+
) -> None:
|
| 185 |
+
super().__init__(image_processor, tokenizer)
|
| 186 |
+
|
| 187 |
+
def __call__(
|
| 188 |
+
self,
|
| 189 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
|
| 190 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 191 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 192 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 193 |
+
max_length: Optional[int] = None,
|
| 194 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 195 |
+
) -> BatchFeature:
|
| 196 |
+
"""
|
| 197 |
+
Preprocess a given (batch) of text/images for a Prismatic VLM; forwards text to the underlying LLM's tokenizer,
|
| 198 |
+
forwards images to PrismaticImageProcessor.
|
| 199 |
+
@param text: The (batch) of text to encode; must be a string or list of strings.
|
| 200 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 201 |
+
@param padding: Sequence padding strategy (if multiple specified) in < True = "longest" | "max_length" | False >
|
| 202 |
+
@param truncation: Truncation strategy for the output sequences; requires `max_length` to be specified
|
| 203 |
+
@param max_length: Maximum length (in tokens) to truncate
|
| 204 |
+
@param return_tensors: Type of return tensors (usually "pt" or TensorType.PYTORCH)
|
| 205 |
+
@return: BatchFeature with keys for `input_ids`, `attention_mask` and `pixel_values`.
|
| 206 |
+
"""
|
| 207 |
+
pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"]
|
| 208 |
+
text_inputs = self.tokenizer(
|
| 209 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# [Validate] Need same number of images and text inputs!
|
| 213 |
+
if pixel_values.shape[0] != text_inputs.input_ids.shape[0]:
|
| 214 |
+
raise ValueError("Batch is malformed; expected same number of images and text inputs!")
|
| 215 |
+
|
| 216 |
+
return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
|
| 217 |
+
|
| 218 |
+
# === Tokenizer Dispatch Utilities =>> check `PreTrainedTokenizerBase` for documentation ===
|
| 219 |
+
def batch_decode(
|
| 220 |
+
self,
|
| 221 |
+
sequences: Union[List[int], List[List[int]], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 222 |
+
skip_special_tokens: bool = False,
|
| 223 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 224 |
+
**kwargs: str,
|
| 225 |
+
) -> List[str]:
|
| 226 |
+
return self.tokenizer.batch_decode(
|
| 227 |
+
sequences=sequences,
|
| 228 |
+
skip_special_tokens=skip_special_tokens,
|
| 229 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 230 |
+
**kwargs,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def decode(
|
| 234 |
+
self,
|
| 235 |
+
token_ids: Union[int, List[int], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 236 |
+
skip_special_tokens: bool = False,
|
| 237 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 238 |
+
**kwargs: str,
|
| 239 |
+
) -> str:
|
| 240 |
+
return self.tokenizer.decode(
|
| 241 |
+
token_ids=token_ids,
|
| 242 |
+
skip_special_tokens=skip_special_tokens,
|
| 243 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 244 |
+
**kwargs,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def model_input_names(self) -> List[str]:
|
| 249 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 250 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 251 |
+
|
| 252 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PrismaticProcessor"
|
| 6 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<PAD>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.0002+SPD+wd-2.0+x-action_queries+lora-r32+dropout-0.0+layerwise_decay--image_aug--OPENVLA-OFT--BRIDGE--20251109-130310--65000_chkpt/tokenizer_config.json
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"0": {
|
| 6 |
+
"content": "<unk>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"1": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"2": {
|
| 22 |
+
"content": "</s>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": false,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"32000": {
|
| 30 |
+
"content": "<PAD>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
}
|
| 37 |
+
},
|
| 38 |
+
"auto_map": {
|
| 39 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 40 |
+
},
|
| 41 |
+
"bos_token": "<s>",
|
| 42 |
+
"clean_up_tokenization_spaces": false,
|
| 43 |
+
"eos_token": "</s>",
|
| 44 |
+
"legacy": false,
|
| 45 |
+
"model_max_length": 2048,
|
| 46 |
+
"pad_token": "<PAD>",
|
| 47 |
+
"padding_side": "right",
|
| 48 |
+
"processor_class": "PrismaticProcessor",
|
| 49 |
+
"sp_model_kwargs": {},
|
| 50 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 51 |
+
"unk_token": "<unk>",
|
| 52 |
+
"use_default_system_prompt": false
|
| 53 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-0.001+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251107-204623/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/added_tokens.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"<PAD>": 32000
|
| 3 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/dataset_statistics.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bridge_orig": {
|
| 3 |
+
"action": {
|
| 4 |
+
"mean": [
|
| 5 |
+
0.00023341900669038296,
|
| 6 |
+
0.00013004800712224096,
|
| 7 |
+
-0.00012762377446051687,
|
| 8 |
+
-0.00015565514331683517,
|
| 9 |
+
-0.0004039339255541563,
|
| 10 |
+
0.00023557626991532743,
|
| 11 |
+
0.5764579176902771
|
| 12 |
+
],
|
| 13 |
+
"std": [
|
| 14 |
+
0.009765932336449623,
|
| 15 |
+
0.013689127750694752,
|
| 16 |
+
0.012667366303503513,
|
| 17 |
+
0.028534261509776115,
|
| 18 |
+
0.030637970194220543,
|
| 19 |
+
0.07691484689712524,
|
| 20 |
+
0.49736908078193665
|
| 21 |
+
],
|
| 22 |
+
"max": [
|
| 23 |
+
0.41691166162490845,
|
| 24 |
+
0.25864794850349426,
|
| 25 |
+
0.21218234300613403,
|
| 26 |
+
3.122201919555664,
|
| 27 |
+
1.8618112802505493,
|
| 28 |
+
6.280478477478027,
|
| 29 |
+
1.0
|
| 30 |
+
],
|
| 31 |
+
"min": [
|
| 32 |
+
-0.4007510244846344,
|
| 33 |
+
-0.13874775171279907,
|
| 34 |
+
-0.22553899884223938,
|
| 35 |
+
-3.2010786533355713,
|
| 36 |
+
-1.8618112802505493,
|
| 37 |
+
-6.279075622558594,
|
| 38 |
+
0.0
|
| 39 |
+
],
|
| 40 |
+
"q01": [
|
| 41 |
+
-0.02872725307941437,
|
| 42 |
+
-0.04170349963009357,
|
| 43 |
+
-0.026093858778476715,
|
| 44 |
+
-0.08092105075716972,
|
| 45 |
+
-0.09288699507713317,
|
| 46 |
+
-0.20718276381492615,
|
| 47 |
+
0.0
|
| 48 |
+
],
|
| 49 |
+
"q99": [
|
| 50 |
+
0.028309678435325586,
|
| 51 |
+
0.040855254605412394,
|
| 52 |
+
0.040161586627364146,
|
| 53 |
+
0.08192047759890528,
|
| 54 |
+
0.07792850524187081,
|
| 55 |
+
0.20382574498653397,
|
| 56 |
+
1.0
|
| 57 |
+
],
|
| 58 |
+
"mask": [
|
| 59 |
+
true,
|
| 60 |
+
true,
|
| 61 |
+
true,
|
| 62 |
+
true,
|
| 63 |
+
true,
|
| 64 |
+
true,
|
| 65 |
+
false
|
| 66 |
+
]
|
| 67 |
+
},
|
| 68 |
+
"proprio": {
|
| 69 |
+
"mean": [
|
| 70 |
+
0.3094063401222229,
|
| 71 |
+
0.030576054006814957,
|
| 72 |
+
0.06454064697027206,
|
| 73 |
+
0.006824111100286245,
|
| 74 |
+
-0.07762739807367325,
|
| 75 |
+
0.10757849365472794,
|
| 76 |
+
0.7084252834320068
|
| 77 |
+
],
|
| 78 |
+
"std": [
|
| 79 |
+
0.0605284720659256,
|
| 80 |
+
0.09188612550497055,
|
| 81 |
+
0.05159845948219299,
|
| 82 |
+
0.1318272054195404,
|
| 83 |
+
0.17031094431877136,
|
| 84 |
+
0.5767266154289246,
|
| 85 |
+
0.3519865572452545
|
| 86 |
+
],
|
| 87 |
+
"max": [
|
| 88 |
+
0.5862360596656799,
|
| 89 |
+
0.4034728705883026,
|
| 90 |
+
0.36494991183280945,
|
| 91 |
+
1.514088749885559,
|
| 92 |
+
1.570796251296997,
|
| 93 |
+
3.1415255069732666,
|
| 94 |
+
1.1154625415802002
|
| 95 |
+
],
|
| 96 |
+
"min": [
|
| 97 |
+
-0.04167502000927925,
|
| 98 |
+
-0.3945816159248352,
|
| 99 |
+
-0.15537554025650024,
|
| 100 |
+
-3.141592502593994,
|
| 101 |
+
-1.4992541074752808,
|
| 102 |
+
-3.14153790473938,
|
| 103 |
+
0.04637829214334488
|
| 104 |
+
],
|
| 105 |
+
"q01": [
|
| 106 |
+
0.17111587673425674,
|
| 107 |
+
-0.16998695254325866,
|
| 108 |
+
-0.05544630073010921,
|
| 109 |
+
-0.366876106262207,
|
| 110 |
+
-0.5443069756031036,
|
| 111 |
+
-1.3536006283760071,
|
| 112 |
+
0.052190229296684265
|
| 113 |
+
],
|
| 114 |
+
"q99": [
|
| 115 |
+
0.45320980012416834,
|
| 116 |
+
0.23518154799938193,
|
| 117 |
+
0.1951873075962065,
|
| 118 |
+
0.3806115746498103,
|
| 119 |
+
0.2789784955978382,
|
| 120 |
+
1.8410426235198971,
|
| 121 |
+
1.0105689764022827
|
| 122 |
+
]
|
| 123 |
+
},
|
| 124 |
+
"num_transitions": 2135463,
|
| 125 |
+
"num_trajectories": 60064
|
| 126 |
+
}
|
| 127 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/lora_adapter/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: pretrained_models/configs-openvla-7b/config.json
|
| 3 |
+
library_name: peft
|
| 4 |
+
---
|
| 5 |
+
|
| 6 |
+
# Model Card for Model ID
|
| 7 |
+
|
| 8 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
## Model Details
|
| 13 |
+
|
| 14 |
+
### Model Description
|
| 15 |
+
|
| 16 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
- **Developed by:** [More Information Needed]
|
| 21 |
+
- **Funded by [optional]:** [More Information Needed]
|
| 22 |
+
- **Shared by [optional]:** [More Information Needed]
|
| 23 |
+
- **Model type:** [More Information Needed]
|
| 24 |
+
- **Language(s) (NLP):** [More Information Needed]
|
| 25 |
+
- **License:** [More Information Needed]
|
| 26 |
+
- **Finetuned from model [optional]:** [More Information Needed]
|
| 27 |
+
|
| 28 |
+
### Model Sources [optional]
|
| 29 |
+
|
| 30 |
+
<!-- Provide the basic links for the model. -->
|
| 31 |
+
|
| 32 |
+
- **Repository:** [More Information Needed]
|
| 33 |
+
- **Paper [optional]:** [More Information Needed]
|
| 34 |
+
- **Demo [optional]:** [More Information Needed]
|
| 35 |
+
|
| 36 |
+
## Uses
|
| 37 |
+
|
| 38 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 39 |
+
|
| 40 |
+
### Direct Use
|
| 41 |
+
|
| 42 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 43 |
+
|
| 44 |
+
[More Information Needed]
|
| 45 |
+
|
| 46 |
+
### Downstream Use [optional]
|
| 47 |
+
|
| 48 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 49 |
+
|
| 50 |
+
[More Information Needed]
|
| 51 |
+
|
| 52 |
+
### Out-of-Scope Use
|
| 53 |
+
|
| 54 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 55 |
+
|
| 56 |
+
[More Information Needed]
|
| 57 |
+
|
| 58 |
+
## Bias, Risks, and Limitations
|
| 59 |
+
|
| 60 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 61 |
+
|
| 62 |
+
[More Information Needed]
|
| 63 |
+
|
| 64 |
+
### Recommendations
|
| 65 |
+
|
| 66 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 67 |
+
|
| 68 |
+
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
| 69 |
+
|
| 70 |
+
## How to Get Started with the Model
|
| 71 |
+
|
| 72 |
+
Use the code below to get started with the model.
|
| 73 |
+
|
| 74 |
+
[More Information Needed]
|
| 75 |
+
|
| 76 |
+
## Training Details
|
| 77 |
+
|
| 78 |
+
### Training Data
|
| 79 |
+
|
| 80 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 81 |
+
|
| 82 |
+
[More Information Needed]
|
| 83 |
+
|
| 84 |
+
### Training Procedure
|
| 85 |
+
|
| 86 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 87 |
+
|
| 88 |
+
#### Preprocessing [optional]
|
| 89 |
+
|
| 90 |
+
[More Information Needed]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
#### Training Hyperparameters
|
| 94 |
+
|
| 95 |
+
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 96 |
+
|
| 97 |
+
#### Speeds, Sizes, Times [optional]
|
| 98 |
+
|
| 99 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 100 |
+
|
| 101 |
+
[More Information Needed]
|
| 102 |
+
|
| 103 |
+
## Evaluation
|
| 104 |
+
|
| 105 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 106 |
+
|
| 107 |
+
### Testing Data, Factors & Metrics
|
| 108 |
+
|
| 109 |
+
#### Testing Data
|
| 110 |
+
|
| 111 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 112 |
+
|
| 113 |
+
[More Information Needed]
|
| 114 |
+
|
| 115 |
+
#### Factors
|
| 116 |
+
|
| 117 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 118 |
+
|
| 119 |
+
[More Information Needed]
|
| 120 |
+
|
| 121 |
+
#### Metrics
|
| 122 |
+
|
| 123 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 124 |
+
|
| 125 |
+
[More Information Needed]
|
| 126 |
+
|
| 127 |
+
### Results
|
| 128 |
+
|
| 129 |
+
[More Information Needed]
|
| 130 |
+
|
| 131 |
+
#### Summary
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
## Model Examination [optional]
|
| 136 |
+
|
| 137 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 138 |
+
|
| 139 |
+
[More Information Needed]
|
| 140 |
+
|
| 141 |
+
## Environmental Impact
|
| 142 |
+
|
| 143 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 144 |
+
|
| 145 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 146 |
+
|
| 147 |
+
- **Hardware Type:** [More Information Needed]
|
| 148 |
+
- **Hours used:** [More Information Needed]
|
| 149 |
+
- **Cloud Provider:** [More Information Needed]
|
| 150 |
+
- **Compute Region:** [More Information Needed]
|
| 151 |
+
- **Carbon Emitted:** [More Information Needed]
|
| 152 |
+
|
| 153 |
+
## Technical Specifications [optional]
|
| 154 |
+
|
| 155 |
+
### Model Architecture and Objective
|
| 156 |
+
|
| 157 |
+
[More Information Needed]
|
| 158 |
+
|
| 159 |
+
### Compute Infrastructure
|
| 160 |
+
|
| 161 |
+
[More Information Needed]
|
| 162 |
+
|
| 163 |
+
#### Hardware
|
| 164 |
+
|
| 165 |
+
[More Information Needed]
|
| 166 |
+
|
| 167 |
+
#### Software
|
| 168 |
+
|
| 169 |
+
[More Information Needed]
|
| 170 |
+
|
| 171 |
+
## Citation [optional]
|
| 172 |
+
|
| 173 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 174 |
+
|
| 175 |
+
**BibTeX:**
|
| 176 |
+
|
| 177 |
+
[More Information Needed]
|
| 178 |
+
|
| 179 |
+
**APA:**
|
| 180 |
+
|
| 181 |
+
[More Information Needed]
|
| 182 |
+
|
| 183 |
+
## Glossary [optional]
|
| 184 |
+
|
| 185 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 186 |
+
|
| 187 |
+
[More Information Needed]
|
| 188 |
+
|
| 189 |
+
## More Information [optional]
|
| 190 |
+
|
| 191 |
+
[More Information Needed]
|
| 192 |
+
|
| 193 |
+
## Model Card Authors [optional]
|
| 194 |
+
|
| 195 |
+
[More Information Needed]
|
| 196 |
+
|
| 197 |
+
## Model Card Contact
|
| 198 |
+
|
| 199 |
+
[More Information Needed]
|
| 200 |
+
### Framework versions
|
| 201 |
+
|
| 202 |
+
- PEFT 0.11.1
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/lora_adapter/adapter_config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"alpha_pattern": {},
|
| 3 |
+
"auto_mapping": {
|
| 4 |
+
"base_model_class": "OpenVLAForActionPrediction",
|
| 5 |
+
"parent_library": "prismatic.extern.hf.modeling_prismatic"
|
| 6 |
+
},
|
| 7 |
+
"base_model_name_or_path": "pretrained_models/configs-openvla-7b/config.json",
|
| 8 |
+
"bias": "none",
|
| 9 |
+
"fan_in_fan_out": false,
|
| 10 |
+
"inference_mode": true,
|
| 11 |
+
"init_lora_weights": "gaussian",
|
| 12 |
+
"layer_replication": null,
|
| 13 |
+
"layers_pattern": null,
|
| 14 |
+
"layers_to_transform": null,
|
| 15 |
+
"loftq_config": {},
|
| 16 |
+
"lora_alpha": 64,
|
| 17 |
+
"lora_dropout": 0.0,
|
| 18 |
+
"megatron_config": null,
|
| 19 |
+
"megatron_core": "megatron.core",
|
| 20 |
+
"modules_to_save": null,
|
| 21 |
+
"peft_type": "LORA",
|
| 22 |
+
"r": 32,
|
| 23 |
+
"rank_pattern": {},
|
| 24 |
+
"revision": null,
|
| 25 |
+
"target_modules": [
|
| 26 |
+
"gate_proj",
|
| 27 |
+
"proj",
|
| 28 |
+
"fc2",
|
| 29 |
+
"down_proj",
|
| 30 |
+
"up_proj",
|
| 31 |
+
"k_proj",
|
| 32 |
+
"fc3",
|
| 33 |
+
"kv",
|
| 34 |
+
"fc1",
|
| 35 |
+
"q",
|
| 36 |
+
"o_proj",
|
| 37 |
+
"qkv",
|
| 38 |
+
"lm_head",
|
| 39 |
+
"v_proj",
|
| 40 |
+
"q_proj"
|
| 41 |
+
],
|
| 42 |
+
"task_type": null,
|
| 43 |
+
"use_dora": false,
|
| 44 |
+
"use_rslora": false
|
| 45 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/preprocessor_config.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoImageProcessor": "processing_prismatic.PrismaticImageProcessor",
|
| 4 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 5 |
+
},
|
| 6 |
+
"image_processor_type": "PrismaticImageProcessor",
|
| 7 |
+
"image_resize_strategy": "resize-naive",
|
| 8 |
+
"input_sizes": [
|
| 9 |
+
[
|
| 10 |
+
3,
|
| 11 |
+
224,
|
| 12 |
+
224
|
| 13 |
+
],
|
| 14 |
+
[
|
| 15 |
+
3,
|
| 16 |
+
224,
|
| 17 |
+
224
|
| 18 |
+
]
|
| 19 |
+
],
|
| 20 |
+
"interpolations": [
|
| 21 |
+
"bicubic",
|
| 22 |
+
"bicubic"
|
| 23 |
+
],
|
| 24 |
+
"means": [
|
| 25 |
+
[
|
| 26 |
+
0.485,
|
| 27 |
+
0.456,
|
| 28 |
+
0.406
|
| 29 |
+
],
|
| 30 |
+
[
|
| 31 |
+
0.5,
|
| 32 |
+
0.5,
|
| 33 |
+
0.5
|
| 34 |
+
]
|
| 35 |
+
],
|
| 36 |
+
"processor_class": "PrismaticProcessor",
|
| 37 |
+
"stds": [
|
| 38 |
+
[
|
| 39 |
+
0.229,
|
| 40 |
+
0.224,
|
| 41 |
+
0.225
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
0.5,
|
| 45 |
+
0.5,
|
| 46 |
+
0.5
|
| 47 |
+
]
|
| 48 |
+
],
|
| 49 |
+
"tvf_crop_params": [
|
| 50 |
+
{
|
| 51 |
+
"output_size": [
|
| 52 |
+
224,
|
| 53 |
+
224
|
| 54 |
+
]
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"output_size": [
|
| 58 |
+
224,
|
| 59 |
+
224
|
| 60 |
+
]
|
| 61 |
+
}
|
| 62 |
+
],
|
| 63 |
+
"tvf_do_letterbox": false,
|
| 64 |
+
"tvf_letterbox_fill": null,
|
| 65 |
+
"tvf_normalize_params": [
|
| 66 |
+
{
|
| 67 |
+
"inplace": false,
|
| 68 |
+
"mean": [
|
| 69 |
+
0.484375,
|
| 70 |
+
0.455078125,
|
| 71 |
+
0.40625
|
| 72 |
+
],
|
| 73 |
+
"std": [
|
| 74 |
+
0.228515625,
|
| 75 |
+
0.2236328125,
|
| 76 |
+
0.224609375
|
| 77 |
+
]
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"inplace": false,
|
| 81 |
+
"mean": [
|
| 82 |
+
0.5,
|
| 83 |
+
0.5,
|
| 84 |
+
0.5
|
| 85 |
+
],
|
| 86 |
+
"std": [
|
| 87 |
+
0.5,
|
| 88 |
+
0.5,
|
| 89 |
+
0.5
|
| 90 |
+
]
|
| 91 |
+
}
|
| 92 |
+
],
|
| 93 |
+
"tvf_resize_params": [
|
| 94 |
+
{
|
| 95 |
+
"antialias": true,
|
| 96 |
+
"interpolation": 3,
|
| 97 |
+
"max_size": null,
|
| 98 |
+
"size": [
|
| 99 |
+
224,
|
| 100 |
+
224
|
| 101 |
+
]
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"antialias": true,
|
| 105 |
+
"interpolation": 3,
|
| 106 |
+
"max_size": null,
|
| 107 |
+
"size": [
|
| 108 |
+
224,
|
| 109 |
+
224
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
],
|
| 113 |
+
"use_fused_vision_backbone": true
|
| 114 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/processing_prismatic.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
processing_prismatic.py
|
| 3 |
+
|
| 4 |
+
HuggingFace-style preprocessor definitions for Prismatic VLMs, inheriting from `ProcessorMixin`. Default configuration
|
| 5 |
+
specifies `siglip-224px+7b`.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Any, ClassVar, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import timm.data
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms.functional as TVF
|
| 13 |
+
from PIL import Image
|
| 14 |
+
from torchvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor
|
| 15 |
+
from transformers import PreTrainedTokenizerBase
|
| 16 |
+
from transformers.image_processing_utils import BatchFeature, ImageProcessingMixin
|
| 17 |
+
from transformers.processing_utils import ProcessorMixin
|
| 18 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
|
| 19 |
+
from transformers.utils import TensorType
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# === Image Processing ===
|
| 23 |
+
def letterbox_pad_transform(image: Image.Image, padding_fill_value: Tuple[int, int, int]) -> Image.Image:
|
| 24 |
+
"""Given a PIL.Image, pad to square by adding a symmetric border around the height/width."""
|
| 25 |
+
(w, h), max_wh = image.size, max(image.size)
|
| 26 |
+
horizontal_pad, vertical_pad = int((max_wh - w) / 2), int((max_wh - h) / 2)
|
| 27 |
+
padding = (horizontal_pad, vertical_pad, horizontal_pad, vertical_pad)
|
| 28 |
+
|
| 29 |
+
return TVF.pad(image, padding, fill=padding_fill_value, padding_mode="constant")
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class PrismaticImageProcessor(ImageProcessingMixin):
|
| 33 |
+
model_input_names: ClassVar[List[str]] = ["pixel_values"]
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
use_fused_vision_backbone: bool = False,
|
| 38 |
+
image_resize_strategy: str = "letterbox",
|
| 39 |
+
input_sizes: Optional[List[Tuple[int, int, int]]] = None,
|
| 40 |
+
interpolations: Optional[List[str]] = None,
|
| 41 |
+
means: Optional[List[Tuple[float, float, float]]] = None,
|
| 42 |
+
stds: Optional[List[Tuple[float, float, float]]] = None,
|
| 43 |
+
**kwargs: str,
|
| 44 |
+
) -> None:
|
| 45 |
+
"""
|
| 46 |
+
Initialize a PrismaticImageProcessor as a wrapper around a torchvision transform; this transform will be
|
| 47 |
+
created by TIMM, and edited to follow our custom `image_resize_strategy` logic.
|
| 48 |
+
@param use_fused_vision_backbone: Boolean indicating single or fused (dual) vision backbone
|
| 49 |
+
@param image_resize_strategy: Prismatic image resize strategy in < resize-naive | resize-crop | letterbox >
|
| 50 |
+
@param input_size: [TIMM :: `data_cfg`] Input image size as tuple (channels, width, height)
|
| 51 |
+
@param interpolation: [TIMM :: `data_cfg`] Interpolation as string (default: "bicubic")
|
| 52 |
+
@param mean: [TIMM :: `data_cfg`] Normalization mean as float tuple (or two-tuple if `fused_backbone`)
|
| 53 |
+
@param std: [TIMM :: `data_cfg`] Normalization std as float tuple (or two-tuple if `fused_backbone`)
|
| 54 |
+
"""
|
| 55 |
+
self.use_fused_vision_backbone = use_fused_vision_backbone
|
| 56 |
+
self.image_resize_strategy = image_resize_strategy
|
| 57 |
+
|
| 58 |
+
# Handle `None` default values
|
| 59 |
+
input_sizes = [(3, 224, 224)] if input_sizes is None else input_sizes
|
| 60 |
+
means = [(0.5, 0.5, 0.5)] if means is None else means
|
| 61 |
+
stds = [(0.5, 0.5, 0.5)] if stds is None else stds
|
| 62 |
+
|
| 63 |
+
# TIMM `data_cfg` Parameters
|
| 64 |
+
self.input_sizes, self.interpolations, self.means, self.stds = input_sizes, interpolations, means, stds
|
| 65 |
+
|
| 66 |
+
# Grab torchvision transforms via TIMM =>> need to parse for specific "functional" transform values!
|
| 67 |
+
self.tvf_resize_params, self.tvf_crop_params, self.tvf_normalize_params = [], [], []
|
| 68 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 69 |
+
|
| 70 |
+
for idx in range(len(input_sizes)):
|
| 71 |
+
transform = timm.data.create_transform(
|
| 72 |
+
input_size=self.input_sizes[idx],
|
| 73 |
+
interpolation=self.interpolations[idx],
|
| 74 |
+
mean=self.means[idx],
|
| 75 |
+
std=self.stds[idx],
|
| 76 |
+
crop_pct=1.0, # Set to 1.0 to ignore cropping (initial Resize sets `input_size`)
|
| 77 |
+
crop_mode="center", # Default crop mode -- no-op when `crop_pct == 1.0`
|
| 78 |
+
is_training=False, # No image augmentations when loading the transform!
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# [Validation] Ensure appropriate transform structure, expected sizes
|
| 82 |
+
if not (
|
| 83 |
+
isinstance(transform, Compose)
|
| 84 |
+
and (len(transform.transforms) == 4)
|
| 85 |
+
and isinstance(transform.transforms[0], Resize)
|
| 86 |
+
and isinstance(transform.transforms[1], CenterCrop)
|
| 87 |
+
and isinstance(transform.transforms[2], ToTensor)
|
| 88 |
+
and isinstance(transform.transforms[3], Normalize)
|
| 89 |
+
and (transform.transforms[0].size == self.input_sizes[idx][-1])
|
| 90 |
+
and (transform.transforms[1].size == self.input_sizes[idx][-2:])
|
| 91 |
+
):
|
| 92 |
+
raise ValueError(f"Unexpected TIMM image transformation structure/sizes: `{transform}`")
|
| 93 |
+
|
| 94 |
+
# HF Image Processors *must* be JSON-serializable; as such, cannot have torchvision. as an attribute.
|
| 95 |
+
# => Instead, we're going to parse the transform and call "torchvision.transforms.functional" (`tvf`)
|
| 96 |
+
resize_t, crop_t, norm_t = transform.transforms[0], transform.transforms[1], transform.transforms[3]
|
| 97 |
+
self.tvf_resize_params.append(
|
| 98 |
+
{
|
| 99 |
+
"size": resize_t.size,
|
| 100 |
+
"interpolation": TVF.pil_modes_mapping[resize_t.interpolation],
|
| 101 |
+
"max_size": None,
|
| 102 |
+
"antialias": True,
|
| 103 |
+
}
|
| 104 |
+
)
|
| 105 |
+
self.tvf_crop_params.append({"output_size": crop_t.size})
|
| 106 |
+
self.tvf_normalize_params.append(
|
| 107 |
+
{
|
| 108 |
+
"mean": norm_t.mean.float().numpy().tolist(),
|
| 109 |
+
"std": norm_t.std.float().numpy().tolist(),
|
| 110 |
+
"inplace": False,
|
| 111 |
+
}
|
| 112 |
+
)
|
| 113 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = False, None
|
| 114 |
+
|
| 115 |
+
# Handle Prismatic `image_resize_strategy`
|
| 116 |
+
if self.image_resize_strategy == "resize-naive":
|
| 117 |
+
self.tvf_resize_params[idx]["size"] = (resize_t.size, resize_t.size)
|
| 118 |
+
elif self.image_resize_strategy == "letterbox":
|
| 119 |
+
self.tvf_do_letterbox, self.tvf_letterbox_fill = True, tuple([int(x * 255) for x in self.means[idx]])
|
| 120 |
+
elif self.image_resize_strategy == "resize-crop":
|
| 121 |
+
pass
|
| 122 |
+
else:
|
| 123 |
+
raise ValueError(f"Image resize strategy `{self.image_resize_strategy}` is not supported!")
|
| 124 |
+
|
| 125 |
+
# Dispatch **kwargs to super()
|
| 126 |
+
super().__init__(**kwargs)
|
| 127 |
+
|
| 128 |
+
def apply_transform(self, img: Image.Image) -> torch.Tensor:
|
| 129 |
+
"""Apply `functional` variant of TIMM's Transform = Compose([Resize -> CenterCrop -> ToTensor -> Normalize])"""
|
| 130 |
+
if self.tvf_do_letterbox:
|
| 131 |
+
img = letterbox_pad_transform(img, self.tvf_letterbox_fill)
|
| 132 |
+
|
| 133 |
+
# [Contract] Fused Backbones expect "channel-stacked" inputs; we'll unpack on the model side!
|
| 134 |
+
imgs_t = []
|
| 135 |
+
for idx in range(len(self.input_sizes)):
|
| 136 |
+
img_idx = TVF.resize(img, **self.tvf_resize_params[idx])
|
| 137 |
+
img_idx = TVF.center_crop(img_idx, **self.tvf_crop_params[idx])
|
| 138 |
+
img_idx_t = TVF.to_tensor(img_idx)
|
| 139 |
+
img_idx_t = TVF.normalize(img_idx_t, **self.tvf_normalize_params[idx])
|
| 140 |
+
imgs_t.append(img_idx_t)
|
| 141 |
+
|
| 142 |
+
# [Contract] `imgs_t` is a list of Tensors of shape [3, input_size, input_size]; stack along dim = 0
|
| 143 |
+
img_t = torch.vstack(imgs_t)
|
| 144 |
+
|
| 145 |
+
return img_t
|
| 146 |
+
|
| 147 |
+
def preprocess(
|
| 148 |
+
self,
|
| 149 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 150 |
+
return_tensors: Optional[Union[str, TensorType]] = None,
|
| 151 |
+
**_: str,
|
| 152 |
+
) -> BatchFeature:
|
| 153 |
+
"""
|
| 154 |
+
Preprocess an image (or batch of images); note that unlike the `transformers :: BaseImageProcessor` we
|
| 155 |
+
explicitly only handle PIL.Image.Image instances for simplicity.
|
| 156 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 157 |
+
@param return_tensors: BatchFeature default Tensor format (e.g., "pt" for torch); if None, returns np.ndarray
|
| 158 |
+
@return: Instance of `transformers :: BatchFeature` with a single key "pixel_values"
|
| 159 |
+
"""
|
| 160 |
+
if not isinstance(images, list):
|
| 161 |
+
images = [images]
|
| 162 |
+
|
| 163 |
+
# Apply `self.img_transform` to each image (will return list of torch.Tensors); stack into "batched" Tensor
|
| 164 |
+
pixel_values = torch.stack([self.apply_transform(img.convert("RGB")) for img in images])
|
| 165 |
+
|
| 166 |
+
# Return BatchFeature =>> note that for compatibility, constructor expects Dict[str, np.ndarray], so we convert
|
| 167 |
+
return BatchFeature(data={"pixel_values": pixel_values.float().numpy()}, tensor_type=return_tensors)
|
| 168 |
+
|
| 169 |
+
def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> BatchFeature:
|
| 170 |
+
return self.preprocess(images, **kwargs)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# === PrismaticProcessor =>> Wraps both ImageProcessor and Tokenizer ===
|
| 174 |
+
# =>> https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava/processing_llava.py
|
| 175 |
+
class PrismaticProcessor(ProcessorMixin):
|
| 176 |
+
attributes: ClassVar[List[str]] = ["image_processor", "tokenizer"]
|
| 177 |
+
image_processor_class: str = "AutoImageProcessor"
|
| 178 |
+
tokenizer_class: str = "AutoTokenizer"
|
| 179 |
+
|
| 180 |
+
def __init__(
|
| 181 |
+
self,
|
| 182 |
+
image_processor: Optional[ImageProcessingMixin] = None,
|
| 183 |
+
tokenizer: Optional[PreTrainedTokenizerBase] = None,
|
| 184 |
+
) -> None:
|
| 185 |
+
super().__init__(image_processor, tokenizer)
|
| 186 |
+
|
| 187 |
+
def __call__(
|
| 188 |
+
self,
|
| 189 |
+
text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
|
| 190 |
+
images: Union[Image.Image, List[Image.Image]],
|
| 191 |
+
padding: Union[bool, str, PaddingStrategy] = False,
|
| 192 |
+
truncation: Optional[Union[bool, str, TruncationStrategy]] = None,
|
| 193 |
+
max_length: Optional[int] = None,
|
| 194 |
+
return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
|
| 195 |
+
) -> BatchFeature:
|
| 196 |
+
"""
|
| 197 |
+
Preprocess a given (batch) of text/images for a Prismatic VLM; forwards text to the underlying LLM's tokenizer,
|
| 198 |
+
forwards images to PrismaticImageProcessor.
|
| 199 |
+
@param text: The (batch) of text to encode; must be a string or list of strings.
|
| 200 |
+
@param images: A (batch of) PIL.Image.Image instance(s) to preprocess.
|
| 201 |
+
@param padding: Sequence padding strategy (if multiple specified) in < True = "longest" | "max_length" | False >
|
| 202 |
+
@param truncation: Truncation strategy for the output sequences; requires `max_length` to be specified
|
| 203 |
+
@param max_length: Maximum length (in tokens) to truncate
|
| 204 |
+
@param return_tensors: Type of return tensors (usually "pt" or TensorType.PYTORCH)
|
| 205 |
+
@return: BatchFeature with keys for `input_ids`, `attention_mask` and `pixel_values`.
|
| 206 |
+
"""
|
| 207 |
+
pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"]
|
| 208 |
+
text_inputs = self.tokenizer(
|
| 209 |
+
text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
# [Validate] Need same number of images and text inputs!
|
| 213 |
+
if pixel_values.shape[0] != text_inputs.input_ids.shape[0]:
|
| 214 |
+
raise ValueError("Batch is malformed; expected same number of images and text inputs!")
|
| 215 |
+
|
| 216 |
+
return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
|
| 217 |
+
|
| 218 |
+
# === Tokenizer Dispatch Utilities =>> check `PreTrainedTokenizerBase` for documentation ===
|
| 219 |
+
def batch_decode(
|
| 220 |
+
self,
|
| 221 |
+
sequences: Union[List[int], List[List[int]], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 222 |
+
skip_special_tokens: bool = False,
|
| 223 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 224 |
+
**kwargs: str,
|
| 225 |
+
) -> List[str]:
|
| 226 |
+
return self.tokenizer.batch_decode(
|
| 227 |
+
sequences=sequences,
|
| 228 |
+
skip_special_tokens=skip_special_tokens,
|
| 229 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 230 |
+
**kwargs,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
def decode(
|
| 234 |
+
self,
|
| 235 |
+
token_ids: Union[int, List[int], torch.Tensor, Any], # `Any` = np.ndarray | tf.Tensor
|
| 236 |
+
skip_special_tokens: bool = False,
|
| 237 |
+
clean_up_tokenization_spaces: Optional[bool] = None,
|
| 238 |
+
**kwargs: str,
|
| 239 |
+
) -> str:
|
| 240 |
+
return self.tokenizer.decode(
|
| 241 |
+
token_ids=token_ids,
|
| 242 |
+
skip_special_tokens=skip_special_tokens,
|
| 243 |
+
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
| 244 |
+
**kwargs,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
@property
|
| 248 |
+
def model_input_names(self) -> List[str]:
|
| 249 |
+
tokenizer_input_names = self.tokenizer.model_input_names
|
| 250 |
+
image_processor_input_names = self.image_processor.model_input_names
|
| 251 |
+
|
| 252 |
+
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/processor_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"auto_map": {
|
| 3 |
+
"AutoProcessor": "processing_prismatic.PrismaticProcessor"
|
| 4 |
+
},
|
| 5 |
+
"processor_class": "PrismaticProcessor"
|
| 6 |
+
}
|
output_flash6/bridge-openvla-oft-2nodes/configs-openvla-7b+bridge+b4+lr-1e-05+AdamW+wd-0+x-action_queries+lora-r32+dropout-0.0--image_aug--OPENVLA-OFT--BRIDGE--20251106-085632--10000_chkpt/special_tokens_map.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "<PAD>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|