Upload folder using huggingface_hub

V4.3-ckpt/ckpt_iter0005000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56c056016d8b8a248e7d54c844b6c004cfe54e1e483921f5b1952b065ca85349
+size 158369093

V4.3-ckpt/ckpt_iter0005000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b531154b45173908020ee9a162a8100f9cc3d8d10deb37b09d68389308a6135d
+size 26443493

V4.3-ckpt/ckpt_iter0010000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da64c8d2b0268d31f45c524311146f16e9cbb688d917100e125834b617c2c9b5
+size 158369093

V4.3-ckpt/ckpt_iter0010000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b380f89583e0e2a2ca080f5ba4036d78b87c669669dd7a6fb2d5ba9d3cda83dd
+size 26443493

V4.3-ckpt/ckpt_iter0015000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d81bfb74f3b4f2052f26112507bdfae7927aa34eb499538107394f7a960d3c8c
+size 158369093

V4.3-ckpt/ckpt_iter0015000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f49055976a46362bf30c1be8ca9157e66a90f2475ed9c76cf0f8137cd86f05b
+size 26443493

V4.3-ckpt/ckpt_iter0020000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e75ff5e61aa6a945214debc3e1a795d94942ff9a808dc0f6477e822cad626c4b
+size 158369093

V4.3-ckpt/ckpt_iter0020000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62837c90687bd793e4f6e7d8913c6f6d151f4d61ddacbcb69d10c29a20fe86a7
+size 26443493

V4.3-ckpt/ckpt_iter0025000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ca26f145d58915af43963f5c3801eedccbbf4206a50fa85cab73d7835811a5d
+size 158369093

V4.3-ckpt/ckpt_iter0025000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c323dd0af112f1acb8c4b73beda2e326aa6f5e78d3c77e65b69aab953e49ead0
+size 26443493

V4.3-ckpt/ckpt_iter0030000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bee656e4312a9700e9e379b217f3e550c356f2ccda11b15e01c4c3f0e22da5ae
+size 158369093

V4.3-ckpt/ckpt_iter0030000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7b9ee7f0523c215814a1ca476a77847a79eff99d1b39acb3a15287ba0e5d27
+size 26443493

V4.3-ckpt/ckpt_iter0035000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8da5825bf83934e65a7210eb7cd1e0bc057bb3a1d1554c9238f559cb8314a361
+size 158369093

V4.3-ckpt/ckpt_iter0035000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ea7171231f41b203c368337ff928a17e16c81df3e0d843a1915594d8e26f495
+size 26443493

V4.3-ckpt/ckpt_iter0040000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5448f30ccdae36c771d6945b984aff7383872024ea17b1afc659f6a8dea2fb6
+size 158369093

V4.3-ckpt/ckpt_iter0040000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e70668941469527626a4fa068217258aa7d5269f56d5beaa4da58395e5928ab5
+size 26443493

V4.3-ckpt/ckpt_iter0045000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:046269135cd8be837f2397b20a241cd657b3ed8146689ebc03f31d416e96aeb4
+size 158369093

V4.3-ckpt/ckpt_iter0045000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eb4432e39cf49637932611eb23f53f5259e0e97936500334183a0d1f872eaa6
+size 26443493

V4.3-ckpt/ckpt_iter0050000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39a023ee1da3bdee2870c96ecc671bc6a877826f3d78939e79fd6705d7133439
+size 158369093

V4.3-ckpt/ckpt_iter0050000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5d9e063bde8ee672fcad75c4f9618615f6edb7c18350b56c04973a2723f6c45
+size 26443493

V4.3-ckpt/ckpt_iter0055000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1da6e6b735e6d45595a608e43ae7d161deb7922f02757636e5b0d3ee06e50e7
+size 158369093

V4.3-ckpt/ckpt_iter0055000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e58283a54fa4c0ddb9d757b5e7f40dd4fcd4a4a0fa1b40ae5a7b69c5354b5741
+size 26443493

V4.3-ckpt/ckpt_iter0060000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03d2ad0d1fc3ac4f3d0dbf083955bbfaf32b893c1221b80cb4f7b430bbf3ea45
+size 158369093

V4.3-ckpt/ckpt_iter0060000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bc97f15576086587cb20c2745cf41c59881c3e2d3d9df2915f8d157ca7d815a
+size 26443493

V4.3-ckpt/ckpt_iter0065000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aead2cd83a525d092ebd67b163a32c1b26f115f5d990a63d5d89d8bc7354eef
+size 158369093

V4.3-ckpt/ckpt_iter0065000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a971fc817b840f3f5f0dce4dbdafec4342e4e6879bf1ac464cade9ad033db11
+size 26443493

V4.3-ckpt/ckpt_iter0070000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd8c3d952773317fc8c7e86456c26d63cb3a0c97d7caaa2c6fd62ec7e39135dd
+size 158369093

V4.3-ckpt/ckpt_iter0070000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff32e409e5a8d8b3f7f71edd2e8cb251890ea7a2be606c6ca03c6a931673c10a
+size 26443493

V4.3-ckpt/ckpt_iter0075000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ed28a79904a7820cbf7759416ff99f0bd13fa938f88192e05b637f641297ad3
+size 158369093

V4.3-ckpt/ckpt_iter0075000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa06fb8ebce63e2be53630a27cca6e736e2d62691c409cab3f8bba9768fb8c69
+size 26443493

V4.3-ckpt/ckpt_iter0080000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360fa7b5e63ace9b88522feea8df6c4e3ce1c4d2d40fd0da960d582e6284af4b
+size 158369093

V4.3-ckpt/ckpt_iter0080000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33aed92487ae27d3fa58c8f3d986c4c9bfa4e5cb8c6ffa0023047b8912b0441
+size 26443493

V4.3-ckpt/ckpt_iter0085000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bae8dd84fdfae832e273b3f4495533d441105ad22a23f09a7721f1fdac88c4b9
+size 158369093

V4.3-ckpt/ckpt_iter0085000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:320ed1fa617cfd7e59180f4035c35a0ba9483500e8747832c532b2304e9fcd26
+size 26443493

V4.3-ckpt/ckpt_iter0090000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5356ed18783ee39e805ebb55315be818e386adfbd5b4af38c4d3c41bf9696c0
+size 158369093

V4.3-ckpt/ckpt_iter0090000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f66984aab3c7fdf34c7058e19c6a175e1d93838e5e7c39cbc4702a221ef1b384
+size 26443493

V4.3-ckpt/ckpt_iter0095000.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:878e1ccefc862a1c2263801b39bac100dfb3586596988f10504cd45e69c7bc6f
+size 158369093

V4.3-ckpt/ckpt_iter0095000.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e6979b3666f1766dc13530a5eea3bd87d317550fd3849b8b3de9c5a9c3e0858
+size 26443493

V4.3-ckpt/config.json

@@ -0,0 +1,243 @@
+{
+  "root_path": "/home/zli",
+  "available_corpus": {
+    "cc3m": {
+      "anno_path": "your_path",
+      "data_root": "",
+      "media_type": "image"
+    },
+    "webvid_10m": {
+      "anno_path": "your_path",
+      "data_root": "",
+      "media_type": "video"
+    },
+    "smol_test": {
+      "anno_path": "/root/IV2/InternVideo2/multi_modality/data_test/smol_test.json",
+      "data_root": "/root/IV2/InternVideo2/multi_modality/data_test/",
+      "media_type": "video"
+    },
+    "slim_kinetics": {
+      "anno_path": "/home/zli/kinetics-dataset/k600/train/train/kinetics_v2.json",
+      "data_root": "/home/zli/kinetics-dataset/k600/train/train",
+      "media_type": "video"
+    },
+    "slim_kinetics_act_val": {
+      "anno_path": "/home/zli/kinetics-dataset/k600/test/kinetics-test.json",
+      "data_root": "/home/zli/kinetics-dataset/k600/test/",
+      "media_type": "video",
+      "is_act_rec": true
+    }
+  },
+  "VisionEncoders": {},
+  "TextEncoders": {
+    "bert": {
+      "name": "bert_base",
+      "pretrained": "bert-base-uncased",
+      "config": "configs/config_bert.json",
+      "d_model": 768,
+      "fusion_layer": 9
+    },
+    "bert_large": {
+      "name": "bert_large",
+      "pretrained": "bert-large-uncased",
+      "config": "configs/config_bert_large.json",
+      "d_model": 1024,
+      "fusion_layer": 19
+    },
+    "med_bert": {
+      "name": "med_bert_base",
+      "pretrained": "bert-base-uncased",
+      "config": "configs/med_config.json",
+      "d_model": 768
+    },
+    "med_bert_large": {
+      "name": "med_bert_large",
+      "pretrained": "bert-base-uncased",
+      "config": "configs/med_large_config.json",
+      "d_model": 768
+    }
+  },
+  "train_corpus": "slim_kinetics",
+  "train_file": {
+    "anno_path": "/home/zli/kinetics-dataset/k600/train/train/kinetics_v2.json",
+    "data_root": "/home/zli/kinetics-dataset/k600/train/train",
+    "media_type": "video"
+  },
+  "test_file": {
+    "act_val": {
+      "anno_path": "/home/zli/kinetics-dataset/k600/test/kinetics-test.json",
+      "data_root": "/home/zli/kinetics-dataset/k600/test/",
+      "media_type": "video",
+      "is_act_rec": true
+    }
+  },
+  "test_types": [
+    "act_val"
+  ],
+  "num_workers": 2,
+  "stop_key": null,
+  "num_frames": 8,
+  "num_frames_test": 8,
+  "batch_size": 16,
+  "batch_size_test": 16,
+  "max_txt_l": 32,
+  "size_t": 224,
+  "inputs": {
+    "image_res": 224,
+    "video_input": {
+      "num_frames": 8,
+      "sample_type": "all",
+      "num_frames_test": 8,
+      "sample_type_test": "all",
+      "random_aug": false
+    },
+    "max_txt_l": {
+      "image": 32,
+      "video": 32
+    },
+    "batch_size": {
+      "image": 16,
+      "video": 16
+    },
+    "batch_size_test": {
+      "image": 16,
+      "video": 16
+    }
+  },
+  "model": {
+    "model_cls": "InternVideo2_CLIP_small",
+    "vision_encoder": {
+      "name": "internvideo2",
+      "in_chans": 3,
+      "patch_size": 14,
+      "img_size": 224,
+      "qkv_bias": false,
+      "drop_path_rate": 0.0,
+      "head_drop_path_rate": 0.0,
+      "embed_dim": 768,
+      "num_heads": 12,
+      "mlp_ratio": 4,
+      "init_values": 0.1,
+      "qk_normalization": true,
+      "depth": 12,
+      "use_flash_attn": true,
+      "use_fused_rmsnorm": true,
+      "use_fused_mlp": true,
+      "fused_mlp_heuristic": 1,
+      "drop_cls_token": false,
+      "attn_pool_num_heads": 16,
+      "clip_embed_dim": 768,
+      "layerscale_no_force_fp32": true,
+      "num_frames": 8,
+      "tubelet_size": 1,
+      "sep_pos_embed": false,
+      "use_checkpoint": false,
+      "checkpoint_num": 0,
+      "align_dim": 512
+    },
+    "streaming_vision_encoder": {
+      "in_chans": 3,
+      "patch_size": 14,
+      "img_size": 224,
+      "vit_qkv_bias": true,
+      "vit_drop_path_rate": 0.05,
+      "student_embed_dim": 384,
+      "student_depth": 4,
+      "student_num_heads": 6,
+      "vit_mlp_ratio": 3.0,
+      "vit_init_values": null,
+      "vit_qk_normalization": false,
+      "vit_sep_pos_embed": true,
+      "vit_norm_layer_type": "rmsnorm",
+      "rnn_type": "lstm",
+      "rnn_hidden_size": 1024,
+      "rnn_num_layers": 1,
+      "fc_hidden_layers": [],
+      "teacher_clip_embed_dim": 768,
+      "student_num_frames_processed_by_vit": 1,
+      "student_tubelet_size_for_vit": 1
+    },
+    "text_encoder": {
+      "name": "mobileclip_b"
+    },
+    "temp": 0.01,
+    "temp_min": 0.01,
+    "freeze_vision": true,
+    "open_vision_clip_projector": false,
+    "freeze_text": true,
+    "open_text_projection": false,
+    "open_text_lora": false,
+    "vision_ckpt_path": "/home/zli/IV2/models/stage1/B14/B14_dist_1B_stage2/pytorch_model.bin",
+    "load_vision_ckpt_from_internvideo2_stage2": false,
+    "text_ckpt_path": "/home/zli/IV2/models/mobileclip_blt.pt",
+    "extra_ckpt_path": "/home/zli/IV2/models/clip/B14/pytorch_model.bin"
+  },
+  "criterion": {
+    "loss_weight": {
+      "vtc": 1.0
+    }
+  },
+  "optimizer": {
+    "opt": "adamW",
+    "lr": 1e-05,
+    "opt_betas": [
+      0.9,
+      0.98
+    ],
+    "weight_decay": 0.01,
+    "max_grad_norm": 0.7,
+    "different_lr": {
+      "enable": false,
+      "module_names": [],
+      "lr": 1e-05
+    }
+  },
+  "scheduler": {
+    "sched": "cosine",
+    "epochs": 1,
+    "min_lr_multi": 0.01,
+    "warmup_epochs": 0.05
+  },
+  "evaluate": false,
+  "deep_fusion": false,
+  "evaluation": {
+    "eval_frame_ensemble": "concat",
+    "eval_x_only": false,
+    "k_test": 128,
+    "eval_offload": true
+  },
+  "use_half_precision": true,
+  "use_bf16": true,
+  "gradient_checkpointing": true,
+  "wandb": {
+    "enable": true,
+    "entity": "qingy2019-conker-mobile-inc-",
+    "project": "window_iv2"
+  },
+  "dist_url": "env://",
+  "device": "cuda",
+  "mode": "pt",
+  "output_dir": "scripts/pretraining/clip/B14/B14",
+  "resume": true,
+  "debug": false,
+  "log_freq": 1,
+  "seed": 42,
+  "save_latest": false,
+  "save_iter": 5000,
+  "eval_freq_steps": 1000,
+  "eval_video_repo_id": "qingy2024/backflip_train",
+  "eval_video_filename": "1.mp4",
+  "eval_plot_output_dir": "scripts/pretraining/clip/B14/cosine_sim_graphs",
+  "auto_resume": true,
+  "pretrained_path": "",
+  "deepspeed": {
+    "enable": true,
+    "stage": 1
+  },
+  "rank": 0,
+  "world_size": 1,
+  "gpu": 0,
+  "distributed": true,
+  "dist_backend": "nccl",
+  "deepspeed_config": "scripts/pretraining/clip/B14/B14/deepspeed_config.json"
+}