Upload folder using huggingface_hub

backup/cfgs/howto-anet_anet_clip_topk20_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 20
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk20_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 20
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk20_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 20
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk20_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 20
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk30_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk30_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk30_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk30_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk40_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 40
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk40_r1_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 40
+width_ratio: 1
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk40_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 40
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs/howto-anet_anet_clip_topk40_r2_iter3_th2_refine_aug(8,0.02)_top3_2stage_inscap_puyu.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_puyu.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 40
+width_ratio: 2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs_base/howto/base_howto-anet_anet_mixlm.yml

@@ -0,0 +1,64 @@
+id: anet
+
+visual_feature_type: c3d
+visual_feature_folder: 'data/anet/features/c3d'
+feature_dim: 500
+invalid_video_json: []
+train_proposal_file: data/generated_proposals/dbg_trainval_top100.json
+eval_proposal_file: data/generated_proposals/dbg_trainval_top100.json
+gt_file_for_eval: ['data/anet/captiondata/val_1.json', 'data/anet/captiondata/val_2.json']
+gt_file_for_para_eval: ['data/anet/captiondata/para/anet_entities_val_1_para.json', 'data/anet/captiondata/para/anet_entities_val_2_para.json']
+train_caption_file: ['data/howto/captiondata/howto100m_train_mixlm.json', 'data/anet/captiondata/train_modified.json']
+val_caption_file: 'data/anet/captiondata/val_1.json'
+
+max_caption_len: 50
+
+dict_file: data/howto/vocabulary_howto_rate2_mixlm_anet.json
+vocab_size: 18884
+# dict_file_for_sim: data/howto/vocabulary_howto_rate5.json
+# vocab_size: 8531
+
+
+train_proposal_type: gt
+train_proposal_sample_num: 30
+sample_method: nearest
+
+epoch: 10
+batch_size: 1
+lr: 0.00005
+learning_rate_decay_start: 8
+learning_rate_decay_every: 3
+learning_rate_decay_rate: 0.5
+weight_decay: 0.0001
+save_all_checkpoint: 0
+
+num_queries: 100
+dec_layers: 2
+enc_layers: 2
+transformer_ff_dim: 512
+transformer_dropout_prob: 0.1
+frame_embedding_num: 100
+caption_decoder_type: light
+att_hid_size: 0
+
+with_box_refine: 1
+
+fix_xcw: 1
+set_cost_caption: 0
+set_cost_giou: 4
+set_cost_bbox: 0
+set_cost_class: 2
+self_iou_loss_coef: 0
+#cost_alpha: 0.5
+#cost_gamma: 1
+#focal_alpha: 0.5
+#focal_gamma: 1
+caption_loss_coef: 2
+giou_loss_coef: 4
+bbox_loss_coef: 0
+cls_loss_coef: 2
+count_loss_coef: 0.5
+max_eseq_length: 10
+lloss_cross_entropy: 0
+lloss_focal_loss: 0
+lloss_gau_mask: 1

backup/cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml

@@ -0,0 +1,61 @@
+id: yc2_tsn_pdvcl
+
+visual_feature_type: ['resnet', 'bn']
+visual_feature_folder: ['data/yc2/features/resnet_bn/', 'data/yc2/features/resnet_bn/']
+feature_dim: 3072
+invalid_video_json: []
+train_caption_file: ['data/howto/captiondata/howto100m_train_mixlm.json', 'data/yc2/captiondata/yc2_train.json']
+val_caption_file: 'data/yc2/captiondata/yc2_val.json'
+gt_file_for_eval: ['data/yc2/captiondata/yc2_val.json']
+gt_file_for_para_eval: ['data/yc2/captiondata/para/para_yc2_val.json']
+max_caption_len: 50
+
+dict_file: data/howto/vocabulary_howto_rate2_mixlm_yc2.json
+vocab_size: 17447
+# dict_file_for_sim: data/howto/vocabulary_howto_rate5.json
+# vocab_size: 8531
+
+
+train_proposal_type: gt
+train_proposal_sample_num: 30
+sample_method: nearest
+
+epoch: 10
+batch_size: 1
+lr: 0.00005
+learning_rate_decay_start: 8
+learning_rate_decay_every: 3
+learning_rate_decay_rate: 0.5
+weight_decay: 0.0001
+save_all_checkpoint: 0
+
+num_queries: 100
+dec_layers: 2
+enc_layers: 2
+transformer_ff_dim: 512
+transformer_dropout_prob: 0.1
+frame_embedding_num: 200
+caption_decoder_type: light
+att_hid_size: 0
+
+with_box_refine: 1
+
+fix_xcw: 1
+set_cost_caption: 0
+set_cost_giou: 4
+set_cost_bbox: 0
+set_cost_class: 2
+self_iou_loss_coef: 0
+#cost_alpha: 0.5
+#cost_gamma: 1
+#focal_alpha: 0.5
+#focal_gamma: 1
+caption_loss_coef: 2
+giou_loss_coef: 4
+bbox_loss_coef: 0
+cls_loss_coef: 2
+count_loss_coef: 0.5
+max_eseq_length: 20
+lloss_cross_entropy: 0
+lloss_focal_loss: 0
+lloss_gau_mask: 1

backup/cfgs_base/howto/howto-anet_anet_anc_(sim_op_order_v2)_CLIP_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_anc_(sim_op_order_v2)_CLIP_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 2
+top_frames: 35
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_anc_(sim_op_order_v2)_UniVL_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_anc_(sim_op_order_v2)_UniVL_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 2
+top_frames: 35
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/CLIP_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 2
+top_frames: 35
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_UniVL_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_UniVL_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-anet_anet_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet/UniVL_feature/text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 2
+top_frames: 35
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_anc_(sim_op_order_v2)_CLIP_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_anc_(sim_op_order_v2)_CLIP_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 3
+top_frames: 10
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_anc_(sim_op_order_v2)_UniVL_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_anc_(sim_op_order_v2)_UniVL_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 3
+top_frames: 10
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 1
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_CLIP_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_CLIP_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 3
+top_frames: 10
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_CLIP_refine_mixlm_v0.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['CLIP']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder: ['/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/video/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 3
+top_frames: 10
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: CLIP
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_mixlm.yml

@@ -0,0 +1,42 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 0
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 0
+refine_pseudo_stage_num: 1
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 12
+
+use_anchor: 0
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_puyu.yml

@@ -3,7 +3,7 @@ base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_puyu.yml
 
 visual_feature_type: ['UniVL']
 visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
-text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+text_feature_folder: ['/mnt/data/pjlab-3090-sport/wuhao/features/howto100m/univl_features/text_puyu', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
 visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
 text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
 feature_dim: 768

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_refine_mixlm.yml

@@ -0,0 +1,46 @@
+id: refine_aug(5,0.3)_top3_1stage
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_mixlm.yml
+
+visual_feature_type: ['UniVL']
+visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
+text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+feature_dim: 768
+hidden_dim: 512
+
+use_pseudo_box: 1
+pseudo_box_aug: 1
+pseudo_box_aug_num: 5
+pseudo_box_aug_ratio: 0.3
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+iteration: 3
+width_th: 2
+statistic_mode: mode
+width_ratio: 1
+window_size: 3
+top_frames: 10
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 30
+
+use_anchor: 0
+pretrained_language_model: UniVL
+disable_contrastive_projection: 1
+
+caption_decoder_type: standard
+cap_nheads: 1
+cap_dec_n_points: 4
+cap_num_feature_levels: 4
+soft_attention: 1
+att_hid_size: 512
+
+num_queries: 100
+
+ec_alpha: 1.0
+
+self_iou_loss_coef: 0.0
+ref_rank_loss_coef: 0.0
+contrastive_loss_start_coef: 0.0

backup/cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_refine_puyu.yml

@@ -3,7 +3,7 @@ base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2_puyu.yml
 
 visual_feature_type: ['UniVL']
 visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
-text_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
+text_feature_folder: ['/mnt/data/pjlab-3090-sport/wuhao/features/howto100m/univl_features/text_puyu', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
 visual_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_visual/']
 text_feature_folder_val: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text/']
 feature_dim: 768

backup/cfgs_base/howto/howto_yc2_ori_(sim_op_order_v2)_UniVL_refine.yml

@@ -1,5 +1,5 @@
 id: refine_aug(5,0.3)_top3_1stage
-base_cfg_path: cfgs_base/howto/base_howto_yc2.yml
+base_cfg_path: cfgs_base/howto/base_howto-yc2_yc2.yml
 
 visual_feature_type: ['UniVL']
 visual_feature_folder: ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/visual']

backup/cfgs_ft_gt/howto-anet_anet_clip_topk30_r1_iter3_th1_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 1
+iteration: 3
+width_th: 1
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs_ft_gt/howto-anet_anet_clip_topk30_r1_iter3_th1_refine_aug(8,0.02)_top3_2stage_inscap_rand2.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-anet_anet_ori_(sim_op_order_v2)_CLIP_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 30
+width_ratio: 1
+iteration: 3
+width_th: 1
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs_ft_gt/howto-yc2_yc2_univl_topk25_r1_iter3_th1_refine_aug(8,0.02)_top3_2stage_inscap.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 25
+width_ratio: 1
+iteration: 3
+width_th: 1
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/cfgs_ft_gt/howto-yc2_yc2_univl_topk25_r1_iter3_th1_refine_aug(8,0.02)_top3_2stage_inscap_rand2.yml

@@ -0,0 +1,19 @@
+id: ''
+base_cfg_path: cfgs_base/howto/howto-yc2_yc2_ori_(sim_op_order_v2)_UniVL_refine.yml
+
+
+pseudo_box_aug_num: 8
+pseudo_box_aug_ratio: 0.02
+pseudo_box_aug_mode: random_range
+refine_pseudo_box: 1
+refine_pseudo_stage_num: 2
+merge_k_boxes: 3
+pseudo_box_type: similarity_op_order_v2
+top_frames: 25
+width_ratio: 1
+iteration: 3
+width_th: 1
+use_query_box_for_refine: 0
+gt_proposal_sample_num: 20
+mil_loss_coef: 0
+merge_criterion: ins_cap_topk

backup/change_config_add.py

@@ -12,10 +12,12 @@ args = parser.parse_args()
 
 
 # Define the folder containing YAML files
-folder_path = 'cfgs_ref'
+# folder_path = 'cfgs_ref'
 # folder_path = 'cfgs_base/anet'
 # folder_path = 'cfgs'
-file_filter = 'yc2'
+folder_path = 'cfgs_yc2_ft_perc'
+
+file_filter = ''
 
 
 
@@ -24,18 +26,18 @@ file_filter = 'yc2'
 # find_string = '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/clip_features/text_proj'
 # find_string = 'data/yc2/captiondata/yc2'
 # find_string = "/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2/UniVL_features/UniVL_text"
-find_string = "UniVL_refine"
+find_string = "ft_gt_percent: 0.25"
 # find_string = "pdvc_mode: 0"
 
 # replace_string = '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/Tasty/features/clip/visual'
 # replace_string = '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/Tasty/features/clip/text'
 # replace_string = 'data/tasty/captiondata/tasty'
 # replace_string = "cfgs_base/tasty/tasty_tsn_pdvcl.yml"
-replace_string = "CLIP_refine"
+replace_string = "ft_gt_percent: 0.75"
 # replace_string = "pdvc_mode: 1"
 
-old_name = 'univl'
-new_name = 'clip'
+old_name = 'perc0.25'
+new_name = 'perc0.75'
 
 def replace_yaml(yaml_file_path, new_file_path, old_string, new_string):
     # Read the YAML file as text

backup/misc/utils.py

@@ -241,7 +241,7 @@ def build_folder(opt):
                 save_foldername += '_C-layer'
             if 'puyu' in opt.train_caption_file[0]:
                 save_foldername += '_puyu'
-            elif 'mix' in opt.train_caption_file[0]:
+            elif 'mixlm' in opt.train_caption_file[0]:
                 save_foldername += '_mixlm'
 
         if opt.id != '':
@@ -281,8 +281,13 @@ def build_folder(opt):
     return save_folder
 
 
-def backup_envir(save_folder):
+def backup_envir(save_folder, opt):
+    cfg_path = opt.cfg_path
+    dir_path = os.path.dirname(cfg_path)
     backup_folders = ['cfgs_base', 'cfgs', 'misc', 'pdvc']
+    if dir_path not in backup_folders:
+        backup_folders.append(dir_path)
+
     backup_files = glob.glob('./*.py')
     for folder in backup_folders:
         shutil.copytree(folder, os.path.join(save_folder, 'backup', folder))

backup/opts.py

@@ -269,6 +269,7 @@ def parse_opts():
 
     # reranking
     parser.add_argument('--ec_alpha', type=float, default=0.3)
+    parser.add_argument('--test', action='store_true', default=False)
     args = parser.parse_args()
 
     if args.cfg_path:

backup/pdvc/pdvc.py

@@ -316,6 +316,8 @@ class PDVC(nn.Module):
                     video_step_alignment = [align_frame_into_steps_op_v1(dt['video_tensor'][i], raw_text_embed[i].to(memory.device), topk=self.opt.top_frames, scale=self.opt.width_ratio, beta=1, order=True, num_iterations=self.opt.iteration) for i in range(N)]  
                 elif self.opt.pseudo_box_type == 'similarity_op_order_v2':
                     video_step_alignment = [align_frame_into_steps_op_order_v2(dt['video_tensor'][i], raw_text_embed[i].to(memory.device), topk=self.opt.top_frames, threshold=self.opt.width_th, ratio=self.opt.width_ratio, iteration=self.opt.iteration) for i in range(N)]
+                elif self.opt.pseudo_box_type == 'similarity_op_v2':
+                    video_step_alignment = [align_frame_into_steps_op_v2(dt['video_tensor'][i], raw_text_embed[i].to(memory.device), topk=self.opt.top_frames, threshold=self.opt.width_th, ratio=self.opt.width_ratio, iteration=self.opt.iteration) for i in range(N)]
                 elif self.opt.pseudo_box_type == 'weight_sim':
                     if self.opt.width_ratio < 0:
                         video_step_alignment = [step_retrieval_weight_sim(dt['video_tensor'][i], raw_text_embed[i].to(memory.device), \

backup/pdvc/video_segmentation.py

@@ -632,6 +632,65 @@ def align_frame_into_steps_op_order_v2(frame_features, step_features, topk=15, t
 
     return (best_bbox, min_loss)
 
+def align_frame_into_steps_op_v2(frame_features, step_features, topk=15, threshold=0.5, ratio=1, iteration=3):
+    # breakpoint()
+    if step_features.shape[0] == 0:
+        return -np.ones(frame_features.shape[0])
+    
+    sim = compute_sim(step_features, frame_features, config_eval_l2norm).cpu()
+    sorted_index = torch.argsort(-sim, dim=1)
+    top_indices_list_global = [sorted_index[i][:topk] for i in range(sim.shape[0])]
+    top_values_list_global = [sim[i][top_indices_list_global[i]] for i in range(sim.shape[0])]
+
+
+    uniform_boxes = uniform_window(frame_features.shape[0], step_features.shape[0])
+
+    iter_bbox_loss = {}
+    for iter in range(iteration):
+        # if iter == 0:
+        #     refined_uniform_boxes = expand_window(uniform_boxes, frame_features.shape[0], step_features.shape[0], ratio)
+        # else:
+        #     refined_uniform_boxes = expand_window(bbox, frame_features.shape[0], step_features.shape[0], ratio) # last bbox
+
+
+        # global: from all frames, local: from refined uniform boxes
+    
+        # top_indices_list_local = [sorted_index[i][(sorted_index[i] >= refined_uniform_boxes[i][0]) & (sorted_index[i] <= refined_uniform_boxes[i][1])][:topk] for i in range(sim.shape[0])]
+        # top_values_list_local = [sim[i][top_indices_list_local[i]] for i in range(sim.shape[0])]
+
+        # size_local = [len(top_indices_list_local[i]) for i in range(sim.shape[0])]
+        # if sum(size_local) < (topk-2) * len(size_local):
+        #     top_indices_list = top_indices_list_global
+        #     top_values_list = top_values_list_global
+        # else:
+        #     top_indices_list = top_indices_list_local
+        #     top_values_list = top_values_list_local
+
+        # top_indices_list = [top_indices_list_global[i] if len(top_indices_list_local[i]) < topk else top_indices_list_local[i] for i in range(sim.shape[0])]
+
+        bbox = []
+        for i in range(len(top_indices_list_global)):
+            filtered_indices = compute_filtered_indices(top_indices_list_global[i].tolist(), top_values_list_global[i].tolist(), threshold)
+            if len(filtered_indices) == 0:
+                filtered_indices = compute_filtered_indices(top_indices_list_global[i].tolist(), top_indices_list_global[i].tolist(), threshold)
+                if len(filtered_indices) == 0:
+                    bbox.append(uniform_boxes[i])
+                    continue
+            bbox.append([min(filtered_indices), max(filtered_indices)])
+
+        # compute bbox loss
+        bbox_loss_list = [compute_bbox_loss(top_indices_list_global[i], bbox[i], top_values_list_global[i]) for i in range(len(top_indices_list_global))]
+        bbox_loss = sum(bbox_loss_list)
+        iter_bbox_loss[iter] = {'loss': bbox_loss, 'bbox': bbox}
+
+    # select the minimum bbox loss and bbox as output
+    min_loss_iter = min(iter_bbox_loss.keys(), key=lambda k: iter_bbox_loss[k]['loss'])
+    min_loss = iter_bbox_loss[min_loss_iter]['loss']
+    best_bbox = iter_bbox_loss[min_loss_iter]['bbox']
+        
+
+    return (best_bbox, min_loss)
+
 
 
 # pesudo box 4: based on fixed window. the result is bad. give up

backup/train.py

@@ -48,8 +48,8 @@ def construct_save_path(opt, save_folder="/mnt/data/pjlab-3090-sport/wuhao/code/
     if len(opt.train_caption_file) == 2:
         if 'puyu' in opt.train_caption_file[0]:
             elements.append('howto_puyu')
-        elif 'mix' in opt.train_caption_file[0]:
-            elements.append('howto_mix')
+        elif 'mixlm' in opt.train_caption_file[0]:
+            elements.append('howto_mixlm')
         else:
             elements.append('howto_llama2')
         elements.append('howto')
@@ -65,8 +65,8 @@ def construct_save_path(opt, save_folder="/mnt/data/pjlab-3090-sport/wuhao/code/
         elif 'howto' in opt.train_caption_file:
             if 'puyu' in opt.train_caption_file:
                 elements.append('howto_puyu')
-            elif 'mix' in opt.train_caption_file:
-                elements.append('howto_mix')
+            elif 'mixlm' in opt.train_caption_file:
+                elements.append('howto_mixlm')
             else:
                 elements.append('howto_llama2')
             # elements.append('howto')
@@ -114,8 +114,97 @@ def train(opt):
     logger = create_logger(save_folder, 'train.log')
     tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))
 
+    # if use mixlm model
+    saved_path = construct_save_path(opt)
+
+    if 'mixlm' in saved_path:
+        # text_feature_folder_mixlm = os.path.join(save_folder, 'text_feature')
+        mixlm_pbox_path = construct_save_path(opt, save_folder='test').replace('.json', '').replace('test/', '')
+        text_feature_folder_mixlm = os.path.join('/mnt/data/Gvlab/wuhao/code/tmp', 'mix_text_feature', mixlm_pbox_path)
+        os.makedirs(text_feature_folder_mixlm, exist_ok=True)
+        if 'clip' in save_folder or 'CLIP' in save_folder:
+            text_feature_folder_llama2 = map_path('/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip/text_proj')
+            text_feature_folder_puyu = '/mnt/data/Gvlab/wuhao/code/clip_frame_feature_extraction/features/howto100m/clip'
+        elif 'univl' in save_folder or 'UniVL' in save_folder or 'Uni' in save_folder:
+            text_feature_folder_llama2 = map_path('/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/UniVL/text')
+            text_feature_folder_puyu = '/mnt/data/pjlab-3090-sport/wuhao/features/howto100m/univl_features/text_puyu'
+
+        if not os.path.exists(saved_path):        
+            llama2_pbox_path = saved_path.replace('mixlm', 'llama2')
+            puyu_pbox_path = saved_path.replace('mixlm', 'puyu')
+            with open(llama2_pbox_path, 'r') as f:
+                llama2_pbox = json.load(f)
+            with open(puyu_pbox_path, 'r') as f:
+                puyu_pbox = json.load(f)
+            
+            mixlm_pbox = {}
+            for video_key in llama2_pbox.keys():
+                if llama2_pbox.get(video_key) is None and puyu_pbox.get(video_key) is None:
+                    mixlm_pbox[video_key] = None
+                elif llama2_pbox.get(video_key) is None:
+                    mixlm_pbox[video_key] = {'box': puyu_pbox[video_key]['box'], 'loss': puyu_pbox[video_key]['loss'], 'llm': 'puyu'}
+                elif puyu_pbox.get(video_key) is None:
+                    mixlm_pbox[video_key] = {'box': llama2_pbox[video_key]['box'], 'loss': llama2_pbox[video_key]['loss'], 'llm': 'llama2'}
+                else:
+                    if llama2_pbox[video_key]['loss'] < puyu_pbox[video_key]['loss']:
+                        mixlm_pbox[video_key] = {'box': llama2_pbox[video_key]['box'], 'loss': llama2_pbox[video_key]['loss'], 'llm': 'llama2'}
+                    else:
+                        mixlm_pbox[video_key] = {'box': puyu_pbox[video_key]['box'], 'loss': puyu_pbox[video_key]['loss'], 'llm': 'puyu'}
+            with open(saved_path, 'w') as f:
+                json.dump(mixlm_pbox, f)
+        
+        with open(saved_path, 'r') as f:
+            mixlm_pbox = json.load(f)
+        with open('data/howto/captiondata/howto100m_train_puyu.json', 'r') as f:
+            meta_puyu = json.load(f)
+        with open('data/howto/captiondata/howto100m_train.json', 'r') as f:
+            meta_llama2 = json.load(f)
+        
+        meta_mixlm = {}
+        for video_key in mixlm_pbox.keys():
+            if mixlm_pbox.get(video_key) is not None and (meta_llama2.get(video_key) is not None or meta_puyu.get(video_key) is not None):
+                if mixlm_pbox[video_key]['llm'] == 'llama2':
+                    meta_mixlm[video_key] = meta_llama2[video_key]
+                    llama2_feature_path = os.path.join(text_feature_folder_llama2, video_key + '.npy')
+                    if not os.path.exists(llama2_feature_path):
+                        continue
+                    # if os.path.exists(llama2_feature_path):
+                    #     os.unlink(llama2_feature_path)
+                    # if not os.path.exists(llama2_feature_path):                        
+                    #     os.symlink(llama2_feature_path, os.path.join(text_feature_folder_mixlm, video_key + '.npy'))
+                    soft_link_path = os.path.join(text_feature_folder_mixlm, video_key + '.npy')
+                    # if os.path.exists(soft_link_path):
+                    #     os.unlink(soft_link_path)
+                    if not os.path.exists(soft_link_path):
+                        # print(os.path.exists(soft_link_path), os.path.exists(llama2_feature_path))
+                        os.symlink(llama2_feature_path, soft_link_path)
+                    # text_feature = np.load(llama2_feature_path)
+                    # if text_feature.shape[0] != len(meta_llama2[video_key]['sentences']):
+                    #     print(f"{video_key} has {text_feature.shape[0]} sentences, but {len(meta_llama2[video_key]['sentences'])} sentences found in meta file")
+                else:
+                    meta_mixlm[video_key] = meta_puyu[video_key]
+                    puyu_feature_path = os.path.join(text_feature_folder_puyu, video_key + '.npy')
+                    if not os.path.exists(puyu_feature_path):
+                        continue
+
+                    soft_link_path = os.path.join(text_feature_folder_mixlm, video_key + '.npy')
+
+                    # if os.path.exists(soft_link_path):
+                    #     os.unlink(soft_link_path)
+                    if not os.path.exists(soft_link_path):
+                        os.symlink(puyu_feature_path, soft_link_path)
+                    # text_feature = np.load(puyu_feature_path)
+                    # if text_feature.shape[0] != len(meta_puyu[video_key]['sentences']):
+                    #     print(f"{video_key} has {text_feature.shape[0]} sentences, but {len(meta_puyu[video_key]['sentences'])} sentences found in meta file")
+        with open(os.path.join(save_folder, 'train_caption_mixlm.json'), 'w') as f:
+            json.dump(meta_mixlm, f)
+        opt.train_caption_file[0] = os.path.join(save_folder, 'train_caption_mixlm.json')
+        opt.text_feature_folder[0] = text_feature_folder_mixlm       
+        # pass
+
+
     if not opt.start_from:
-        backup_envir(save_folder)
+        backup_envir(save_folder, opt)
         logger.info('backup evironment completed !')
 
     saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}
@@ -136,6 +225,8 @@ def train(opt):
                 if prev_opt.get(opt_name) != vars(opt).get(opt_name):
                     logger.info('Change opt {} : {} --> {}'.format(opt_name, prev_opt.get(opt_name),
                                                                    vars(opt).get(opt_name)))
+    print(opt.text_feature_folder)
+    print(opt.train_caption_file)
     if len(opt.visual_feature_folder) == 2:
         train_dataset_1 = PropSeqDataset(opt.train_caption_file[0],
                                             [opt.visual_feature_folder[0]],
@@ -195,7 +286,6 @@ def train(opt):
     model.train()
 
     # try to load saved pbox
-    saved_path = construct_save_path(opt)
     if os.path.exists(saved_path):
         try:
             with open(saved_path, 'r') as f:
@@ -322,10 +412,11 @@ def train(opt):
             # if dt['video_key'][0] != 'LGArj9Do0xc':
             #     continue
             # # for fast debugging
-            # if trained_samples > 5:
-            #     break
-            # else:
-            #     trained_samples += 1
+            if opt.test:
+                if trained_samples > 5:
+                    break
+                else:
+                    trained_samples += 1
             # if trained_samples < 1714:
             #     trained_samples += 1
             #     continue
@@ -486,7 +577,7 @@ def train(opt):
 
         epoch += 1
 
-        if epoch == 1 and model.pseudo_boxes is not None and 'hyper' not in opt.train_caption_file[0]:
+        if epoch == 1 and model.pseudo_boxes is not None and 'mixlm' not in opt.train_caption_file[0]:
             # save the pseudo boxes
             pbox_save_path = construct_save_path(opt)
             if not os.path.exists(pbox_save_path):

backup/train_fewshot.py

@@ -68,7 +68,7 @@ def train(opt):
     tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))
 
     if not opt.start_from:
-        backup_envir(save_folder)
+        backup_envir(save_folder, opt)
         logger.info('backup evironment completed !')
 
     saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}

backup/train_ft2_gt.py

@@ -147,7 +147,7 @@ def train(opt):
     tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))
 
     if not opt.start_from:
-        backup_envir(save_folder)
+        backup_envir(save_folder, opt)
         logger.info('backup evironment completed !')
 
     saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}
@@ -190,16 +190,17 @@ def train(opt):
         # train_dataset.translator = train_dataset_1.translator
 
     else:
-        print('the script only support two dataset for pretrain and target task respectively')
-        exit(1)
+        # print('the script only support two dataset for pretrain and target task respectively')
+        # exit(1)
         train_dataset_target = PropSeqDataset(opt.train_caption_file,
                                     opt.visual_feature_folder,
                                     opt.text_feature_folder,
                                     opt.dict_file, True, 'gt',
                                     opt)
-        train_loader_target = DataLoader(train_dataset_target, batch_size=opt.batch_size,
+        subset_data = PercentageSubsetDataset(train_dataset_target, opt.ft_gt_percent)
+        train_loader_target = DataLoader(subset_data, batch_size=opt.batch_size,
                                 shuffle=True, num_workers=opt.nthreads, collate_fn=collate_fn, worker_init_fn=_init_fn)
-        train_dataloaders = [train_loader_target]
+        # train_dataloaders = [train_loader_target]
 
     # val_dataset = PropSeqDataset(opt.val_caption_file,
     #                              opt.visual_feature_folder,

backup/train_pre_ft_gt.py

@@ -45,7 +45,7 @@ import copy
 a100_folder = ['/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/youcook2', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/Tasty/features', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Tasty/UniVL_feature', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/huabin/dataset/Anet', '/cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features']
 r3090_folder = ['/mnt/data/Gvlab/wuhao/features/yc2', '/mnt/data/Gvlab/wuhao/features/tasty', '/mnt/data/Gvlab/wuhao/features/tasty/univl', '/mnt/data/Gvlab/wuhao/features/anet', '/mnt/data/Gvlab/wuhao/features/howto100m']
 
-pretrain_data_mode = 'single' # 'mix' or 'seq' or 'single'
+pretrain_data_mode = 'mix' # 'mix' or 'seq' or 'single'
 
 # /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features -> /mnt/data/Gvlab/wuhao/features/howto100m
 # /cpfs01/shared/Gvlab-A100/Gvlab-A100_hdd/wuhao/howto100m/features/clip -> /mnt/data/Gvlab/wuhao/features/howto100m/clip_features
@@ -122,7 +122,7 @@ def train(opt):
     tf_writer = SummaryWriter(os.path.join(save_folder, 'tf_summary'))
 
     if not opt.start_from:
-        backup_envir(save_folder)
+        backup_envir(save_folder, opt)
         logger.info('backup evironment completed !')
 
     saved_info = {'best': {}, 'last': {}, 'history': {}, 'eval_history': {}}