initial commit

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+LICENSE.txt filter=lfs diff=lfs merge=lfs -text
+README.md filter=lfs diff=lfs merge=lfs -text
+nervenc filter=lfs diff=lfs merge=lfs -text
+patch_tubelet filter=lfs diff=lfs merge=lfs -text
+teconerv filter=lfs diff=lfs merge=lfs -text

LICENSE.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149d807167937014a4633d3a72ce59e201b01087b08c901257bd17c416481bc5
+size 1071

README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881c0ec81d89da3e6b8b32c9efd8a46e14a71ee5218de4b7c4908f151c34cb24
+size 1519

nervenc/480p_finetuned_baseline/cfg.yaml

@@ -0,0 +1,103 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 8
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 8
+    num_workers: 16
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 480
+        - 640
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_2'
+        strds_w: '5_4_4_4_2'
+        ks: '1_3'
+        hid_dim: 32
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 32
+    n_tokens: '32_256_32_24_0'
+    token_dims: '200_288_288_288_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 50
+eval_epoch: 200
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/nervenc/pre_finetune/pre_finetune_480p_baseline/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/480p_finetuned_baseline
+  instance_tag: 480p_finetuned_baseline
+  tot_gpus: 1
+  cudnn: false
+  port: '29600'
+  wandb_upload: false
+  wandb_exp_name: null
+  wandb_run_id: none
+  distributed: false
+  rank: 0
+  world_size: 1
+  gpu: null

nervenc/480p_finetuned_baseline/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b99efdba84aee0c111efe1ba651de06e342142478b6bdf32a06eb79ed4b9451a
+size 514966594

nervenc/480p_finetuned_baseline_small/cfg.yaml

@@ -0,0 +1,103 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 8
+    num_workers: 24
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 8
+    num_workers: 24
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 480
+        - 640
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_2'
+        strds_w: '5_4_4_4_2'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '20_160_20_20_0'
+    token_dims: '125_120_288_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 50
+eval_epoch: 200
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/nervenc/pre_finetune/pre_finetune_480p_baseline_small/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/480p_finetuned_baseline_small
+  instance_tag: 480p_finetuned_baseline_small
+  tot_gpus: 1
+  cudnn: false
+  port: '29600'
+  wandb_upload: false
+  wandb_exp_name: null
+  wandb_run_id: none
+  distributed: false
+  rank: 0
+  world_size: 1
+  gpu: null

nervenc/480p_finetuned_baseline_small/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e100335142ea79a40ae1259c0fb25a50512a8a9117313def7d618c52955f07d5
+size 507680130

nervenc/720p_finetuned_baseline/cfg.yaml

@@ -0,0 +1,104 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 8
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 8
+    num_workers: 16
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 720
+        - 1280
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_3'
+        strds_w: '5_4_4_4_4'
+        ks: '1_3'
+        hid_dim: 56
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 56
+    n_tokens: '56_448_112_112_0'
+    token_dims: '350_504_224_168_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/nervenc/pre_finetune/pre_finetune_720p_baseline/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/720p_finetuned_baseline
+  instance_tag: 720p_finetuned_baseline
+  tot_gpus: 4
+  cudnn: false
+  port: '4645'
+  wandb_upload: false
+  wandb_exp_name: null
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_run_id: none

nervenc/720p_finetuned_baseline/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea0687cabbee361487cd9a4c49128ca7be7cfb63c0865d9a3e6fbfededdb42c
+size 570814594

nervenc/pre_finetune/pre_finetune_480p_baseline/cfg.yaml

@@ -0,0 +1,101 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 8
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 8
+    num_workers: 16
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 480
+        - 640
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_2'
+        strds_w: '5_4_4_4_2'
+        ks: '1_3'
+        hid_dim: 32
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 32
+    n_tokens: '32_256_32_24_0'
+    token_dims: '200_288_288_288_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 150
+eval_epoch: 200
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/pre_finetune/pre_finetune_480p_baseline
+  instance_tag: pre_finetune_480p_baseline
+  tot_gpus: 1
+  cudnn: false
+  port: '29600'
+  wandb_upload: false
+  wandb_exp_name: null
+  wandb_run_id: none
+  distributed: false
+  rank: 0
+  world_size: 1
+  gpu: null

nervenc/pre_finetune/pre_finetune_480p_baseline/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3dfb4c404d2e17ca15cff392a7ba43cb726b5f136c37dce70c6aecffe97ab9f
+size 514966466

nervenc/pre_finetune/pre_finetune_480p_baseline_small/cfg.yaml

@@ -0,0 +1,101 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 8
+    num_workers: 24
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 8
+    num_workers: 24
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 480
+        - 640
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_2'
+        strds_w: '5_4_4_4_2'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '20_160_20_20_0'
+    token_dims: '125_120_288_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 150
+eval_epoch: 150
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/pre_finetune/pre_finetune_480p_baseline_small
+  instance_tag: pre_finetune_480p_baseline_small
+  tot_gpus: 1
+  cudnn: false
+  port: '29531'
+  wandb_upload: false
+  wandb_exp_name: null
+  wandb_run_id: none
+  distributed: false
+  rank: 0
+  world_size: 1
+  gpu: null

nervenc/pre_finetune/pre_finetune_480p_baseline_small/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff513e0f0a4537f91990e11e726ad0c0e2b83a9d280e97e8968999621d889478
+size 507680002

nervenc/pre_finetune/pre_finetune_720p_baseline/cfg.yaml

@@ -0,0 +1,101 @@
+trainer: nerv_enc_trainer
+train_dataset:
+  name: vidrec_dataset_clip_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+  loader:
+    batch_size: 4
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_clip_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 4
+    num_workers: 16
+model:
+  name: nerv_enc
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 720
+        - 1280
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3_3'
+        strds_w: '5_4_4_4_4'
+        ks: '1_3'
+        hid_dim: 56
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 56
+    n_tokens: '56_448_112_112_0'
+    token_dims: '350_504_224_168_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2800
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+  lr_type: step
+max_epoch: 150
+eval_epoch: 200
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: nervenc
+  save_dir: checkpoints/nervenc/pre_finetune/pre_finetune_720p_baseline
+  instance_tag: pre_finetune_720p_baseline
+  tot_gpus: 1
+  cudnn: false
+  port: '29600'
+  wandb_upload: false
+  wandb_exp_name: null
+  wandb_run_id: none
+  distributed: false
+  rank: 0
+  world_size: 1
+  gpu: null

nervenc/pre_finetune/pre_finetune_720p_baseline/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10119e6ad5c1e0b27f2ae99a383913b991a5f15f0422cbb297776e9fd467cc69
+size 570814466

patch_tubelet/320x160_finetuned_patch/cfg.yaml

@@ -0,0 +1,112 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_56_4_0'
+    token_dims: '196_252_196_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/320x160_finetuned_patch
+  instance_tag: 320x160_finetuned_patch
+  tot_gpus: 4
+  cudnn: false
+  port: '9503'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/320x160_finetuned_patch/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:885d3aae37c82efc6547a924a225202fb6c641461afb3e9acb75a21633834153
+size 495460270

patch_tubelet/320x160_finetuned_patch_small/cfg.yaml

@@ -0,0 +1,112 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_16_4_0'
+    token_dims: '140_252_98_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch_small/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/320x160_finetuned_patch_small
+  instance_tag: 320x160_finetuned_patch_small
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/320x160_finetuned_patch_small/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40204a65e856def367f0d2202d96fdb2f979092d36f4ada67301e161b12d4bc3
+size 493782254

patch_tubelet/320x240_finetuned_patch_train_720p/cfg.yaml

@@ -0,0 +1,112 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 240
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 16
+    n_tokens: '10_80_16_0'
+    token_dims: '200_240_240_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_720p/epoch-last.pth
+finetune_same_model: true
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/320x240_finetuned_patch_train_720p
+  instance_tag: 320x240_finetuned_patch_train_720p
+  tot_gpus: 8
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 8
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/320x240_finetuned_patch_train_720p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ed7817598226c11e07ea5254e3761aa09c2cfd38594f84e3fb7eae6856767fe
+size 498296558

patch_tubelet/pre_finetune/pre_finetune_320x160_patch/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_56_4_0'
+    token_dims: '196_252_196_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch
+  instance_tag: pre_finetune_320x160_patch
+  tot_gpus: 4
+  cudnn: false
+  port: '9503'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_320x160_patch/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdf8df38940c13fa6a33943cfdd367e9e115c4dea622ddfdb77f4acb3baf6ef0
+size 495460206

patch_tubelet/pre_finetune/pre_finetune_320x160_patch_small/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_16_4_0'
+    token_dims: '140_252_98_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 150
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch_small
+  instance_tag: pre_finetune_320x160_patch_small
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_320x160_patch_small/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4541bd2866e16cf7b968ff9fca2594bc225f8f6fb822970c4a021060bf2fdc80
+size 493782126

patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_480p/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 240
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 240
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 240
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 16
+    n_tokens: '10_80_16_0'
+    token_dims: '200_240_240_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 150
+eval_epoch: 150
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_480p
+  instance_tag: pre_finetune_320x240_patch_train_480p
+  tot_gpus: 4
+  cudnn: false
+  port: '29827'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_480p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c00485b39f96c5bdb62bfda1ff5e1bd7c7f4854d0bbc1cb470a0f6deea717ad4
+size 498296430

patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_720p/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+  csv_paths:
+    uvg_720: uvg_hd_720p.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 240
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 16
+    n_tokens: '10_80_16_0'
+    token_dims: '200_240_240_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 150
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_720p
+  instance_tag: pre_finetune_320x240_patch_train_720p
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_720p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a020901b01b660432d31d7709618a93e93091653855f25f0728c7a5089ae135
+size 498296430

patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_480p/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 270
+    - 384
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 270
+    - 384
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 270
+        - 384
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '6_5_3_3'
+        strds_w: '6_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '16_100_16_0'
+    token_dims: '180_240_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 150
+eval_epoch: 150
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_480p
+  instance_tag: pre_finetune_384x270_patch_train_480p
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_480p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2301668741892201c35f9710155bac0f13437d26b39096902f17ab39a9f58d68
+size 499705518

patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_720p/cfg.yaml

@@ -0,0 +1,110 @@
+trainer: nerv_enc_trainer_full_res
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: 'no'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 270
+    - 384
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 270
+    - 384
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 270
+        - 384
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '6_5_3_3'
+        strds_w: '6_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '16_100_16_0'
+    token_dims: '180_240_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 150
+eval_epoch: 150
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+env:
+  exp_name: patch_tubelet
+  save_dir: checkpoints/patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_720p
+  instance_tag: pre_finetune_384x270_patch_train_720p
+  tot_gpus: 4
+  cudnn: false
+  port: '34306'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_720p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c2afa676bd450479b9ffc32a29af06b6ba499255dc98d3a996a1b2899ec0362
+size 499705518

teconerv/320x160_pairs_teco/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_56_4_0'
+    token_dims: '196_252_196_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/320x160_pairs_teco
+  instance_tag: 320x160_pairs_teco
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/320x160_pairs_teco/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ed76b258306ec03a2c14ed37d2fd223debdf84ff4589a5e3876ddef8436ff6
+size 495460398

teconerv/320x160_pairs_teco_small/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 160
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 160
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_2'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 14
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 14
+    n_tokens: '5_16_4_0'
+    token_dims: '140_252_98_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x160_patch_small/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/320x160_pairs_teco_small
+  instance_tag: 320x160_pairs_teco_small
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/320x160_pairs_teco_small/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b09a04bdbe61ed92046b528cb73d6a72817c8a945c7514d59d29e094ed4627
+size 493782382

teconerv/320x240_pairs_teco/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 240
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 240
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 240
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 16
+    n_tokens: '10_80_16_0'
+    token_dims: '200_240_240_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_480p/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/320x240_pairs_teco
+  instance_tag: 320x240_pairs_teco
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/320x240_pairs_teco/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0508493a47424adb8c317a2cef6adca4294063c6f8c79f487f616e1ff44739fa
+size 498296686

teconerv/320x240_pairs_teco_train_720p/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 240
+    - 320
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 240
+        - 320
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '5_4_4_3'
+        strds_w: '5_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 16
+    n_tokens: '10_80_16_0'
+    token_dims: '200_240_240_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_320x240_patch_train_720p/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/320x240_pairs_teco_train_720p
+  instance_tag: 320x240_pairs_teco_train_720p
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/320x240_pairs_teco_train_720p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d69f34fdaafeb71586eaf9d5c23dab3d540826a5b234f9003775d4c0df00
+size 498296686

teconerv/384x270_pairs_teco/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_480p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 270
+    - 384
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 480
+    - 640
+    tubelet_size:
+    - 270
+    - 384
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 270
+        - 384
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '6_5_3_3'
+        strds_w: '6_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '16_100_16_0'
+    token_dims: '180_240_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_480p/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/384x270_pairs_teco
+  instance_tag: 384x270_pairs_teco
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/384x270_pairs_teco/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbdb918f792590202bb56a40b89e684d9cd9a7e04eac73705a91562b5d33d7cb
+size 499705774

teconerv/384x270_pairs_teco_train_720p/cfg.yaml

@@ -0,0 +1,115 @@
+trainer: nerv_enc_trainer_full_res_pairs
+train_dataset:
+  name: vidrec_dataset_patch_tubelet_sampler_lazy_pairs
+  args:
+    root_path: data/dataset_meta
+    split: train
+    frame_num: 8
+    rand_augment: '1_2_5'
+    csv_file: k400_2023_train_cls400_50_720p.js
+    cls_vid_num: '400_25'
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 270
+    - 384
+    scale: 1
+    aspect_ratio: 1
+    rand_flip: 'no'
+    clips_per_video: 1
+    tubelets_per_clip: 1
+  loader:
+    batch_size: 32
+    num_workers: 16
+test_dataset:
+  name: vidrec_dataset_patch_tubelet_inference_lazy_pairs_uvg
+  args:
+    root_path: data/dataset_meta
+    frame_num: 8
+    cls_vid_num: -1_-1
+    crop_size:
+    - 720
+    - 1280
+    tubelet_size:
+    - 270
+    - 384
+  csv_paths:
+    uvg: uvg_hd.csv
+  frames:
+    input: none
+    output: none
+  loader:
+    batch_size: 32
+    num_workers: 16
+model:
+  name: nerv_enc_full_res_pairs
+  args:
+    tokenizer:
+      name: vidrec_tokenizer
+      args:
+        input_size:
+        - 270
+        - 384
+        patch_size: 32
+        padding: 0
+        frame_num: 8
+        eval_frames: none
+        img_groups: 1
+    hyponet:
+      name: hypo_convnets_full_res
+      args:
+        in_dim: 1
+        out_dim: 3
+        out_bias: tanh
+        strds_h: '6_5_3_3'
+        strds_w: '6_4_4_4'
+        ks: '1_3'
+        hid_dim: 20
+        size: none
+        act: gelu
+        use_pe: true
+        pe_dim: 20
+    n_tokens: '16_100_16_0'
+    token_dims: '180_240_180_0'
+    transformer_encoder:
+      name: transformer_encoder
+      args:
+        dim: 720
+        depth: 6
+        n_head: 12
+        head_dim: 64
+        ff_dim: 2880
+optimizer:
+  name: adam
+  args:
+    lr: 0.0001
+    weight_decay: 0.0
+  lr_type: step
+max_epoch: 50
+eval_epoch: 50
+vis_epoch: 2000
+dump_ckt: 'no'
+dump_pred: 'no'
+dump_video: 'no'
+generate_from_single_frame: false
+finetune_model: checkpoints/patch_tubelet/pre_finetune/pre_finetune_384x270_patch_train_720p/epoch-last.pth
+finetune_same_model: false
+param_reg_mode: mod
+param_reg_lambda_l1: 0.1
+param_reg_lambda_l2: 0.0
+env:
+  exp_name: teconerv
+  save_dir: checkpoints/teconerv/384x270_pairs_teco_train_720p
+  instance_tag: 384x270_pairs_teco_train_720p
+  tot_gpus: 4
+  cudnn: false
+  port: '15419'
+  wandb_upload: false
+  rank: 0
+  world_size: 4
+  gpu: 0
+  distributed: true
+  dist_backend: nccl
+  wandb_exp_name: null
+  wandb_run_id: none

teconerv/384x270_pairs_teco_train_720p/epoch-last.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac262b0a9e57ddf044c31693944a9d679d1aa79e28b850f2c8335f0f5abcb671
+size 499705774