Upload trained VideoMAE with metrics and all figures

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+confusion_matrix_test_50.png filter=lfs diff=lfs merge=lfs -text
+confusion_matrix_train_50.png filter=lfs diff=lfs merge=lfs -text
+confusion_matrix_valid_50.png filter=lfs diff=lfs merge=lfs -text
+longtail_f1_vs_freq.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,73 @@
+---
+library_name: transformers
+license: cc-by-nc-4.0
+base_model: MCG-NJU/videomae-base
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: VideoMAE_WLASL_2000__200_epoch_p20_longtail
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# VideoMAE_WLASL_2000__200_epoch_p20_longtail
+
+This model is a fine-tuned version of [MCG-NJU/videomae-base](https://huggingface.co/MCG-NJU/videomae-base) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 4.6155
+- Accuracy: 0.0207
+- Top 1 Accuracy: 0.0207
+- Top 5 Accuracy: 0.0592
+- Top 10 Accuracy: 0.1124
+- Macro Precision: 0.0005
+- Macro Recall: 0.0117
+- Macro F1: 0.0010
+- Pearson Corr: nan
+- Spearman Corr: nan
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 8
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- training_steps: 180
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy | Top 1 Accuracy | Top 5 Accuracy | Top 10 Accuracy | Macro Precision | Macro Recall | Macro F1 | Pearson Corr | Spearman Corr |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:--------------:|:--------------:|:---------------:|:---------------:|:------------:|:--------:|:------------:|:-------------:|
+| 18.6831       | 1.0   | 180  | 4.6155          | 0.0207   | 0.0207         | 0.0592         | 0.1124          | 0.0005          | 0.0117       | 0.0010   | nan          | nan           |
+
+
+### Framework versions
+
+- Transformers 4.46.1
+- Pytorch 2.5.1+cu124
+- Datasets 3.1.0
+- Tokenizers 0.20.1

config.json

@@ -0,0 +1,235 @@
+{
+  "_name_or_path": "MCG-NJU/videomae-base",
+  "architectures": [
+    "VideoMAEForVideoClassification"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "decoder_hidden_size": 384,
+  "decoder_intermediate_size": 1536,
+  "decoder_num_attention_heads": 6,
+  "decoder_num_hidden_layers": 4,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "accident",
+    "1": "africa",
+    "2": "all",
+    "3": "apple",
+    "4": "basketball",
+    "5": "bed",
+    "6": "before",
+    "7": "bird",
+    "8": "birthday",
+    "9": "black",
+    "10": "blue",
+    "11": "book",
+    "12": "bowling",
+    "13": "brown",
+    "14": "but",
+    "15": "can",
+    "16": "candy",
+    "17": "chair",
+    "18": "change",
+    "19": "cheat",
+    "20": "city",
+    "21": "clothes",
+    "22": "color",
+    "23": "computer",
+    "24": "cook",
+    "25": "cool",
+    "26": "corn",
+    "27": "cousin",
+    "28": "cow",
+    "29": "dance",
+    "30": "dark",
+    "31": "deaf",
+    "32": "decide",
+    "33": "doctor",
+    "34": "dog",
+    "35": "drink",
+    "36": "eat",
+    "37": "enjoy",
+    "38": "family",
+    "39": "fine",
+    "40": "finish",
+    "41": "fish",
+    "42": "forget",
+    "43": "full",
+    "44": "give",
+    "45": "go",
+    "46": "graduate",
+    "47": "hat",
+    "48": "hearing",
+    "49": "help",
+    "50": "hot",
+    "51": "how",
+    "52": "jacket",
+    "53": "kiss",
+    "54": "language",
+    "55": "last",
+    "56": "later",
+    "57": "letter",
+    "58": "like",
+    "59": "man",
+    "60": "many",
+    "61": "medicine",
+    "62": "meet",
+    "63": "mother",
+    "64": "need",
+    "65": "no",
+    "66": "now",
+    "67": "orange",
+    "68": "paint",
+    "69": "paper",
+    "70": "pink",
+    "71": "pizza",
+    "72": "play",
+    "73": "pull",
+    "74": "purple",
+    "75": "right",
+    "76": "same",
+    "77": "school",
+    "78": "secretary",
+    "79": "shirt",
+    "80": "short",
+    "81": "son",
+    "82": "study",
+    "83": "table",
+    "84": "tall",
+    "85": "tell",
+    "86": "thanksgiving",
+    "87": "thin",
+    "88": "thursday",
+    "89": "time",
+    "90": "walk",
+    "91": "want",
+    "92": "what",
+    "93": "white",
+    "94": "who",
+    "95": "woman",
+    "96": "work",
+    "97": "wrong",
+    "98": "year",
+    "99": "yes"
+  },
+  "image_size": 224,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "accident": 0,
+    "africa": 1,
+    "all": 2,
+    "apple": 3,
+    "basketball": 4,
+    "bed": 5,
+    "before": 6,
+    "bird": 7,
+    "birthday": 8,
+    "black": 9,
+    "blue": 10,
+    "book": 11,
+    "bowling": 12,
+    "brown": 13,
+    "but": 14,
+    "can": 15,
+    "candy": 16,
+    "chair": 17,
+    "change": 18,
+    "cheat": 19,
+    "city": 20,
+    "clothes": 21,
+    "color": 22,
+    "computer": 23,
+    "cook": 24,
+    "cool": 25,
+    "corn": 26,
+    "cousin": 27,
+    "cow": 28,
+    "dance": 29,
+    "dark": 30,
+    "deaf": 31,
+    "decide": 32,
+    "doctor": 33,
+    "dog": 34,
+    "drink": 35,
+    "eat": 36,
+    "enjoy": 37,
+    "family": 38,
+    "fine": 39,
+    "finish": 40,
+    "fish": 41,
+    "forget": 42,
+    "full": 43,
+    "give": 44,
+    "go": 45,
+    "graduate": 46,
+    "hat": 47,
+    "hearing": 48,
+    "help": 49,
+    "hot": 50,
+    "how": 51,
+    "jacket": 52,
+    "kiss": 53,
+    "language": 54,
+    "last": 55,
+    "later": 56,
+    "letter": 57,
+    "like": 58,
+    "man": 59,
+    "many": 60,
+    "medicine": 61,
+    "meet": 62,
+    "mother": 63,
+    "need": 64,
+    "no": 65,
+    "now": 66,
+    "orange": 67,
+    "paint": 68,
+    "paper": 69,
+    "pink": 70,
+    "pizza": 71,
+    "play": 72,
+    "pull": 73,
+    "purple": 74,
+    "right": 75,
+    "same": 76,
+    "school": 77,
+    "secretary": 78,
+    "shirt": 79,
+    "short": 80,
+    "son": 81,
+    "study": 82,
+    "table": 83,
+    "tall": 84,
+    "tell": 85,
+    "thanksgiving": 86,
+    "thin": 87,
+    "thursday": 88,
+    "time": 89,
+    "walk": 90,
+    "want": 91,
+    "what": 92,
+    "white": 93,
+    "who": 94,
+    "woman": 95,
+    "work": 96,
+    "wrong": 97,
+    "year": 98,
+    "yes": 99
+  },
+  "layer_norm_eps": 1e-12,
+  "model_type": "videomae",
+  "norm_pix_loss": true,
+  "num_attention_heads": 12,
+  "num_channels": 3,
+  "num_frames": 16,
+  "num_hidden_layers": 12,
+  "patch_size": 16,
+  "problem_type": "single_label_classification",
+  "qkv_bias": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.1",
+  "tubelet_size": 2,
+  "use_mean_pooling": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb41e99ca1aa4af41540e5e25f3d54535bae1353169b90b63e8305f5fc2ccfae
+size 345238832

per_class_report.csv

@@ -0,0 +1,103 @@
+,precision,recall,f1-score,support,train_freq,pearson_corr,spearman_corr
+accident,0.0,0.0,0.0,3,1.0,,
+africa,0.0,0.0,0.0,3,1.0,,
+all,0.0,0.0,0.0,3,1.0,,
+apple,0.0,0.0,0.0,2,1.0,,
+basketball,0.0,0.0,0.0,2,1.0,,
+bed,0.0,0.0,0.0,2,1.0,,
+before,0.0,0.0,0.0,4,1.0,,
+bird,0.0,0.0,0.0,2,1.0,,
+birthday,0.0,0.0,0.0,3,1.0,,
+black,0.0,0.0,0.0,3,1.0,,
+blue,0.0,0.0,0.0,2,1.0,,
+book,0.018691588785046728,1.0,0.03669724770642202,4,1.0,,
+bowling,0.0,0.0,0.0,2,1.0,,
+brown,0.0,0.0,0.0,2,1.0,,
+but,0.0,0.0,0.0,3,1.0,,
+can,0.0,0.0,0.0,2,1.0,,
+candy,0.0,0.0,0.0,3,1.0,,
+chair,0.0,0.0,0.0,3,1.0,,
+change,0.0,0.0,0.0,2,1.0,,
+cheat,0.0,0.0,0.0,2,1.0,,
+city,0.0,0.0,0.0,2,1.0,,
+clothes,0.0,0.0,0.0,3,1.0,,
+color,0.0,0.0,0.0,2,1.0,,
+computer,0.0,0.0,0.0,5,1.0,,
+cook,0.0,0.0,0.0,2,1.0,,
+cool,0.0,0.0,0.0,3,1.0,,
+corn,0.0,0.0,0.0,3,1.0,,
+cousin,0.0,0.0,0.0,3,1.0,,
+cow,0.0,0.0,0.0,3,1.0,,
+dance,0.0,0.0,0.0,2,1.0,,
+dark,0.0,0.0,0.0,3,1.0,,
+deaf,0.0,0.0,0.0,3,1.0,,
+decide,0.0,0.0,0.0,2,1.0,,
+doctor,0.0,0.0,0.0,3,1.0,,
+dog,0.0,0.0,0.0,3,1.0,,
+drink,0.0,0.0,0.0,4,1.0,,
+eat,0.0,0.0,0.0,2,1.0,,
+enjoy,0.0,0.0,0.0,2,1.0,,
+family,0.0,0.0,0.0,2,1.0,,
+fine,0.0,0.0,0.0,3,1.0,,
+finish,0.0,0.0,0.0,3,1.0,,
+fish,0.0,0.0,0.0,3,1.0,,
+forget,0.0,0.0,0.0,2,1.0,,
+full,0.0,0.0,0.0,2,1.0,,
+give,0.0,0.0,0.0,2,1.0,,
+go,0.0,0.0,0.0,3,1.0,,
+graduate,0.0,0.0,0.0,2,1.0,,
+hat,0.0,0.0,0.0,2,1.0,,
+hearing,0.0,0.0,0.0,2,1.0,,
+help,0.0,0.0,0.0,3,1.0,,
+hot,0.0,0.0,0.0,3,1.0,,
+how,0.0,0.0,0.0,3,1.0,,
+jacket,0.0,0.0,0.0,2,1.0,,
+kiss,0.0,0.0,0.0,3,1.0,,
+language,0.0,0.0,0.0,3,1.0,,
+last,0.0,0.0,0.0,3,1.0,,
+later,0.0,0.0,0.0,2,1.0,,
+letter,0.0,0.0,0.0,3,1.0,,
+like,0.0,0.0,0.0,3,1.0,,
+man,0.0,0.0,0.0,2,1.0,,
+many,0.0,0.0,0.0,3,1.0,,
+medicine,0.0,0.0,0.0,2,1.0,,
+meet,0.0,0.0,0.0,2,1.0,,
+mother,0.0,0.0,0.0,3,1.0,,
+need,0.0,0.0,0.0,2,1.0,,
+no,0.0,0.0,0.0,3,1.0,,
+now,0.0,0.0,0.0,3,1.0,,
+orange,0.0,0.0,0.0,3,1.0,,
+paint,0.0,0.0,0.0,2,1.0,,
+paper,0.0,0.0,0.0,2,1.0,,
+pink,0.0,0.0,0.0,3,1.0,,
+pizza,0.0,0.0,0.0,3,1.0,,
+play,0.0,0.0,0.0,2,1.0,,
+pull,0.0,0.0,0.0,2,1.0,,
+purple,0.0,0.0,0.0,2,1.0,,
+right,0.0,0.0,0.0,2,1.0,,
+same,0.0,0.0,0.0,3,1.0,,
+school,0.0,0.0,0.0,2,1.0,,
+secretary,0.0,0.0,0.0,3,1.0,,
+shirt,0.0,0.0,0.0,3,1.0,,
+short,0.0,0.0,0.0,2,1.0,,
+son,0.0,0.0,0.0,2,1.0,,
+study,0.0,0.0,0.0,2,1.0,,
+table,0.0,0.0,0.0,3,1.0,,
+tall,0.0,0.0,0.0,3,1.0,,
+tell,0.0,0.0,0.0,2,1.0,,
+thanksgiving,0.0,0.0,0.0,3,1.0,,
+thin,0.0,0.0,0.0,3,1.0,,
+thursday,0.0,0.0,0.0,2,1.0,,
+time,0.0,0.0,0.0,2,1.0,,
+walk,0.0,0.0,0.0,3,1.0,,
+want,0.0,0.0,0.0,2,1.0,,
+what,0.0,0.0,0.0,3,1.0,,
+white,0.0,0.0,0.0,2,1.0,,
+who,0.0,0.0,0.0,3,1.0,,
+woman,0.0,0.0,0.0,3,1.0,,
+work,0.0,0.0,0.0,2,1.0,,
+wrong,0.0,0.0,0.0,2,1.0,,
+year,0.0,0.0,0.0,3,1.0,,
+yes,0.0,0.0,0.0,3,1.0,,
+macro avg,0.00018691588785046728,0.01,0.0003669724770642202,258,,,
+weighted avg,0.00028979207418677096,0.015503875968992248,0.0005689495768437523,258,,,

preprocessor_config.json

@@ -0,0 +1,26 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "VideoMAEImageProcessor",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:622a4b2c480ab98f4218ea94d80223d25e7e66f749f9449f80783b9d4a8e973c
+size 5368