Upload folder using huggingface_hub

Readme.md

@@ -0,0 +1,51 @@
+# LongFormer Classification Model
+
+## Model Overview
+- **Model Type**: LongFormer
+- **Task**: Text Classification
+- **Framework**: Hugging Face Transformers
+- **Selected Checkpoint**: checkpoint-714
+
+## Model Details
+- **Base Model**: allenai/longformer-base-4096
+- **Max Sequence Length**: 4096 tokens
+- **Model Size**: ~149M parameters
+- **Training Data**: LF_Labelled.csv
+
+## Training Information
+- **Training Date**: October 17-18, 2023
+- **Hardware Used**: GPU (CUDA)
+- **Training Steps**: 714
+- **Optimizer**: AdamW
+- **Learning Rate**: 2e-5
+
+## Model Files
+- `model.safetensors`: Model weights
+- `config.json`: Model architecture configuration
+- `tokenizer.json`: Tokenizer configuration
+- `vocab.json`: Vocabulary file
+- `merges.txt`: BPE merges
+- `optimizer.pt`: Optimizer state
+- `trainer_state.json`: Training state and metrics
+
+## Usage
+```python
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+model_path = "best_model/"
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForSequenceClassification.from_pretrained(model_path)
+```
+
+## Citation
+```bibtex
+@misc{longformer_classification_2023,
+  author = {[Miruzen]},
+  title = {LongFormer Text Classification Model},
+  year = {2023},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+}
+```
+
+Note: Replace the placeholder information in brackets with your specific details.

checkpoint-714/config.json

@@ -0,0 +1,53 @@
+{
+  "architectures": [
+    "LongformerForSequenceClassification"
+  ],
+  "attention_mode": "longformer",
+  "attention_probs_dropout_prob": 0.1,
+  "attention_window": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "negative",
+    "1": "neutral",
+    "2": "positive"
+  },
+  "ignore_attention_mask": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 0,
+    "neutral": 1,
+    "positive": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 4098,
+  "model_type": "longformer",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "onnx_export": false,
+  "pad_token_id": 1,
+  "problem_type": "single_label_classification",
+  "sep_token_id": 2,
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "vocab_size": 50265
+}

checkpoint-714/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee03f55926fe9d1ea2e1fbf6953d2a25a6a59146ff2dbc435cb3a0630f7a332
+size 14645

checkpoint-714/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a99e1c04333b112c3dead49d9a1e5cb9b108be4d2ec0b265b66336f5d39f01d
+size 1383

checkpoint-714/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:767beb689495c50074030315a3e85a069553f68b8d6fc18ada19ae0d4ff25011
+size 1465

checkpoint-714/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-714/tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "LongformerTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

checkpoint-714/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8497a935ae5b53de8200aba72d0ee850572929ea833a1b47df109a3c6e0dd8bd
+size 5905

config.json

@@ -0,0 +1,53 @@
+{
+  "architectures": [
+    "LongformerForSequenceClassification"
+  ],
+  "attention_mode": "longformer",
+  "attention_probs_dropout_prob": 0.1,
+  "attention_window": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "negative",
+    "1": "neutral",
+    "2": "positive"
+  },
+  "ignore_attention_mask": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "negative": 0,
+    "neutral": 1,
+    "positive": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 4098,
+  "model_type": "longformer",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "onnx_export": false,
+  "pad_token_id": 1,
+  "problem_type": "single_label_classification",
+  "sep_token_id": 2,
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 1,
+  "vocab_size": 50265
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f4855974f95d343dbd03f5595e14bece833b7061fb94bc1372febc4fab72b17
+size 594681260

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68854ebfbf33e117faf74236126816804dd9932f3111132ea3d6f401322d9dde
+size 1189526731

runs/Oct17_19-10-37_6bcb8a5f9bf6/events.out.tfevents.1760728242.6bcb8a5f9bf6.23580.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41ae85016f94e72b28bf74ca11e1b7937c7769012dc715635b8b41289a5c27d0
+size 5416

runs/Oct17_19-38-41_6bcb8a5f9bf6/events.out.tfevents.1760729924.6bcb8a5f9bf6.30330.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e21aa3bf22de4fc2901bd5fac3d68cf86e22d8e94b3c570f0719e95ba6d7f2d7
+size 5415

runs/Oct17_19-53-25_6bcb8a5f9bf6/events.out.tfevents.1760730809.6bcb8a5f9bf6.34317.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba25b369f1f5922e35ff5662ab72f4a06f4749d520ad55a6ea203056e0106dde
+size 5415

runs/Oct17_20-20-10_6bcb8a5f9bf6/events.out.tfevents.1760732413.6bcb8a5f9bf6.37927.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ea5ea49d49f0f1f11a82e94f38a35df24684350840a3933423421a2ba6dee2c
+size 5415

runs/Oct18_01-04-56_9ce0259bbb4f/events.out.tfevents.1760749501.9ce0259bbb4f.498.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111140f7ddcd2ca64d59d588358162fa4daada44e227df6b4ce538595ea4f6fe
+size 5415

runs/Oct18_10-56-57_191877b48d26/events.out.tfevents.1760785021.191877b48d26.1449.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:845413291b00316a64021683e2f5e6c104f7f731c9986a81de0ed05f179ecfec
+size 5415

runs/Oct18_11-17-17_191877b48d26/events.out.tfevents.1760786240.191877b48d26.5233.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b78c2cdf76e41002ac6743f38a5cbe8fde3331ec9a86f599efcd866c75e5f53a
+size 5415

runs/Oct18_11-38-21_191877b48d26/events.out.tfevents.1760787504.191877b48d26.10887.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34a97e190e00ed0c8b0a0d3443dc8549aa0f3054a99410779f53a355bfe5953f
+size 5415

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "LongformerTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,177 @@
+{
+  "best_global_step": 714,
+  "best_metric": 0.7832178763628642,
+  "best_model_checkpoint": "/content/drive/MyDrive/Skripsi/output/LongFormer/best_model/checkpoint-714",
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 714,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.21019442984760903,
+      "grad_norm": 4.426321983337402,
+      "learning_rate": 9.65686274509804e-06,
+      "loss": 0.8342,
+      "step": 50
+    },
+    {
+      "epoch": 0.42038885969521805,
+      "grad_norm": 8.081218719482422,
+      "learning_rate": 9.306722689075631e-06,
+      "loss": 0.6552,
+      "step": 100
+    },
+    {
+      "epoch": 0.6305832895428272,
+      "grad_norm": 13.813941955566406,
+      "learning_rate": 8.956582633053222e-06,
+      "loss": 0.5327,
+      "step": 150
+    },
+    {
+      "epoch": 0.8407777193904361,
+      "grad_norm": 15.198319435119629,
+      "learning_rate": 8.606442577030813e-06,
+      "loss": 0.4968,
+      "step": 200
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.8308823529411765,
+      "eval_f1": 0.7542367470980773,
+      "eval_loss": 0.4402936100959778,
+      "eval_precision": 0.7549875294509866,
+      "eval_recall": 0.7534970480092084,
+      "eval_runtime": 267.13,
+      "eval_samples_per_second": 9.164,
+      "eval_steps_per_second": 3.055,
+      "step": 238
+    },
+    {
+      "epoch": 1.0504466631634262,
+      "grad_norm": 9.528948783874512,
+      "learning_rate": 8.256302521008404e-06,
+      "loss": 0.4899,
+      "step": 250
+    },
+    {
+      "epoch": 1.2606410930110352,
+      "grad_norm": 13.074601173400879,
+      "learning_rate": 7.906162464985995e-06,
+      "loss": 0.4595,
+      "step": 300
+    },
+    {
+      "epoch": 1.4708355228586443,
+      "grad_norm": 9.981877326965332,
+      "learning_rate": 7.556022408963586e-06,
+      "loss": 0.4191,
+      "step": 350
+    },
+    {
+      "epoch": 1.6810299527062533,
+      "grad_norm": 9.738601684570312,
+      "learning_rate": 7.205882352941177e-06,
+      "loss": 0.3928,
+      "step": 400
+    },
+    {
+      "epoch": 1.8912243825538622,
+      "grad_norm": 10.563243865966797,
+      "learning_rate": 6.855742296918768e-06,
+      "loss": 0.3938,
+      "step": 450
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.8378267973856209,
+      "eval_f1": 0.7525604819272612,
+      "eval_loss": 0.4178110957145691,
+      "eval_precision": 0.7918661592956496,
+      "eval_recall": 0.7238836613451252,
+      "eval_runtime": 266.9708,
+      "eval_samples_per_second": 9.17,
+      "eval_steps_per_second": 3.057,
+      "step": 476
+    },
+    {
+      "epoch": 2.1008933263268523,
+      "grad_norm": 12.036543846130371,
+      "learning_rate": 6.50560224089636e-06,
+      "loss": 0.3576,
+      "step": 500
+    },
+    {
+      "epoch": 2.3110877561744614,
+      "grad_norm": 14.761561393737793,
+      "learning_rate": 6.155462184873951e-06,
+      "loss": 0.3408,
+      "step": 550
+    },
+    {
+      "epoch": 2.5212821860220704,
+      "grad_norm": 13.06762409210205,
+      "learning_rate": 5.805322128851542e-06,
+      "loss": 0.3219,
+      "step": 600
+    },
+    {
+      "epoch": 2.7314766158696795,
+      "grad_norm": 6.505790710449219,
+      "learning_rate": 5.455182072829132e-06,
+      "loss": 0.3129,
+      "step": 650
+    },
+    {
+      "epoch": 2.9416710457172885,
+      "grad_norm": 14.373762130737305,
+      "learning_rate": 5.105042016806723e-06,
+      "loss": 0.3312,
+      "step": 700
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.8443627450980392,
+      "eval_f1": 0.7832178763628642,
+      "eval_loss": 0.407488614320755,
+      "eval_precision": 0.7723397049441368,
+      "eval_recall": 0.7975281041748842,
+      "eval_runtime": 266.6187,
+      "eval_samples_per_second": 9.182,
+      "eval_steps_per_second": 3.061,
+      "step": 714
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 1428,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 4,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9219670817855454.0,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8497a935ae5b53de8200aba72d0ee850572929ea833a1b47df109a3c6e0dd8bd
+size 5905