Upload 3-epoch ModernBERT-Large pointwise sentence scorer

config.json

@@ -0,0 +1,91 @@
+{
+  "architectures": [
+    "ModernBertForSequenceClassification"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": null,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 2624,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "problem_type": "regression",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "vocab_size": 50368
+}

eval_metrics.json

@@ -0,0 +1,14 @@
+{
+  "eval_loss": 0.022096654400229454,
+  "eval_rmse": 0.1486494345775639,
+  "eval_mae": 0.10976211726665497,
+  "eval_pearson": 0.8568720334438967,
+  "eval_spearman": 0.856351157285778,
+  "eval_article_spearman": 0.7742035575368817,
+  "eval_article_ndcg_at_3": 0.9210334527568008,
+  "eval_article_ndcg_at_5": 0.91633834365026,
+  "eval_runtime": 132.4829,
+  "eval_samples_per_second": 51.275,
+  "eval_steps_per_second": 25.641,
+  "epoch": 3.0
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b240855d759e0ed38acd5112396437a7fea56307b144be4c0354be09552470a
+size 1583347532

run_summary.md

@@ -0,0 +1,44 @@
+# CNN/DailyMail ModernBERT Pointwise Run
+
+- model_size: `large`
+- model_name: `answerdotai/ModernBERT-large`
+- resolved_model_name: `/lustre/fsmisc/dataset/HuggingFace_Models/answerdotai/ModernBERT-large`
+- target_mode: `model_balanced`
+- target_field: `salience_score_model_balanced`
+- input_field: `model_input_text`
+- train_file: `data/CNN_DM/representative_subset/salience_generation/recovery_f2_all6/sentence_scorer_data/modernbert_pointwise/train.jsonl`
+- dev_file: `data/CNN_DM/representative_subset/salience_generation/recovery_f2_all6/sentence_scorer_data/modernbert_pointwise/dev.jsonl`
+- train_rows: `59643`
+- dev_rows: `6793`
+- train_articles: `1000`
+- dev_articles: `200`
+- max_length: `4096`
+- learning_rate: `2e-05`
+- weight_decay: `0.01`
+- num_train_epochs: `3.0`
+- warmup_ratio: `0.06`
+- per_device_train_batch_size: `2`
+- per_device_eval_batch_size: `2`
+- gradient_accumulation_steps: `8`
+- effective_train_batch_size: `16`
+- eval_strategy: `epoch`
+- save_strategy: `epoch`
+- early_stopping_patience: `2`
+- bf16: `True`
+- fp16: `False`
+- gradient_checkpointing: `True`
+
+## Eval Metrics
+
+- epoch: `3.0`
+- eval_article_ndcg_at_3: `0.9210334527568008`
+- eval_article_ndcg_at_5: `0.91633834365026`
+- eval_article_spearman: `0.7742035575368817`
+- eval_loss: `0.022096654400229454`
+- eval_mae: `0.10976211726665497`
+- eval_pearson: `0.8568720334438967`
+- eval_rmse: `0.1486494345775639`
+- eval_runtime: `132.4829`
+- eval_samples_per_second: `51.275`
+- eval_spearman: `0.856351157285778`
+- eval_steps_per_second: `25.641`

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "is_local": true,
+  "local_files_only": true,
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e757282be0fafb916eedeb1c8b4c1bc3015e09163d84da046e3bec1b8f505d8
+size 5393