checkpoint_metadata.json

{
  "release_stage": "mlmcontinued (stage 2, epoch 25)",
  "reported_loss": 0.8958267427277438,
  "parameter_count": 273051864,
  "fp16_export": true,
  "tokenizer": "google-bert/bert-base-uncased",
  "pretraining_stages": [
    {
      "stage": 1,
      "name": "encoder",
      "objective": "token-level masked language modeling at 20% mask rate 80/10/10 split",
      "epochs": 24,
      "seq_len": 256,
      "final_train_loss": 1.0678829201169648,
      "final_train_accuracy": 76.41802635495705
    },
    {
      "stage": 2,
      "name": "mlmcontinued",
      "objective": "whole-word-masking continued pretraining at 18% mask rate",
      "epochs": 25,
      "seq_len": 256,
      "final_train_loss": 0.8958267427277438,
      "final_train_accuracy": 79.66191907459469
    }
  ],
  "mlm_eval_overall_pass_rate": 0.947,
  "mlm_eval_passed_cases": 517,
  "mlm_eval_total_cases": 546
}