pretrain_config.json

[
  {
    "model_name": "MikeGreen2710/mlm_listing_checkpoint-32442",
    "tokenizer_name": "VinAI/phobert-base",
    "whole_word_masking": true,
    "mlm_probability": 0.15,
    "max_length": 256,
    "batch_size": 16,
    "gradient_accumulation_steps": 1,
    "learning_rate": 3e-05,
    "warmup_steps": 300,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "max_steps": 30000,
    "eval_steps": 250,
    "early_stopping_patience": 5,
    "freeze_layers": 0,
    "use_bf16": true,
    "use": false,
    "note": "Already domain-adapted PhoBERT. Lighter run \u2014 it just needs a top-up on your specific listing corpus.",
    "pretrained": true,
    "pretrained_at": "2026-05-13 06:20:17",
    "perplexity": 1.9174,
    "output_dir": "pretrained_checkpoints/MikeGreen2710__mlm_listing_checkpoint-32442"
  },
  {
    "model_name": "Fsoft-AIC/videberta-base",
    "tokenizer_name": "Fsoft-AIC/videberta-base",
    "drop_token_type_ids": true,
    "whole_word_masking": false,
    "mlm_probability": 0.15,
    "max_length": 256,
    "batch_size": 16,
    "gradient_accumulation_steps": 1,
    "learning_rate": 2e-05,
    "warmup_steps": 500,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "max_steps": 500000,
    "eval_steps": 500,
    "early_stopping_patience": 8,
    "freeze_layers": 0,
    "use_bf16": true,
    "use": false,
    "note": "",
    "pretrained": true,
    "pretrained_at": "2026-05-12 21:14:14",
    "perplexity": 2.6166,
    "output_dir": "pretrained_checkpoints/Fsoft-AIC__videberta-base"
  },
  {
    "model_name": "microsoft/mdeberta-v3-base",
    "tokenizer_name": "microsoft/mdeberta-v3-base",
    "drop_token_type_ids": true,
    "whole_word_masking": false,
    "mlm_probability": 0.15,
    "max_length": 256,
    "batch_size": 16,
    "gradient_accumulation_steps": 1,
    "learning_rate": 2e-05,
    "warmup_steps": 500,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "max_steps": 500000,
    "eval_steps": 500,
    "early_stopping_patience": 8,
    "freeze_layers": 0,
    "use_bf16": true,
    "use": false,
    "note": "Multilingual DeBERTa. drop_token_type_ids required. WWM off \u2014 SentencePiece tokeniser doesn't carry word_ids needed for WWM.",
    "pretrained": true,
    "pretrained_at": "2026-05-12 21:14:14",
    "perplexity": 2.6166,
    "output_dir": "pretrained_checkpoints/microsoft__mdeberta-v3-base"
  },
  {
    "model_name": "NlpHUST/electra-base-vn",
    "tokenizer_name": "NlpHUST/electra-base-vn",
    "whole_word_masking": false,
    "mlm_probability": 0.15,
    "max_length": 256,
    "batch_size": 16,
    "gradient_accumulation_steps": 1,
    "learning_rate": 3e-05,
    "warmup_steps": 400,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "max_steps": 15000,
    "eval_steps": 5000,
    "early_stopping_patience": 8,
    "freeze_layers": 0,
    "use_bf16": true,
    "use": false,
    "note": "ELECTRA architecture \u2014 true DAPT requires a generator+discriminator setup. Using MLM pragmatically here: it domain-adapts the encoder without needing the full RTD machinery.",
    "pretrained": true,
    "pretrained_at": "2026-05-12 21:54:54",
    "perplexity": 2.3859,
    "output_dir": "pretrained_checkpoints/NlpHUST__electra-base-vn"
  },
  {
    "model_name": "google/rembert",
    "tokenizer_name": "google/rembert",
    "whole_word_masking": false,
    "mlm_probability": 0.15,
    "max_length": 256,
    "batch_size": 4,
    "gradient_accumulation_steps": 4,
    "learning_rate": 1e-05,
    "warmup_steps": 800,
    "weight_decay": 0.01,
    "lr_scheduler_type": "cosine",
    "max_steps": 20000,
    "eval_steps": 500,
    "early_stopping_patience": 8,
    "freeze_layers": 6,
    "use_bf16": true,
    "use": true,
    "note": "RemBERT is large (~560M params). Smaller effective batch (batch_size=4 * grad_accum=4 = 16), lower LR, and bottom 6 layers frozen to keep memory manageable and training stable.",
    "pretrained": true,
    "pretrained_at": "2026-05-15 16:06:34",
    "perplexity": 1.9685,
    "output_dir": "pretrained_checkpoints/google__rembert"
  }
]