training_metadata.json

{
  "model": "SpliNet",
  "dataset": "allenai/c4 en",
  "training_tokens": 2000000000,
  "validation_tokens": 5120000,
  "tokenizer_sha256": "119ec6b2af9cbbc56f297bd606b69f79f6e3130a34ee56122ee813a58d15bb9d",
  "vocab_size": 32000,
  "sequence_length": 512,
  "layers": 12,
  "hidden_size": 768,
  "heads": 12,
  "ffn_size": 3072,
  "spline_order": 2,
  "spline_radius": 16,
  "spline_sidedness": "single",
  "physical_batch": 160,
  "effective_batch": 1024,
  "optimizer": "AdamW",
  "base_learning_rate": 0.0001,
  "weight_decay": 0.01,
  "precision": "bfloat16",
  "gpu": "NVIDIA A100-SXM4-80GB",
  "final_validation_loss": 4.6145022583007815,
  "final_validation_perplexity": 100.93757518673716,
  "best_validation_loss": 4.6145022583007815,
  "openreview": "https://openreview.net/forum?id=nWHnuiEF3C",
  "github": "https://github.com/AngshulMajumdar/SpliNet"
}