File size: 866 Bytes
4ec5e47 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | {
"model": "SpliNet",
"dataset": "allenai/c4 en",
"training_tokens": 2000000000,
"validation_tokens": 5120000,
"tokenizer_sha256": "119ec6b2af9cbbc56f297bd606b69f79f6e3130a34ee56122ee813a58d15bb9d",
"vocab_size": 32000,
"sequence_length": 512,
"layers": 12,
"hidden_size": 768,
"heads": 12,
"ffn_size": 3072,
"spline_order": 2,
"spline_radius": 16,
"spline_sidedness": "single",
"physical_batch": 160,
"effective_batch": 1024,
"optimizer": "AdamW",
"base_learning_rate": 0.0001,
"weight_decay": 0.01,
"precision": "bfloat16",
"gpu": "NVIDIA A100-SXM4-80GB",
"final_validation_loss": 4.6145022583007815,
"final_validation_perplexity": 100.93757518673716,
"best_validation_loss": 4.6145022583007815,
"openreview": "https://openreview.net/forum?id=nWHnuiEF3C",
"github": "https://github.com/AngshulMajumdar/SpliNet"
} |