train_args.json

{
  "data": "fineweb_edu_200M_v1024.uint16.bin",
  "base": "LowOnMind-1M",
  "derived_from": "DedeProGames/LowOnMind-300k",
  "dataset": "HuggingFaceFW/fineweb-edu:sample-10BT",
  "tokenizer": "byte-level BPE 1024",
  "params": 985152,
  "seq_len": 512,
  "batch_size": 64,
  "grad_accum": 1,
  "max_steps": 6103,
  "total_tokens": 199983104,
  "lr": 0.0015,
  "min_lr": 0.00015,
  "warmup_steps": 250,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "val_tokens": 2000000,
  "eval_every": 500,
  "eval_batches": 40,
  "device": "cuda",
  "dtype": "float16",
  "compile": true,
  "seed": 1337,
  "train_minutes": 10.6,
  "final_val_loss": 2.9908,
  "final_val_ppl": 19.901,
  "bits_per_char": 1.8361,
  "chars_per_token": 2.35,
  "reused_tokenizer_from": "DedeProGames/LowOnMind-300k",
  "baseline": {
    "name": "LowOnMind-300k",
    "params": 296960,
    "val_loss": 3.2982,
    "val_ppl": 27.06,
    "bpc": 2.03,
    "bench_elo": 833,
    "bench_acc": 0.266
  },
  "lexicon": {
    "model_real_word_rate": 0.9798,
    "corpus_real_word_rate": 0.9839,
    "words_scored": 5647,
    "samples": 64,
    "top_nonwords": [
      [
        "sculieness",
        1
      ],
      [
        "lockholm",
        1
      ],
      [
        "scul",
        1
      ],
      [
        "purpled",
        1
      ],
      [
        "paradigmar",
        1
      ],
      [
        "prefection",
        1
      ],
      [
        "frushing",
        1
      ],
      [
        "fullly",
        1
      ],
      [
        "laxation",
        1
      ],
      [
        "gulliel",
        1
      ],
      [
        "annal",
        1
      ],
      [
        "backg",
        1
      ],
      [
        "warehorship",
        1
      ],
      [
        "bulken",
        1
      ],
      [
        "colle",
        1
      ]
    ]
  },
  "samples": [
    {
      "prompt": "The ",
      "text": "The veil-group, where is used to solve solar passwords and vegetables. It is often very important to reduce their treatments, including checkouts, labs, maintenance, mental groups, chemicals and maintenance costs. These literally confirmed elements are the most commonly used in the elementary front, solar energy, a"
    },
    {
      "prompt": "Photosynthesis is ",
      "text": "Photosynthesis is essential to prevent disease infection. These types of disease include:\n\u2022 Transformation of Brook Scientific World Organization\n\u2022 Effectiveness of a diagnosis is commonly diagnosed with diagnosis. The Cultural Disorder is an important factor in diagnosis. Analysis of this purpose can be considered."
    },
    {
      "prompt": "In 1969, ",
      "text": "In 1969, 8, 1999, Michael Kerzin, Anderson, 1997, Scotland, 1993, 1,359, p. 547\u2013414\nOur war was chosen in the West of Europe, Papua Party, Ottoman Economy, Pluto Mountains, and Evans, 1854, 1946, Maine, 1888-81, p"
    },
    {
      "prompt": "Students should ",
      "text": "Students should identify their learners and adults.\nThe United States has free quotations in the world that can help them with this curriculum and finding problem concerning these funding.\nOne major solutions for malnutrition in South East and Africa is a way to limit half its own loop. This is the best way to ensure that the drills of the state and"
    }
  ],
  "history": [
    {
      "step": 500,
      "val_loss": 3.6402,
      "val_ppl": 38.099
    },
    {
      "step": 1000,
      "val_loss": 3.3854,
      "val_ppl": 29.528
    },
    {
      "step": 1500,
      "val_loss": 3.2837,
      "val_ppl": 26.673
    },
    {
      "step": 2000,
      "val_loss": 3.2305,
      "val_ppl": 25.292
    },
    {
      "step": 2500,
      "val_loss": 3.187,
      "val_ppl": 24.216
    },
    {
      "step": 3000,
      "val_loss": 3.1488,
      "val_ppl": 23.308
    },
    {
      "step": 3500,
      "val_loss": 3.118,
      "val_ppl": 22.602
    },
    {
      "step": 4000,
      "val_loss": 3.0804,
      "val_ppl": 21.766
    },
    {
      "step": 4500,
      "val_loss": 3.0496,
      "val_ppl": 21.107
    },
    {
      "step": 5000,
      "val_loss": 3.0375,
      "val_ppl": 20.853
    },
    {
      "step": 5500,
      "val_loss": 3.0119,
      "val_ppl": 20.326
    },
    {
      "step": 6000,
      "val_loss": 3.0165,
      "val_ppl": 20.42
    },
    {
      "step": 6103,
      "val_loss": 3.0055,
      "val_ppl": 20.195
    }
  ]
}