train_args.json

{
  "data": "fineweb_edu_200M_v1024.uint16.bin",
  "base": "LowOnMind-5M",
  "derived_from": "DedeProGames/LowOnMind-300k, DedeProGames/LowOnMind-1M",
  "dataset": "HuggingFaceFW/fineweb-edu:sample-10BT",
  "tokenizer": "byte-level BPE 1024",
  "params": 4920384,
  "seq_len": 512,
  "batch_size": 64,
  "grad_accum": 1,
  "max_steps": 6103,
  "total_tokens": 199983104,
  "lr": 0.0012,
  "min_lr": 0.00012,
  "warmup_steps": 250,
  "weight_decay": 0.1,
  "grad_clip": 1.0,
  "val_tokens": 2000000,
  "eval_every": 500,
  "eval_batches": 40,
  "device": "cuda",
  "dtype": "float16",
  "compile": true,
  "seed": 1337,
  "train_minutes": 27.0,
  "final_val_loss": 2.5828,
  "final_val_ppl": 13.235,
  "bits_per_char": 1.5856,
  "chars_per_token": 2.35,
  "reused_tokenizer_from": "DedeProGames/LowOnMind-300k",
  "embedding_share": 0.04,
  "baseline_300k": {
    "name": "LowOnMind-300k",
    "params": 296960,
    "val_loss": 3.2982,
    "val_ppl": 27.06,
    "bpc": 2.03,
    "bench_elo": 833,
    "bench_acc": 0.266,
    "bench_lang": 0.46,
    "real_word": null
  },
  "baseline_1m": {
    "name": "LowOnMind-1M",
    "params": 985152,
    "val_loss": 2.9908,
    "val_ppl": 19.9,
    "bpc": 1.836,
    "bench_elo": 843,
    "bench_acc": 0.289,
    "bench_lang": 0.52,
    "real_word": 0.98
  },
  "lexicon": {
    "model_real_word_rate": 0.963,
    "corpus_real_word_rate": 0.9839,
    "words_scored": 5398,
    "samples": 64,
    "top_nonwords": [
      [
        "shima",
        13
      ],
      [
        "meikimo",
        11
      ],
      [
        "amiravicis",
        8
      ],
      [
        "illuminator",
        3
      ],
      [
        "hymenola",
        3
      ],
      [
        "phillipsburg",
        3
      ],
      [
        "perleti",
        3
      ],
      [
        "almanine",
        2
      ],
      [
        "morrisky",
        2
      ],
      [
        "anthocyanin",
        2
      ],
      [
        "shekarwons",
        2
      ],
      [
        "ziko",
        1
      ],
      [
        "basiles",
        1
      ],
      [
        "gooverton",
        1
      ],
      [
        "mcleil",
        1
      ]
    ]
  },
  "samples": [
    {
      "prompt": "The ",
      "text": "The Xylococcus cannot sound alterarian and apnea.\nSimilarly, a bacteria may be calcium, and the Xylococcus cannot be cut into the mouth. This cannot be reduced and the muscles can\u2019t be treated by slowing the mouth. Bacteria can also cause problems in the vomiting and sleeping of the bact"
    },
    {
      "prompt": "Photosynthesis is ",
      "text": "Photosynthesis is essential to the human tissues. At the end of the vision, the body can act as a method that makes it easier to understand the brain. What is the brain, it can be the first to use the brain to perform a brain function, but it is important to continue to live. For example, in addition to the brain cells that receive the brain, we have to"
    },
    {
      "prompt": "In 1969, ",
      "text": "In 1969, 86 percent of the people from the Palestine candidates had received a higher rate of jobs, and they were brought to the Civil War.\nPalestinian candidates initiated a free financial and copyright against the War of the Region. The country was the second century bankers, now bankers, and money, within which they were"
    },
    {
      "prompt": "Students should ",
      "text": "Students should identify their strengths, and gain control of their bodies.\nThe results of that study were introduced in the BMCP (September 2018). BMCP (September 2015) and the researchers identified the benefits of drugs in humans, and these studies have been released to the study. The results suggested that drugs that were strong"
    }
  ],
  "history": [
    {
      "step": 500,
      "val_loss": 3.3853,
      "val_ppl": 29.528
    },
    {
      "step": 1000,
      "val_loss": 3.1327,
      "val_ppl": 22.936
    },
    {
      "step": 1500,
      "val_loss": 2.9676,
      "val_ppl": 19.446
    },
    {
      "step": 2000,
      "val_loss": 2.8651,
      "val_ppl": 17.55
    },
    {
      "step": 2500,
      "val_loss": 2.7969,
      "val_ppl": 16.393
    },
    {
      "step": 3000,
      "val_loss": 2.7501,
      "val_ppl": 15.644
    },
    {
      "step": 3500,
      "val_loss": 2.7143,
      "val_ppl": 15.093
    },
    {
      "step": 4000,
      "val_loss": 2.6789,
      "val_ppl": 14.569
    },
    {
      "step": 4500,
      "val_loss": 2.6498,
      "val_ppl": 14.151
    },
    {
      "step": 5000,
      "val_loss": 2.6273,
      "val_ppl": 13.837
    },
    {
      "step": 5500,
      "val_loss": 2.6074,
      "val_ppl": 13.564
    },
    {
      "step": 6000,
      "val_loss": 2.6067,
      "val_ppl": 13.554
    },
    {
      "step": 6103,
      "val_loss": 2.6016,
      "val_ppl": 13.486
    }
  ]
}