{
  "architectures": [
    "KsByteForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_ksbyte.KsByteConfig",
    "AutoModelForCausalLM": "modeling_ksbyte.KsByteForCausalLM",
    "AutoTokenizer": [
      "tokenization_ksbyte.KsByteTokenizer",
      null
    ]
  },
  "batch_size": 16,
  "best_val_bpb": 0.9592911933406559,
  "beta1": 0.9,
  "beta2": 0.95,
  "bf16": true,
  "bos_token_id": 256,
  "byte_vocab": 256,
  "ctx_len": 2048,
  "d_model": 384,
  "data_dir": "/cache/data",
  "data_meta": {
    "bos_id": 256,
    "bytes_per_word": 9.865595756933395,
    "counts": {
      "test": 3074698,
      "train": 45362173,
      "val": 1622371
    },
    "dtype": "uint16",
    "eos_id": 257,
    "normalization": {
      "digit_policy": "keep_native",
      "remove_diacritics": false,
      "zwnj_policy": "keep"
    },
    "source": "Omarrran/KS-PRET-5M_5_million_kashmiri_Pretrainning_LLM_dataset_12M_tokens_2026",
    "stats": {
      "content_bytes": 50058684,
      "dropped_dup": 0,
      "dropped_ksr": 0,
      "dropped_short": 0,
      "kept": 279,
      "raw": 279,
      "words": 5074066
    },
    "vocab_size": 259
  },
  "dedup": true,
  "device": "auto",
  "digit_policy": "keep_native",
  "doc_attention_mask": true,
  "dropout": 0.1,
  "dtype": "float32",
  "early_stop_patience": 6,
  "eos_token_id": 257,
  "epochs": 4.0,
  "eval_interval": 250,
  "eval_iters": 100,
  "generate_every": 1000,
  "generate_tokens": 160,
  "grad_accum": 4,
  "grad_clip": 1.0,
  "hf_dataset": "Omarrran/KS-PRET-5M_5_million_kashmiri_Pretrainning_LLM_dataset_12M_tokens_2026",
  "hf_revision": null,
  "keep_mixed_script": true,
  "label_smoothing": 0.0,
  "local_text_file": null,
  "log_interval": 20,
  "lr": 0.0004,
  "max_chars": 100000,
  "max_patches": 320,
  "max_steps": 5000,
  "min_chars": 2,
  "min_ks_ratio": 0.9,
  "min_lr_ratio": 0.1,
  "mlp_ratio": 2.6667,
  "model_type": "ksbyte",
  "n_global": 6,
  "n_heads": 6,
  "n_kv_heads": 2,
  "n_layers": 6,
  "n_local_in": 2,
  "n_local_out": 2,
  "num_workers": 4,
  "output_dir": "/outputs",
  "pad_token_id": 258,
  "qk_norm": true,
  "rebuild_data": false,
  "remove_diacritics": false,
  "resume": "auto",
  "rope_theta": 10000.0,
  "run_name": "ksbyte-spacebyte-full",
  "save_interval": 100,
  "save_total_limit": 3,
  "seed": 42,
  "source_checkpoint": "best.pt",
  "split_seed": 13,
  "test_frac": 0.05,
  "text_col": "auto",
  "tf32": true,
  "tie_embeddings": true,
  "torch_compile": false,
  "training_step": 3250,
  "transformers_compatible_revision": "v1",
  "transformers_version": "4.57.6",
  "use_cache": false,
  "use_wandb": false,
  "val_frac": 0.05,
  "variant": "spacebyte",
  "vocab_size": 259,
  "wandb_project": "ks-byte-lm",
  "warmup_ratio": 0.03,
  "weight_decay": 0.1,
  "z_loss_weight": 0.0001,
  "zwnj_policy": "keep",
  "library_name": "transformers"
}