training_config.json

{
  "run_name": "JL-Code-Python-97M-v0.1-base",
  "status": "completed",
  "model_repository": "jumplander/JL-Code-Python-97M",
  "objective": "causal-language-modeling",
  "training_from_scratch": true,
  "base_model": null,
  "model_parameters": 97536768,
  "total_optimizer_steps": 10000,
  "micro_batch_size": 1,
  "gradient_accumulation_steps": 32,
  "effective_batch_size_sequences": 32,
  "sequence_length": 1024,
  "tokens_per_optimizer_step": 32768,
  "estimated_tokens_processed": 327680000,
  "optimizer": {
    "name": "AdamW",
    "learning_rate": 0.0003,
    "minimum_learning_rate": 3e-05,
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "weight_decay": 0.1,
    "fused_when_available": true
  },
  "scheduler": {
    "name": "cosine-decay",
    "warmup_steps": 200
  },
  "gradient_clipping": {
    "enabled": true,
    "max_norm": 1.0
  },
  "precision": {
    "device": "cuda",
    "dtype": "bfloat16",
    "automatic_mixed_precision": true,
    "gradient_scaler_used_for_fp16_only": true
  },
  "checkpointing": {
    "save_interval_steps": 500,
    "format_version": 1,
    "includes_optimizer_state": true,
    "checkpoint_file": "jumplander_python_100m.pt"
  },
  "evaluation": {
    "validation_interval_steps": 250,
    "validation_batches": 20,
    "benchmark_results_published": false
  },
  "logging": {
    "log_interval_steps": 10
  },
  "random_seed": 1337,
  "num_workers": 0,
  "hardware": {
    "gpu": "NVIDIA GeForce RTX 3060",
    "gpu_memory_gb": 12,
    "observed_peak_allocated_vram_gb_approx": 1.86,
    "observed_tokens_per_second_range_approx": [
      8400,
      9000
    ]
  },
  "tokenizer": {
    "type": "Byte-Level BPE",
    "target_vocab_size": 16384,
    "minimum_frequency": 2,
    "byte_fallback": true
  },
  "data": {
    "sources": [
      {
        "id": "codeparrot/codeparrot-clean",
        "role": "python-source-pretraining"
      },
      {
        "id": "google-research-datasets/mbpp",
        "role": "english-to-python-and-held-out-tasks"
      }
    ],
    "accepted_base_samples": 20000,
    "train_rows": 35855,
    "validation_rows": 389,
    "test_rows": 793,
    "source_records_read": 10809,
    "rejected_exact_duplicates": 391,
    "rejected_generated_files": 246,
    "rejected_secret_like_samples": 75,
    "rejected_syntax_invalid_samples": 1523,
    "rejected_oversized_samples": 268,
    "fill_in_the_middle_variants": true,
    "python_ast_validation": true,
    "sha256_exact_deduplication": true
  }
}