Hugging Face's logo

jumplander
/
JL-Code-Python-97M

Text Generation
PyTorch
English
jl_code_python
code
python
python-code
code-generation
code-completion
causal-language-modeling
decoder-only
programming-language-model
small-language-model
from-scratch
random-initialization
custom-architecture
custom-pytorch
byte-level-bpe
rope
rmsnorm
swiglu
consumer-gpu
rtx-3060
research
jumplander
jl-code
Model card Files
xet
 Community
JL-Code-Python-97M
File size: 2,507 Bytes
1fcab03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
{
  "run_name": "JL-Code-Python-97M-v0.1-base",
  "status": "completed",
  "model_repository": "jumplander/JL-Code-Python-97M",
  "objective": "causal-language-modeling",
  "training_from_scratch": true,
  "base_model": null,
  "model_parameters": 97536768,
  "total_optimizer_steps": 10000,
  "micro_batch_size": 1,
  "gradient_accumulation_steps": 32,
  "effective_batch_size_sequences": 32,
  "sequence_length": 1024,
  "tokens_per_optimizer_step": 32768,
  "estimated_tokens_processed": 327680000,
  "optimizer": {
    "name": "AdamW",
    "learning_rate": 0.0003,
    "minimum_learning_rate": 3e-05,
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "weight_decay": 0.1,
    "fused_when_available": true
  },
  "scheduler": {
    "name": "cosine-decay",
    "warmup_steps": 200
  },
  "gradient_clipping": {
    "enabled": true,
    "max_norm": 1.0
  },
  "precision": {
    "device": "cuda",
    "dtype": "bfloat16",
    "automatic_mixed_precision": true,
    "gradient_scaler_used_for_fp16_only": true
  },
  "checkpointing": {
    "save_interval_steps": 500,
    "format_version": 1,
    "includes_optimizer_state": true,
    "checkpoint_file": "jumplander_python_100m.pt"
  },
  "evaluation": {
    "validation_interval_steps": 250,
    "validation_batches": 20,
    "benchmark_results_published": false
  },
  "logging": {
    "log_interval_steps": 10
  },
  "random_seed": 1337,
  "num_workers": 0,
  "hardware": {
    "gpu": "NVIDIA GeForce RTX 3060",
    "gpu_memory_gb": 12,
    "observed_peak_allocated_vram_gb_approx": 1.86,
    "observed_tokens_per_second_range_approx": [
      8400,
      9000
    ]
  },
  "tokenizer": {
    "type": "Byte-Level BPE",
    "target_vocab_size": 16384,
    "minimum_frequency": 2,
    "byte_fallback": true
  },
  "data": {
    "sources": [
      {
        "id": "codeparrot/codeparrot-clean",
        "role": "python-source-pretraining"
      },
      {
        "id": "google-research-datasets/mbpp",
        "role": "english-to-python-and-held-out-tasks"
      }
    ],
    "accepted_base_samples": 20000,
    "train_rows": 35855,
    "validation_rows": 389,
    "test_rows": 793,
    "source_records_read": 10809,
    "rejected_exact_duplicates": 391,
    "rejected_generated_files": 246,
    "rejected_secret_like_samples": 75,
    "rejected_syntax_invalid_samples": 1523,
    "rejected_oversized_samples": 268,
    "fill_in_the_middle_variants": true,
    "python_ast_validation": true,
    "sha256_exact_deduplication": true
  }
}