config.json

{
  "_name_or_path": "jumplander/JL-Code-Python-97M",
  "model_name": "JL-Code-Python-97M",
  "model_family": "JL-Code",
  "release_version": "0.1-base",
  "model_type": "jl_code_python",
  "architectures": [
    "JumpLanderPythonModel"
  ],
  "framework": "pytorch",
  "task": "causal-language-modeling",
  "domain": "python",
  "natural_language": [
    "en"
  ],
  "parameter_count": 97536768,
  "trainable_parameter_count": 97536768,
  "training_from_scratch": true,
  "pretrained_checkpoint_used": false,
  "requires_custom_code": true,
  "transformers_compatible": false,
  "checkpoint_file": "jumplander_python_100m.pt",
  "checkpoint_format": "custom-pytorch-training-checkpoint-v1",
  "source_file": "jumplander_python_100m.py",
  "tokenizer_file": "tokenizer.json",
  "vocab_size": 16384,
  "max_position_embeddings": 1024,
  "max_seq_len": 1024,
  "hidden_size": 768,
  "intermediate_size": 2048,
  "num_hidden_layers": 12,
  "num_attention_heads": 12,
  "head_dim": 64,
  "rope_theta": 10000.0,
  "position_embedding_type": "rope",
  "rms_norm_eps": 1e-05,
  "normalization": "rmsnorm",
  "hidden_act": "silu",
  "mlp_type": "swiglu",
  "attention_type": "causal-self-attention",
  "attention_implementation": "torch-scaled-dot-product-attention",
  "attention_bias": false,
  "mlp_bias": false,
  "tie_word_embeddings": true,
  "initializer_range": 0.02,
  "dropout": 0.0,
  "gradient_checkpointing": true,
  "torch_dtype": "bfloat16",
  "tokenizer_type": "byte-level-bpe",
  "tokenizer_byte_fallback": true,
  "special_tokens": [
    "<pad>",
    "<unk>",
    "<bos>",
    "<eos>",
    "<file_start>",
    "<file_end>",
    "<fim_prefix>",
    "<fim_suffix>",
    "<fim_middle>",
    "<instruction>",
    "<response>"
  ]
}