Upload final model (step 78) and all checkpoints at 2025-07-16T19:21:53.266656

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+step0/models--EleutherAI--pythia-14m/blobs/1d531ed95e6a866521b683d8054fdf83ed660886e5cdf24c83552a009101aa6a filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,7 @@
+# EleutherAI/pythia-14m_distilled_from_pythia-14m Checkpoints
+
+This repository contains the final trained model and intermediate checkpoints.
+
+- The main directory contains the fully trained model (checkpoint 0).
+- The `checkpoints` directory contains all intermediate checkpoints.
+

checkpoints/checkpoint-1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a77182bbaed58df9c3149b1209ef372d1d4d92aa1dcce2ad2e21e0f9db4424
+size 110369580

checkpoints/checkpoint-78.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:773b42e1c4be67f2e6a19749bf3dd991ce65a2c9cd997a5a090009dd85d28c08
+size 110369911

config.json

@@ -0,0 +1,12 @@
+{
+  "architectures": [
+    "HFHookedTransformer"
+  ],
+  "hidden_size": 128,
+  "n_ctx": 2048,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "vocab_size": 50304
+}

data.yaml

@@ -0,0 +1,14 @@
+task: pile10k
+task_config: '{''task'': ''pile10k'', ''dataset_path'': ''timaeus/dsir-pile-10k'',
+  ''output_type'': ''loglikelihood_rolling'', ''training_split'': ''train'', ''test_split'':
+  ''train'', ''validation_split'': None, ''doc_to_target'': ''{{contents}}'', ''doc_to_text'':
+  ''{{contents}}'', ''process_docs'': None, ''process_results'': None, ''should_decontaminate'':
+  True, ''doc_to_decontamination_query'': ''{{page}}'', ''metric_list'': [{''metric'':
+  ''word_perplexity''}, {''metric'': ''byte_perplexity''}, {''metric'': ''bits_per_byte''}],
+  ''metadata'': {''version'': 1.0}, ''dataset_kwargs'': {''trust_remote_code'': True}}'
+dataset_split: training_split
+include_path: shared/aether/config/tasks/
+save_path: .//data/
+force_reload: 'False'
+truncate: auto
+prefix: ''

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.45.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e8bbb7fd722696d4d3e4a3e5472e8db98f4577219cb9db57afa513599b33c33
+size 53797624

model.yaml

@@ -0,0 +1,6 @@
+dtype: torch.bfloat16
+implementation: pythia
+model_name: EleutherAI/pythia-14m
+n_ctx: '2048'
+n_heads: '4'
+n_layers: '6'

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

step0/models--EleutherAI--pythia-14m/blobs/1d531ed95e6a866521b683d8054fdf83ed660886e5cdf24c83552a009101aa6a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d531ed95e6a866521b683d8054fdf83ed660886e5cdf24c83552a009101aa6a
+size 53331721

step0/models--EleutherAI--pythia-14m/blobs/3068f16784f184f2b6a048f1c242a5040fde4916

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.29.2"
+}

step0/models--EleutherAI--pythia-14m/blobs/a7e13441cbf9bd1527e87f7fe1abc961d8a83ded

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.29.2",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

step0/models--EleutherAI--pythia-14m/refs/main

@@ -0,0 +1 @@
+f33025648652797a390d8c54835273845b437161

step0/models--EleutherAI--pythia-14m/refs/step0

@@ -0,0 +1 @@
+5e79bd3bc1ebd3f03db0af32781a176d10237d60

step0/models--EleutherAI--pythia-14m/snapshots/5e79bd3bc1ebd3f03db0af32781a176d10237d60/config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.29.2",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

step0/models--EleutherAI--pythia-14m/snapshots/5e79bd3bc1ebd3f03db0af32781a176d10237d60/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.29.2"
+}

step0/models--EleutherAI--pythia-14m/snapshots/5e79bd3bc1ebd3f03db0af32781a176d10237d60/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d531ed95e6a866521b683d8054fdf83ed660886e5cdf24c83552a009101aa6a
+size 53331721

tokenizer_config.json

@@ -0,0 +1,222 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "<PAD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

training.yaml

@@ -0,0 +1,140 @@
+output_dir: checkpoints/EleutherAI/pythia-14m
+overwrite_output_dir: 'False'
+do_train: 'False'
+do_eval: 'False'
+do_predict: 'False'
+eval_strategy: IntervalStrategy.NO
+prediction_loss_only: 'False'
+per_device_train_batch_size: '8'
+per_device_eval_batch_size: '8'
+per_gpu_train_batch_size: None
+per_gpu_eval_batch_size: None
+gradient_accumulation_steps: '8'
+eval_accumulation_steps: None
+eval_delay: '0'
+torch_empty_cache_steps: None
+learning_rate: '0.001'
+weight_decay: '0.0'
+adam_beta1: '0.9'
+adam_beta2: '0.999'
+adam_epsilon: 1e-08
+max_grad_norm: '1.0'
+num_train_epochs: '3.0'
+max_steps: '-1'
+lr_scheduler_type: SchedulerType.LINEAR
+lr_scheduler_kwargs: '{}'
+warmup_ratio: '0.0'
+warmup_steps: '0'
+log_level: warning
+log_level_replica: warning
+log_on_each_node: 'True'
+logging_dir: checkpoints/EleutherAI/pythia-14m/runs/Jul16_19-19-58_0082549b2b6f
+logging_strategy: IntervalStrategy.STEPS
+logging_first_step: 'True'
+logging_steps: '250'
+logging_nan_inf_filter: 'True'
+save_strategy: IntervalStrategy.STEPS
+save_steps: '300'
+save_total_limit: None
+save_safetensors: 'True'
+save_on_each_node: 'False'
+save_only_model: 'False'
+restore_callback_states_from_checkpoint: 'False'
+no_cuda: 'False'
+use_cpu: 'False'
+use_mps_device: 'False'
+seed: '42'
+data_seed: None
+jit_mode_eval: 'False'
+use_ipex: 'False'
+bf16: 'False'
+fp16: 'False'
+fp16_opt_level: O1
+half_precision_backend: auto
+bf16_full_eval: 'False'
+fp16_full_eval: 'False'
+tf32: None
+local_rank: '0'
+ddp_backend: None
+tpu_num_cores: None
+tpu_metrics_debug: 'False'
+debug: '[]'
+dataloader_drop_last: 'False'
+eval_steps: None
+dataloader_num_workers: '0'
+dataloader_prefetch_factor: None
+past_index: '-1'
+run_name: EleutherAI/pythia-14m_distilled_from_pythia-14m
+disable_tqdm: 'False'
+remove_unused_columns: 'False'
+label_names: '[''input_ids'']'
+load_best_model_at_end: 'False'
+metric_for_best_model: None
+greater_is_better: None
+ignore_data_skip: 'False'
+fsdp: '[]'
+fsdp_min_num_params: '0'
+fsdp_config: '{''min_num_params'': 0, ''xla'': False, ''xla_fsdp_v2'': False, ''xla_fsdp_grad_ckpt'':
+  False}'
+fsdp_transformer_layer_cls_to_wrap: None
+accelerator_config: '{''split_batches'': False, ''dispatch_batches'': None, ''even_batches'':
+  True, ''use_seedable_sampler'': True, ''non_blocking'': False, ''gradient_accumulation_kwargs'':
+  None, ''use_configured_state'': False}'
+deepspeed: None
+label_smoothing_factor: '0.0'
+optim: OptimizerNames.ADAMW_TORCH
+optim_args: None
+adafactor: 'False'
+group_by_length: 'False'
+length_column_name: length
+report_to: '[''wandb'']'
+ddp_find_unused_parameters: None
+ddp_bucket_cap_mb: None
+ddp_broadcast_buffers: None
+dataloader_pin_memory: 'True'
+dataloader_persistent_workers: 'False'
+skip_memory_metrics: 'True'
+use_legacy_prediction_loop: 'False'
+push_to_hub: 'True'
+resume_from_checkpoint: None
+hub_model_id: test-distillation
+hub_strategy: HubStrategy.EVERY_SAVE
+hub_token: None
+hub_private_repo: 'False'
+hub_always_push: 'False'
+gradient_checkpointing: 'False'
+gradient_checkpointing_kwargs: None
+include_inputs_for_metrics: 'False'
+eval_do_concat_batches: 'True'
+fp16_backend: auto
+evaluation_strategy: None
+push_to_hub_model_id: None
+push_to_hub_organization: None
+push_to_hub_token: None
+_n_gpu: '1'
+mp_parameters: ''
+auto_find_batch_size: 'False'
+full_determinism: 'False'
+torchdynamo: None
+ray_scope: last
+ddp_timeout: '1800'
+torch_compile: 'False'
+torch_compile_backend: None
+torch_compile_mode: None
+dispatch_batches: None
+split_batches: None
+include_tokens_per_second: 'False'
+include_num_input_tokens_seen: 'False'
+neftune_noise_alpha: None
+optim_target_modules: None
+batch_eval_metrics: 'False'
+eval_on_start: 'False'
+use_liger_kernel: 'False'
+eval_use_gather_object: 'False'
+checkpoints_dir: .//checkpoints/
+init_step: '0'
+save_log_steps: '250'
+bucket_name: devinterp-language
+s3_folder: checkpoints/tetrahedron-3m
+delete_after_upload: 'False'
+push_to_aws: 'False'