Upload folder using huggingface_hub

.gitattributes

@@ -26,9 +26,6 @@ checkpoints/kd_step_50.pt filter=lfs diff=lfs merge=lfs -text
 checkpoints/kd_step_1000.pt filter=lfs diff=lfs merge=lfs -text
 checkpoints/kd_step_5000.pt filter=lfs diff=lfs merge=lfs -text
 checkpoints/kd_step_10000.pt filter=lfs diff=lfs merge=lfs -text
-checkpoints_kd_gptj_9L/kd_step_1000.pt filter=lfs diff=lfs merge=lfs -text
-checkpoints_kd_gptj_9L/kd_step_2000.pt filter=lfs diff=lfs merge=lfs -text
-checkpoints_kd_gptj_9L/kd_step_3000.pt filter=lfs diff=lfs merge=lfs -text
 checkpoints_kd_gptj_9L/kd_step_4000.pt filter=lfs diff=lfs merge=lfs -text
 checkpoints_kd_neo27/kd_step_11000.pt filter=lfs diff=lfs merge=lfs -text
-kd11k_9L.pt filter=lfs diff=lfs merge=lfs -text
+checkpoints_kd_neo27_9L/kd_step_14000.pt filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/eval_dist_11000-checkpoint.json

@@ -0,0 +1,7 @@
+{
+  "perplexity": 73.67675779080741,
+  "avg_loss_nats": 4.299687387053869,
+  "total_tokens_evaluated": 2613248,
+  "total_params": 92527232,
+  "eval_time_sec": 66.57
+}

checkpoints_kd_gptj_9L/kd_step_2000.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ce2784e6aa62da49c5f6de11c0aeb2972931bc88331d1a196b72bebef045143
-size 393895052

checkpoints_kd_gptj_9L/kd_step_3000.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7b0bed1ba6f8c919d4b4946274c13ed897d44df1a1aa3fa61e987bac37ed4db6
-size 393895052

checkpoints_kd_gptj_9L/kd_step_1000.pt → checkpoints_kd_neo27_9L/kd_step_14000.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45451b6e98ccb12d8f8c37977beb1ac52467eb47c1981330ca1e5fc60db87726
-size 393895052
+oid sha256:42079ddb1fa4628b3d571d5769efcf7992901e6bb146c29d55bb0cb9ec6ea0b2
+size 393895159

eval_dist_neo_11000.json

@@ -0,0 +1,7 @@
+{
+  "perplexity": 73.01793707231893,
+  "avg_loss_nats": 4.290705124285255,
+  "total_tokens_evaluated": 5169152,
+  "total_params": 92527232,
+  "eval_time_sec": 133.63
+}

eval_dist_neo_9L_1000.json

@@ -0,0 +1,7 @@
+{
+  "perplexity": 73.76881593399138,
+  "avg_loss_nats": 4.300936093987829,
+  "total_tokens_evaluated": 5169152,
+  "total_params": 98464240,
+  "eval_time_sec": 143.52
+}

kd11k_9L.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:79f1475cc5f848de7808318189853cddf5d10f346e63d3c185a8c4cfcc9a7a2f
-size 393893280

train_eval_cmd.txt

@@ -25,13 +25,15 @@ python train.py --config optuna_results/best_dense_config.yaml \
 
 python3 evaluate.py \
   --model_dir . \
-  --checkpoint_filename ./checkpoints_kd_gptj_9L/kd_step_2000.pt \
+  --checkpoint_filename ./checkpoints_kd_neo27_9L/kd_step_3000.pt \
   --data val.bin \
-  --output_json eval_dist_gptj_9L_2000.json
+  --output_json eval_dist_neo_9L_3000.json
 
 -----
 
-LATEST=$(ls -t ./checkpoints_kd_neo27/kd_step_*.pt | head -1)
+# distill into 8L
+
+LATEST=$(ls -t ./checkpoints_kd_neo27_9L/kd_step_*.pt | head -1)
 echo "Resuming from $LATEST"
 
 export NUMEXPR_MAX_THREADS=64
@@ -47,11 +49,20 @@ python distill.py \
   --save_every 1000 --epochs 1 \
   --save_dir ./checkpoints_kd_neo27
 
-python distill.py --student_checkpoint ./kd11k_9L.pt \
+# distill into 9L
+
+LATEST=$(ls -t ./checkpoints_kd_neo27_9L/kd_step_*.pt | head -1)
+echo "Resuming from $LATEST"
+
+export NUMEXPR_MAX_THREADS=64
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python distill.py --student_checkpoint "$LATEST" \
     --kd_mode logit --teacher EleutherAI/gpt-neo-2.7B \
     --dataset HuggingFaceFW/fineweb --dataset_name sample-10BT --streaming \
     --max_train_samples 300000 --lr 2e-5 --kd_alpha 0.3 --kd_temperature 1.0 \
     --batch_size 6 --accumulation_steps 1 --num_workers 2 \
+    --save_every 1000 --epochs 1 \
     --save_dir ./checkpoints_kd_neo27_9L
 
 -----