Overwrite adapter with checkpoint-125 (r64 epoch-1)

README.md

@@ -0,0 +1,55 @@
+---
+base_model: google/gemma-3-27b-it
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- lora
+- peft
+- gemma
+- entropy
+---
+
+# Entropy LoRA (Gemma 3 27B IT) - Updated Adapter
+
+This repository contains a PEFT LoRA adapter for `google/gemma-3-27b-it`.
+
+This upload supersedes the previous `entropy-v1` adapter with the **epoch-1 checkpoint** from the on-prem PEFT run (`r=64`).
+
+## vLLM (runtime LoRA)
+
+Important: this adapter is **rank 64**, so vLLM must be started with `--max-lora-rank 64` (or higher).
+
+Example:
+
+```bash
+vllm serve google/gemma-3-27b-it \
+  --served-model-name google/gemma-3-27b-it \
+  --enable-lora \
+  --max-lora-rank 64 \
+  --lora-modules entropy-v1=ysong21/entropy-v1-lora
+```
+
+## Transformers + PEFT
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+
+base = "google/gemma-3-27b-it"
+adapter = "ysong21/entropy-v1-lora"
+
+tok = AutoTokenizer.from_pretrained(base)
+model = AutoModelForCausalLM.from_pretrained(base, device_map="auto")
+model = PeftModel.from_pretrained(model, adapter)
+```
+
+## Offline Eval (held-out)
+
+Validation set: `data/validation.no_overlap.jsonl` (70 examples).
+
+- Base `google/gemma-3-27b-it`: `bits_per_char=0.99565`
+- Previous adapter (`ysong21/entropy-v1-lora`, old): `bits_per_char=0.36646`
+- This adapter (epoch-1, `r=64`): `bits_per_char=0.35877`
+- Baseline: `N8Programs/Unslopper-30B-A3B-bf16`: `bits_per_char=0.37522`
+
+Note: token-based `ppl_cond` is not directly comparable across tokenizers/models; we rely on char-normalized `bits_per_char` for cross-model comparisons.

adapter_config.json

@@ -1,39 +1,46 @@
 {
+  "alora_invocation_tokens": null,
   "alpha_pattern": {},
+  "arrow_config": null,
   "auto_mapping": null,
-  "base_model_name_or_path": null,
+  "base_model_name_or_path": "google/gemma-3-27b-it",
   "bias": "none",
   "corda_config": null,
+  "ensure_weight_tying": false,
   "eva_config": null,
   "exclude_modules": null,
   "fan_in_fan_out": false,
-  "inference_mode": false,
+  "inference_mode": true,
   "init_lora_weights": true,
   "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
+  "lora_alpha": 128,
   "lora_bias": false,
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
     "up_proj",
-    "q_proj",
+    "k_proj",
+    "down_proj",
     "gate_proj",
     "o_proj",
-    "v_proj",
-    "k_proj",
-    "down_proj"
+    "q_proj",
+    "v_proj"
   ],
+  "target_parameters": null,
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,
   "use_dora": false,
+  "use_qalora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e0e196aa158d3a11ef4b0b023921fae6410a0eedf6d74a5b6ff10c5d4dff1d8
-size 454197288
+oid sha256:ba6ec6b5878095233f6c2c21b1429aa46469ca968a20a6086aef9400048c878e
+size 1864199752

stats.json

@@ -1 +1,11 @@
-{"world_size": 4, "epochs": 1, "steps": 94, "seqs": 999, "tokens": 2906680, "last_epoch_steps": 0, "last_epoch_seqs": 0, "last_epoch_tokens": 0, "total_seqs": 999, "nan_in_loss_seqs": 0, "experiment_tracking_run_id": null, "loss_ema": 1.6103686253345062, "loss_sum": 33.14478254318237, "mtp_loss_ema": 0, "mtp_loss_sum": 0, "distillation_loss_ema": 0, "distillation_loss_sum": 0, "hard_loss_ema": 0, "hard_loss_sum": 0, "eval_losses_avg": []}
+{
+  "base_model": "google/gemma-3-27b-it",
+  "adapter_source": "data/entropy-v2-lora-r64-e5/checkpoint-125",
+  "eval": {
+    "data": "data/validation.no_overlap.jsonl",
+    "examples": 70,
+    "bits_per_char": 0.35876985070947004,
+    "ppl_cond": 3.075335556196064,
+    "loss_mean": 1.123414019271094
+  }
+}

train_config.json

@@ -1,61 +1,35 @@
 {
-  "comet": false,
-  "comet_api_key": null,
-  "comet_workspace": null,
-  "comet_project": null,
-  "comet_run_id": "v6vlpnyh",
-  "wandb": true,
-  "wandb_entity": "maxsong-carnegie-mellon-university",
-  "wandb_project": "entropy",
-  "wandb_run_id": "v6vlpnyh",
-  "base_model_dir": "/llm-downloader-destination/base/fireworks/gemma-3-27b-it/hf",
-  "output_model_dir": "gs://fireworks-artifacts-maxx1999syp-bybv7vrv-254f13/tuned-model-v6vlpnyh/5a2aa8/gemma-3-27b-entropy-02082026/checkpoint",
-  "checkpoint_dir": "/dev/shm/checkpoints",
-  "gcs_checkpoint_dir": "gs://fireworks-artifacts-maxx1999syp-bybv7vrv-254f13/tuned-model-v6vlpnyh/5a2aa8/gemma-3-27b-entropy-02082026/checkpoints/checkpoints",
-  "max_checkpoints_to_keep": 1,
-  "checkpoint_interval": 3600,
-  "save_final_checkpoint": false,
-  "train": true,
-  "learning_rate": 0.0002,
-  "learning_rate_warmup_steps": 0,
-  "grad_accum_steps": 1,
-  "epochs": 1,
-  "early_stop": false,
-  "seed": 42,
-  "dataset_dir": "/mnt/staging/dataset",
-  "eval_auto_carveout": false,
-  "eval_dataset_dir": null,
-  "train_limit": null,
-  "max_context_len": 8192,
-  "batch_size": 32768,
-  "batch_size_samples": null,
-  "max_data_workers": 0,
-  "min_evals_per_epoch": 1,
-  "max_evals_per_epoch": 5,
-  "precision": null,
-  "status_file": "gs://fireworks-fine-tuning-job-status/sftj-maxx1999syp-bybv7vrv-v6vlpnyh-5d74d2ea-e4c7-4e3a-ae4d-deb107b98a9e",
-  "billing_file": "gs://fireworks-fine-tuning-metadata/sftj-maxx1999syp-bybv7vrv-v6vlpnyh/billing-5d74d2ea-e4c7-4e3a-ae4d-deb107b98a9e",
-  "metrics_file": "gs://fireworks-fine-tuning-metadata/sftj-maxx1999syp-bybv7vrv-v6vlpnyh/metrics.jsonl",
-  "trainer_logs_file": null,
-  "profile": null,
-  "weight_sharding": null,
-  "activation_sharding": null,
-  "empty_weights": false,
-  "nan_ratio_threshold": 0.05,
-  "fast_api_port": 80,
-  "optimizer": "adamw",
-  "optimizer_weight_decay": 0.01,
-  "target_shard_size_gb": null,
-  "enable_fast_processor": false,
-  "peft_addon_dir": null,
-  "lora_rank": 32,
-  "lora_dropout": 0.05,
-  "template_kind": "conversation",
-  "template": null,
-  "mtp_config": { "enable_mtp": false, "freeze_base_model": false, "num_draft_tokens": 1 },
-  "distillation_alpha": null,
-  "qat": true,
-  "kld": false,
-  "teft_tokens": [],
-  "skip_dataset_filtering": false
+  "trainer": "transformers.Trainer",
+  "peft": "LoRA",
+  "base_model": "google/gemma-3-27b-it",
+  "dataset": "N8Programs/unslop-good",
+  "objective": "PPL_cond on assistant tokens only; prompt masked up to and including <start_of_turn>model",
+  "max_length": 8704,
+  "lora": {
+    "r": 64,
+    "alpha": 128,
+    "dropout": 0.05,
+    "target_modules": ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+  },
+  "optim": {
+    "optimizer": "adamw_torch_fused",
+    "learning_rate": 0.0001,
+    "lr_scheduler": "cosine",
+    "warmup_ratio": 0.03,
+    "weight_decay": 0.0
+  },
+  "batching": {
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 8
+  },
+  "precision": {
+    "bf16": true,
+    "tf32": true,
+    "gradient_checkpointing": true
+  },
+  "epochs": 5,
+  "selected_checkpoint": {
+    "checkpoint": "checkpoint-125",
+    "epoch": 1
+  }
 }