cpu run

README.md

@@ -109,6 +109,16 @@ python evaluation/compare_lora.py \
   --lora_dir outputs/tinyllama-lora \
   --prompt "Explain LoRA in one sentence."
 ```
+```bash
+python evaluation/compare_lora.py \
+  --base_model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
+  --lora_dir outputs/tinyllama-lora \
+  --prompt "Explain LoRA in one sentence." \
+  --device cpu \
+  --torch_dtype float32
+
+  ```
+
 Optional flags: `--max_new_tokens`, `--temperature`, `--top_p`, `--torch_dtype`.
 
 ## Troubleshooting
@@ -152,4 +162,21 @@ python train_tulu.py \
   --input_field input \
   --output_field output
 
+  ===
+  only cpu
+  python train_tulu.py \
+  --model_name TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
+  --output_dir outputs/tinyllama-lora \
+  --offload_folder offload \
+  --device cuda \
+  --torch_dtype auto \
+  --max_seq_length 512 \
+  --per_device_batch_size 2 \
+  --gradient_accumulation_steps 8 \
+  --num_train_epochs 1 \
+  --use_4bit \
+  --instruction_field instruction \
+  --input_field input \
+  --output_field output
+
 

evaluation/compare_lora.py

@@ -20,6 +20,12 @@ def parse_args():
         choices=["auto", "float16", "bfloat16", "float32"],
         help="Force dtype for model load.",
     )
+    p.add_argument(
+        "--device",
+        default="auto",
+        choices=["auto", "cpu", "cuda", "mps"],
+        help="Force device map; on CPU use this to keep everything on host.",
+    )
     return p.parse_args()
 
 
@@ -29,6 +35,10 @@ def resolve_dtype(name: str) -> Optional[torch.dtype]:
     return {"float16": torch.float16, "bfloat16": torch.bfloat16, "float32": torch.float32}[name]
 
 
+def resolve_device_map(device: str):
+    return {"": "cpu"} if device == "cpu" else "auto"
+
+
 def generate(model, tokenizer, prompt: str, max_new_tokens: int, temperature: float, top_p: float) -> str:
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     with torch.inference_mode():
@@ -44,18 +54,19 @@ def generate(model, tokenizer, prompt: str, max_new_tokens: int, temperature: fl
 
 def main():
     args = parse_args()
-    torch_dtype = resolve_dtype(args.torch_dtype)
+    torch_dtype = resolve_dtype(args.torch_dtype) or (torch.float32 if args.device == "cpu" else None)
+    device_map = resolve_device_map(args.device) if args.device != "auto" else "auto"
 
     tokenizer = AutoTokenizer.from_pretrained(args.lora_dir, use_fast=False)
 
     base_model = AutoModelForCausalLM.from_pretrained(
         args.base_model,
-        device_map="auto",
+        device_map=device_map,
         torch_dtype=torch_dtype,
     )
     lora_wrapped = AutoModelForCausalLM.from_pretrained(
         args.base_model,
-        device_map="auto",
+        device_map=device_map,
         torch_dtype=torch_dtype,
     )
     lora_wrapped = PeftModel.from_pretrained(lora_wrapped, args.lora_dir)

evaluation/simple_inference.py

@@ -1,25 +1,59 @@
+import argparse
+from typing import Optional
+
 import torch
 from peft import PeftConfig, PeftModel
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
+def resolve_dtype(name: str, device: str) -> Optional[torch.dtype]:
+    if name == "auto":
+        # On CPU, default to fp32; otherwise let transformers pick.
+        return torch.float32 if device == "cpu" else None
+    return {"float16": torch.float16, "bfloat16": torch.bfloat16, "float32": torch.float32}[name]
+
+
+def resolve_device_map(device: str):
+    return {"": "cpu"} if device == "cpu" else "auto"
+
+
+def parse_args():
+    p = argparse.ArgumentParser(description="Run a quick LoRA inference.")
+    p.add_argument("--lora_dir", default="outputs/tinyllama-lora", help="Path to LoRA adapter folder.")
+    p.add_argument("--prompt", default="### Instruction:\nExplain LoRA in one sentence.\n\n### Input:\nN/A\n\n### Response:\n")
+    p.add_argument("--max_new_tokens", type=int, default=128)
+    p.add_argument("--temperature", type=float, default=0.7)
+    p.add_argument("--top_p", type=float, default=0.9)
+    p.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda", "mps"])
+    p.add_argument("--torch_dtype", default="auto", choices=["auto", "float16", "bfloat16", "float32"])
+    return p.parse_args()
+
+
 def main():
-    lora_dir = "outputs/tinyllama-lora"  # change to your adapter path
-    cfg = PeftConfig.from_pretrained(lora_dir)
+    args = parse_args()
+    cfg = PeftConfig.from_pretrained(args.lora_dir)
     base_model = cfg.base_model_name_or_path  # base model id/path
+    torch_dtype = resolve_dtype(args.torch_dtype, args.device)
+    device_map = resolve_device_map(args.device) if args.device != "auto" else "auto"
 
-    tokenizer = AutoTokenizer.from_pretrained(lora_dir, use_fast=False)
+    tokenizer = AutoTokenizer.from_pretrained(args.lora_dir, use_fast=False)
     model = AutoModelForCausalLM.from_pretrained(
         base_model,
-        device_map="auto",
-        torch_dtype=torch.float16,
+        device_map=device_map,
+        torch_dtype=torch_dtype,
     )
-    model = PeftModel.from_pretrained(model, lora_dir)
+    model = PeftModel.from_pretrained(model, args.lora_dir)
 
-    prompt = "### Instruction:\nExplain LoRA in one sentence.\n\n### Input:\nN/A\n\n### Response:\n"
+    prompt = args.prompt
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     with torch.inference_mode():
-        out = model.generate(**inputs, max_new_tokens=128, do_sample=True, temperature=0.7)
+        out = model.generate(
+            **inputs,
+            max_new_tokens=args.max_new_tokens,
+            do_sample=True,
+            temperature=args.temperature,
+            top_p=args.top_p,
+        )
     print(tokenizer.decode(out[0], skip_special_tokens=True))
 
 

train_tulu.py

@@ -283,6 +283,9 @@ def main():
     data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
     # Pad/batch causal LM examples.
 
+    # Choose optimizer: paged_adamw_32bit for 4-bit GPU; fall back to AdamW on CPU/no-4bit.
+    optim_name = "paged_adamw_32bit" if cfg.use_4bit and not force_cpu else "adamw_torch"
+
     training_args = TrainingArguments(
         output_dir=cfg.output_dir,
         per_device_train_batch_size=cfg.per_device_batch_size,
@@ -295,7 +298,7 @@ def main():
         bf16=use_bf16,
         fp16=use_fp16,
         report_to=["wandb"],
-        optim="paged_adamw_32bit",
+        optim=optim_name,
     )
     # Trainer configuration (logging, saving, optimizer, precision).