run

Dockerfile

@@ -1,7 +1,7 @@
 # Hugging Face Spaces (Docker SDK) - FastAPI + Uvicorn using uv
 # Docs: https://huggingface.co/docs/hub/spaces-sdks-docker
 
-FROM python:3.13-slim
+FROM python:3.12-slim
 
 # Create a non-root user matching HF Spaces expectations
 RUN useradd -m -u 1000 user

fine_tune_llama32_1b.py

@@ -0,0 +1,147 @@
+import torch
+from datasets import load_dataset
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    BitsAndBytesConfig,
+)
+from peft import LoraConfig, prepare_model_for_kbit_training, get_peft_model
+from trl import SFTTrainer, SFTConfig
+
+# ========================= CONFIG =========================
+model_name = "meta-llama/Llama-3.2-1B-Instruct"
+dataset_name = "mlabonne/FineTome-100k"
+
+# Training hyperparameters (tune these based on your GPU VRAM)
+max_seq_length = 2048          # Llama 3.2 supports 128k, but 2048 is safe & fast
+batch_size = 4                 # reduce to 2 or 1 if OOM
+gradient_accumulation_steps = 4
+num_train_epochs = 1           # set to 2 or 3 if you want better quality (longer training)
+learning_rate = 2e-4
+max_steps = None               # set a number (e.g. 2000) if you want to train only part of the dataset
+
+output_dir = "./llama32-1b-finetuned-finetome"
+
+# ====================== LOAD MODEL (4-bit QLoRA) ======================
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    quantization_config=bnb_config,
+    device_map="auto",           # automatically puts layers on GPU
+    trust_remote_code=True,
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
+
+# Prepare for QLoRA
+model = prepare_model_for_kbit_training(model)
+
+# LoRA config (QLoRA paper defaults work great for 1B model)
+lora_config = LoraConfig(
+    r=16,                          # 8 or 32 also fine
+    lora_alpha=32,
+    target_modules="all-linear",   # modern way for Llama (q_proj, k_proj, etc.)
+    lora_dropout=0.05,
+    bias="none",
+    task_type="CAUSAL_LM",
+)
+
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()   # should show ~0.5-1% trainable params
+
+# ====================== LOAD & PREPARE DATASET ======================
+dataset = load_dataset(dataset_name, split="train")
+
+# Convert ShareGPT "conversations" → standard messages format
+def map_to_messages(example):
+    messages = []
+    for turn in example["conversations"]:
+        role = "user" if turn["from"] == "human" else \
+               "assistant" if turn["from"] == "gpt" else "system"
+        messages.append({"role": role, "content": turn["value"]})
+    return {"messages": messages}
+
+dataset = dataset.map(map_to_messages, remove_columns=["conversations", "source", "score"])
+
+# Format with Llama-3.2 chat template (this creates the final "text" column)
+def formatting_func(example):
+    text = tokenizer.apply_chat_template(
+        example["messages"],
+        tokenize=False,
+        add_generation_prompt=False
+    )
+    return {"text": text}
+
+dataset = dataset.map(formatting_func, remove_columns=["messages"])
+
+# Optional: use only first 10k examples for quick test
+# dataset = dataset.select(range(10000))
+
+# ====================== TRAINER ======================
+training_args = SFTConfig(
+    output_dir=output_dir,
+    per_device_train_batch_size=batch_size,
+    gradient_accumulation_steps=gradient_accumulation_steps,
+    gradient_checkpointing=True,          # saves VRAM
+    learning_rate=learning_rate,
+    num_train_epochs=num_train_epochs,
+    max_steps=max_steps,
+    warmup_steps=100,
+    logging_steps=10,
+    save_steps=500,
+    save_total_limit=2,
+    fp16=False,                           # bfloat16 is used via compute_dtype
+    bf16=torch.cuda.is_bf16_supported(),
+    optim="paged_adamw_8bit",
+    max_seq_length=max_seq_length,
+    packing=True,                         # packs multiple examples into one sequence (faster)
+    dataset_text_field="text",
+    report_to="none",                     # change to "tensorboard" if you want logs
+)
+
+trainer = SFTTrainer(
+    model=model,
+    tokenizer=tokenizer,
+    train_dataset=dataset,
+    args=training_args,
+    # peft_config is NOT needed because we already did get_peft_model
+)
+
+print("Starting training...")
+trainer.train()
+
+# Save the LoRA adapter
+trainer.model.save_pretrained(output_dir)
+tokenizer.save_pretrained(output_dir)
+print(f"✅ Training finished! LoRA adapter saved to {output_dir}")
+
+
+#model merger from peft import AutoPeftModelForCausalLM
+
+model = AutoPeftModelForCausalLM.from_pretrained(
+    output_dir,
+    device_map="auto",
+    torch_dtype=torch.bfloat16
+)
+model = model.merge_and_unload()
+model.save_pretrained("llama32-1b-finetuned-merged")
+tokenizer.save_pretrained("llama32-1b-finetuned-merged")
+
+
+from peft import AutoPeftModelForCausalLM
+
+model = AutoPeftModelForCausalLM.from_pretrained(
+    output_dir,
+    device_map="auto",
+    torch_dtype=torch.bfloat16
+)
+model = model.merge_and_unload()
+model.save_pretrained("llama32-1b-finetuned-merged")
+tokenizer.save_pretrained("llama32-1b-finetuned-merged")

pyproject.toml

@@ -5,6 +5,36 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
-    "fastapi>=0.135.2",
-    "uvicorn>=0.42.0",
+   "accelerate>=1.13.0",
+   "bitsandbytes>=0.49.2; sys_platform == 'linux' and platform_machine == 'x86_64'",
+   "datasets>=4.8.4",
+   "fastapi>=0.135.2",
+   "huggingface-hub>=1.8.0",
+   "peft>=0.18.1",
+   "torch>=2.5.0",
+   "torchaudio>=2.5.0",
+   "transformers>=5.4.0",
+   "trl>=0.29.1",
+   "uvicorn>=0.42.0",
+   "torchvision>=0.20.0",
 ]
+
+[tool.uv]
+# Resolve and lock for both:
+# - local dev on macOS (Apple Silicon)
+# - Hugging Face Spaces runtime (Linux x86_64)
+environments = [
+   "sys_platform == 'darwin' and platform_machine == 'arm64'",
+   "sys_platform == 'linux' and platform_machine == 'x86_64'",
+]
+
+[tool.uv.sources]
+# Use CUDA-enabled PyTorch wheels on Linux; other platforms will use the default index (PyPI).
+torch = [{ index = "pytorch-cu121", marker = "sys_platform == 'linux'" }]
+torchvision = [{ index = "pytorch-cu121", marker = "sys_platform == 'linux'" }]
+torchaudio = [{ index = "pytorch-cu121", marker = "sys_platform == 'linux'" }]
+
+[[tool.uv.index]]
+name = "pytorch-cu121"
+url = "https://download.pytorch.org/whl/cu121"
+explicit = true