Upload folder using huggingface_hub

README.md

@@ -1,12 +1,12 @@
 # 🧠 Myanmar LLM Training
 
-Training script for Myanmar language model using Qwen2.5-0.5B-Instruct.
+Fine-tune **Llama-3.1-8B-Instruct** with Myanmar language dataset.
 
 ## 📋 Requirements
 
 - Python 3.8+
-- GPU with 8GB+ VRAM (recommended)
-- HuggingFace Account
+- GPU with 16GB+ VRAM (recommended)
+- HuggingFace Account with Llama access
 
 ## 🚀 Quick Start
 
@@ -21,6 +21,8 @@ huggingface-cli login
 # Enter your token
 ```
 
+**Note:** Llama requires accepting the license at https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct
+
 ### 3. Run training
 ```bash
 python train.py
@@ -30,10 +32,18 @@ python train.py
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
-| MODEL_NAME | Qwen/Qwen2.5-0.5B-Instruct | Base model |
+| MODEL_NAME | meta-llama/Llama-3.1-8B-Instruct | Base model |
 | num_train_epochs | 3 | Training iterations |
-| per_device_train_batch_size | 4 | Batch size |
-| learning_rate | 2e-5 | Learning rate |
+| per_device_train_batch_size | 2 | Batch size (4-bit) |
+| gradient_accumulation_steps | 8 | Effective batch |
+| learning_rate | 1e-5 | Learning rate |
+
+## 📊 Features
+
+- ✅ 4-bit quantization (NF4) - အနည်းဆုံး VRAM နဲ့ run လုပ်နိုင်ပါသည်။
+- ✅ Gradient checkpointing - Memory ချွေတာပါသည်။
+- ✅ Test/Validation evaluation - နှစ်ခုလုံးအတွက် စမ်းသပ်ပါသည်။
+- ✅ BF16 mixed precision - ပိုမိုတိကျတဲ့ training။
 
 ## 📊 Training Data
 
@@ -42,33 +52,38 @@ Dataset: [amkyawdev/myanmar-llm-data](https://huggingface.co/datasets/amkyawdev/
 | Split | Samples |
 |-------|---------|
 | Train | 1000 |
-| Test | 1000 |
 | Validation | 1000 |
+| Test | 1000 |
 
 ## 💾 Output
 
-Trained model saved to `./myanmar-llm-output/`
+Trained model saved to `./myanmar-llama-output/`
 
 ## 📤 Upload to HuggingFace
 
 ```bash
-cd myanmar-llm-output
-huggingface-cli upload amkyawdev/my-myanmar-llm-v1 . --repo-type model
+cd myanmar-llama-output
+huggingface-cli upload amkyawdev/my-myanmar-llama . --repo-type model
 ```
 
-## 🖥️ Run on Google Colab
+## 🖥️ Google Colab
 
 ```python
 # Install
-!pip install transformers datasets torch
+!pip install transformers datasets torch bitsandbytes accelerate
 
 # Login
 from huggingface_hub import login
 login("YOUR_TOKEN")
 
-# Run training script
+# Run
 %run train.py
 ```
 
+## ⚠️ Important
+
+1. Llama license လိုပါသည်။ https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct မှာ Accept လုပ်ပါသည်။
+2. Token မှာLlama access ရှိရပါသည်။
+
 ---
 Built by amkyawdev

requirements.txt

@@ -4,4 +4,6 @@ transformers>=4.36.0
 datasets>=2.14.0
 torch>=2.0.0
 accelerate>=0.20.0
-tensorboard>=2.12.0
+tensorboard>=2.12.0
+bitsandbytes>=0.41.0
+scikit-learn>=1.0.0

train.py

@@ -1,6 +1,6 @@
 """
 Myanmar LLM Training Script
-Fine-tune Qwen2.5-0.5B with Myanmar dataset
+Fine-tune Llama-3.1-8B-Instruct with Myanmar dataset
 """
 
 import json
@@ -11,28 +11,75 @@ from transformers import (
     AutoTokenizer,
     TrainingArguments,
     Trainer,
-    DataCollatorForLanguageModeling
+    DataCollatorForLanguageModeling,
+    EvalPrediction,
 )
+from transformers import BitsAndBytesConfig
 import torch
+from sklearn.metrics import accuracy_score
 
 # Config
-MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
-OUTPUT_DIR = "./myanmar-llm-output"
+MODEL_NAME = "meta-llama/Llama-3.1-8B-Instruct"
+OUTPUT_DIR = "./myanmar-llama-output"
 DATASET_PATH = "amkyawdev/myanmar-llm-data"
 
+# Quantization config for low VRAM
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype="float16",
+    bnb_4bit_use_double_quant=True,
+)
+
 def format_conversation(example):
-    """Format conversation for training"""
+    """Format conversation for Llama chat template"""
     messages = example["messages"]
     text = ""
     for msg in messages:
-        if msg["role"] == "system":
-            text += f"<|im_start|>system\n{msg['content']}<|im_end|>\n"
-        elif msg["role"] == "user":
-            text += f"<|im_start|>user\n{msg['content']}<|im_end|>\n"
-        elif msg["role"] == "assistant":
-            text += f"<|im_start|>assistant\n{msg['content']}<|im_end|>\n"
+        role = msg["role"]
+        content = msg["content"]
+        if role == "system":
+            text += f"<|start_header_id|>system<|end_header_id|>\n\n{content}<|eot_id|>"
+        elif role == "user":
+            text += f"<|start_header_id|>user<|end_header_id|>\n\n{content}<|eot_id|>"
+        elif role == "assistant":
+            text += f"<|start_header_id|>assistant<|end_header_id|>\n\n{content}<|eot_id|>"
+    # Add separator
+    text += "<|start_header_id|>assistant<|end_header_id|>\n\n"
     return {"text": text}
 
+def preprocess_function(examples, tokenizer, max_length=2048):
+    """Tokenize the text"""
+    # Add prompt suffix for assistant response
+    texts = [text + "<|start_header_id|>assistant<|end_header_id|>\n\n" for text in examples["text"]]
+    
+    tokenized = tokenizer(
+        texts,
+        truncation=True,
+        max_length=max_length,
+        padding="max_length",
+        return_tensors=None,
+    )
+    
+    # Labels same as input_ids (causal LM)
+    tokenized["labels"] = tokenized["input_ids"].copy()
+    return tokenized
+
+def compute_metrics(eval_pred):
+    """Compute perplexity as evaluation metric"""
+    logits, labels = eval_pred
+    # Shift for causal LM
+    logits = logits[:-1]
+    labels = labels[1:]
+    
+    # Calculate perplexity
+    loss = torch.nn.functional.cross_entropy(
+        torch.tensor(logits), 
+        torch.tensor(labels),
+        ignore_index=-100
+    )
+    return {"perplexity": torch.exp(loss).item()}
+
 def load_data():
     """Load and prepare Myanmar dataset"""
     print("📂 Loading dataset...")
@@ -46,14 +93,16 @@ def load_data():
     return dataset
 
 def main():
-    print("=" * 50)
-    print("🧠 Myanmar LLM Training")
-    print("=" * 50)
+    print("=" * 60)
+    print("🧠 Myanmar LLM Training - Llama 3.1 8B")
+    print("=" * 60)
     
     # Check GPU
     if torch.cuda.is_available():
-        print(f"✅ GPU: {torch.cuda.get_device_name(0)}")
-        print(f"   VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB")
+        gpu_name = torch.cuda.get_device_name(0)
+        vram = torch.cuda.get_device_properties(0).total_memory / 1e9
+        print(f"✅ GPU: {gpu_name}")
+        print(f"   VRAM: {vram:.2f} GB")
     else:
         print("⚠️ No GPU - will use CPU (very slow)")
     
@@ -61,55 +110,74 @@ def main():
     print(f"\n📥 Loading model: {MODEL_NAME}")
     tokenizer = AutoTokenizer.from_pretrained(
         MODEL_NAME,
-        trust_remote_code=True
+        trust_remote_code=True,
+        padding_side="right",
     )
     
     # Set pad token
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token = tokenizer.eos_token
     
-    # Load model
+    # Load model with 4-bit quantization
+    print("🔄 Loading model with 4-bit quantization...")
     model = AutoModelForCausalLM.from_pretrained(
         MODEL_NAME,
+        quantization_config=bnb_config,
         trust_remote_code=True,
-        torch_dtype=torch.float16,
-        device_map="auto"
+        device_map="auto",
     )
     
+    # Disable gradient checkpointing for stability
+    model.gradient_checkpointing_enable()
+    
     # Load dataset
     dataset = load_data()
     
-    # Split for validation
+    # Preprocess
+    print("🔧 Tokenizing...")
+    for split in dataset:
+        dataset[split] = dataset[split].map(
+            lambda x: preprocess_function(x, tokenizer),
+            batched=True,
+            remove_columns=dataset[split].column_names,
+        )
+    
     train_dataset = dataset["train"]
     eval_dataset = dataset["validation"]
+    test_dataset = dataset["test"]
     
     print(f"\n📊 Dataset:")
     print(f"   Train: {len(train_dataset)} samples")
-    print(f"   Eval: {len(eval_dataset)} samples")
+    print(f"   Validation: {len(eval_dataset)} samples")
+    print(f"   Test: {len(test_dataset)} samples")
     
     # Training args
     training_args = TrainingArguments(
         output_dir=OUTPUT_DIR,
         num_train_epochs=3,
-        per_device_train_batch_size=4,
-        per_device_eval_batch_size=4,
-        gradient_accumulation_steps=4,
-        learning_rate=2e-5,
+        per_device_train_batch_size=2,
+        per_device_eval_batch_size=2,
+        gradient_accumulation_steps=8,
+        learning_rate=1e-5,
         warmup_ratio=0.1,
         logging_steps=10,
         save_steps=100,
         eval_steps=100,
         save_total_limit=2,
+        fp16=False,
         bf16=True,
         remove_unused_columns=False,
         optim="adamw_torch",
         report_to="none",
+        load_best_model_at_end=True,
+        eval_strategy="steps",
+        save_strategy="steps",
     )
     
     # Data collator
     data_collator = DataCollatorForLanguageModeling(
         tokenizer=tokenizer,
         mlm=False,
+        pad_to_multiple_of=8,
     )
     
     # Trainer
@@ -119,22 +187,28 @@ def main():
         train_dataset=train_dataset,
         eval_dataset=eval_dataset,
         data_collator=data_collator,
+        compute_metrics=compute_metrics,
     )
     
     # Train
     print("\n🚀 Starting training...")
     trainer.train()
     
+    # Evaluate on test set
+    print("\n📝 Evaluating on test set...")
+    test_results = trainer.evaluate(test_dataset)
+    print(f"Test Results: {test_results}")
+    
     # Save model
     print("\n💾 Saving model...")
-    model.save_pretrained(OUTPUT_DIR)
+    trainer.save_model(OUTPUT_DIR)
     tokenizer.save_pretrained(OUTPUT_DIR)
     
-    print(f"\n✅ Training complete! Model saved to: {OUTPUT_DIR}")
+    print(f"\n✅ Training complete!")
+    print(f"   Model: {OUTPUT_DIR}")
     print(f"\n📤 Upload to HuggingFace:")
-    print(f"   huggingface-cli login")
     print(f"   cd {OUTPUT_DIR}")
-    print(f"   hf_upload amkyawdev/my-myanmar-llm . --repo-type model")
+    print(f"   hf upload amkyawdev/my-myanmar-llama . --repo-type model")
 
 if __name__ == "__main__":
     main()