load_finetuned.py

import torch
from transformers import (
    AutoModelForSeq2SeqLM, AutoTokenizer, BitsAndBytesConfig
)
from peft import PeftModel

def load_model(model_name, finetune_type):
    """Loads a fine-tuned model from the Hugging Face repository based on its type."""
    if model_name not in MODEL_REPOS:
        raise ValueError(f"Invalid model name. Choose from: {list(MODEL_REPOS.keys())}")
    
    if finetune_type not in MODEL_REPOS[model_name]:
        raise ValueError(f"Invalid finetune type. Choose from: {list(MODEL_REPOS[model_name].keys())}")

    repo_name = MODEL_REPOS[model_name][finetune_type]
    device = "cuda" if torch.cuda.is_available() else "cpu"

    # Load tokenizer
    tokenizer = AutoTokenizer.from_pretrained(repo_name)

    if model_name == "mT5":  # 4-bit quantized + QLoRA fine-tuned
        print(f"Loading {model_name} with {finetune_type} finetuning, 4-bit quantization, and QLoRA...")

        # Load model with 4-bit quantization settings
        quant_config = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_compute_dtype=torch.float16,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type="nf4"
        )

        base_model_name = "google/mt5-xl"  # Use correct base model
        model = AutoModelForSeq2SeqLM.from_pretrained(base_model_name, quantization_config=quant_config, device_map="auto")

        # Apply fine-tuned LoRA adapter
        model = PeftModel.from_pretrained(model, repo_name)

    elif model_name == "mBART50":  # Normally fine-tuned
        print(f"Loading {model_name} with {finetune_type} fine-tuning...")

        model = AutoModelForSeq2SeqLM.from_pretrained(repo_name)
        model.to(device)

    else:
        raise ValueError(f"Unknown model: {model_name}")

    print(f"{model_name} ({finetune_type}) loaded successfully!")
    return model, tokenizer

MODEL_REPOS = {
    "mT5": {
        "english": "darpanaswal/mT5-english-finetuned",
        "multilingual": "darpanaswal/mT5-multilingual-finetuned",
        "crosslingual": "darpanaswal/mT5-crosslingual-finetuned",
    },
    "mBART50": {
        "english": "darpanaswal/mBART50-english-finetuned",
        "multilingual": "darpanaswal/mBART50-multilingual-finetuned",
        "crosslingual": "darpanaswal/mBART50-crosslingual-finetuned",
    },
}