binhquoc/alm-add-granite-non

# !pip install transformers==4.52.4 
import torch
from peft import PeftModel
from transformers import AutoProcessor, AutoModelForCausalLM, BitsAndBytesConfig, AutoModelForSpeechSeq2Seq

model_id = "ibm-granite/granite-speech-3.3-8b"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

base_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, 
                                    device_map="auto",
                                    torch_dtype=torch.bfloat16,
                                    quantization_config=bnb_config,
                                    cache_dir = os.getenv('CACHE_DIR'))

model = PeftModel.from_pretrained(base_model, "binhquoc/alm-add-granite-non")

processor = AutoProcessor.from_pretrained(model_id, cache_dir = os.getenv('CACHE_DIR'))