|
|
--- |
|
|
tags: |
|
|
- autotrain |
|
|
- text-generation |
|
|
widget: |
|
|
- text: This is a private NLP model trained with data from SequioaDB |
|
|
datasets: |
|
|
- wangzhang/sdb |
|
|
library_name: adapter-transformers |
|
|
--- |
|
|
|
|
|
# This is a private NLP model trained with data from SequioaDB |
|
|
|
|
|
``` |
|
|
import torch |
|
|
from peft import PeftModel |
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig |
|
|
|
|
|
model_name = "TinyPixel/Llama-2-7B-bf16-sharded" |
|
|
adapters_name = 'wangzhang/Llama2-sequoiaDB' |
|
|
|
|
|
model = AutoModelForCausalLM.from_pretrained( |
|
|
model_name, |
|
|
load_in_4bit=True, |
|
|
torch_dtype=torch.bfloat16, |
|
|
device_map="auto", |
|
|
max_memory= {i: '24000MB' for i in range(torch.cuda.device_count())}, |
|
|
quantization_config=BitsAndBytesConfig( |
|
|
load_in_4bit=True, |
|
|
bnb_4bit_compute_dtype=torch.bfloat16, |
|
|
bnb_4bit_use_double_quant=True, |
|
|
bnb_4bit_quant_type='nf4' |
|
|
), |
|
|
) |
|
|
model = PeftModel.from_pretrained(model, adapters_name) |
|
|
tokenizer = AutoTokenizer.from_pretrained(model_name) |
|
|
``` |
