blackmount8

/

falcon-7b-instruct-ct2-int8_float16

like 0

I haven't tested this but this should help handle the int8_float16

import ctranslate2
from transformers import AutoTokenizer

model_name = "blackmount8/falcon-7b-instruct-ct2-int8_float16"

tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, padding_side="left", truncation_side="left")
model = ctranslate2.Generator(model_name, device="auto", compute_type="float16")

input_text = ["What is the meaning of stonehenge?", "Hello mate!"]

input_ids = tokenizer(input_text, return_tensors="pt", padding=True, truncation=True).input_ids
input_tokens = [tokenizer.convert_ids_to_tokens(ele) for ele in input_ids]

outputs = model.generate_batch(input_tokens, max_length=128)

output_tokens = [
    ele.sequences_ids[0] for ele in outputs
]

output = tokenizer.batch_decode(output_tokens)

print(output)