Spaces:
Sleeping
Sleeping
File size: 1,580 Bytes
f4f553b 5ba8994 f4f553b 5ba8994 f4f553b 5ba8994 f4f553b 5ba8994 f4f553b 5ba8994 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import torch
import threading
from transformers import (
AutoTokenizer,
AutoModelForSeq2SeqLM,
TextIteratorStreamer,
)
MODEL_NAME = "facebook/nllb-200-distilled-600M"
print("Loading tokenizer and model...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)
model.eval()
# -------- normal (non-streaming) ----------
def translate(text: str):
inputs = tokenizer(text, return_tensors="pt").to(device)
lang_id = tokenizer.convert_tokens_to_ids("arb_Arab")
with torch.no_grad():
outputs = model.generate(
inputs["input_ids"],
forced_bos_token_id=lang_id,
max_length=300,
)
return tokenizer.decode(outputs[0], skip_special_tokens=True)
# -------- streaming ----------
def stream_translate(text: str):
inputs = tokenizer(text, return_tensors="pt").to(device)
lang_id = tokenizer.convert_tokens_to_ids("arb_Arab")
streamer = TextIteratorStreamer(
tokenizer,
skip_special_tokens=True,
skip_prompt=True,
)
generation_kwargs = dict(
input_ids=inputs["input_ids"],
forced_bos_token_id=lang_id,
max_length=300,
streamer=streamer,
)
# Run generation in background thread
thread = threading.Thread(
target=model.generate,
kwargs=generation_kwargs,
)
thread.start()
# Yield tokens as they are generated
for token in streamer:
yield token
|