|
|
|
|
|
import torch |
|
|
from transformers import AutoTokenizer, AutoModelForCausalLM |
|
|
from peft import PeftModel |
|
|
from huggingface_hub import snapshot_download |
|
|
|
|
|
BASE_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" |
|
|
ADAPTER_PATH = "GilbertAkham/deepseek-R1-multitask-lora" |
|
|
|
|
|
class EndpointHandler: |
|
|
def __init__(self, path=""): |
|
|
print("🚀 Loading base model...") |
|
|
self.tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True) |
|
|
|
|
|
base_model = AutoModelForCausalLM.from_pretrained( |
|
|
BASE_MODEL, |
|
|
torch_dtype=torch.float16, |
|
|
device_map="auto", |
|
|
trust_remote_code=True |
|
|
) |
|
|
|
|
|
print(f"🔗 Downloading LoRA adapter from {ADAPTER_PATH}...") |
|
|
adapter_local_path = snapshot_download(repo_id=ADAPTER_PATH, allow_patterns=["*adapter*"]) |
|
|
print(f"📁 Adapter files cached at {adapter_local_path}") |
|
|
|
|
|
print("🧩 Attaching LoRA adapter...") |
|
|
self.model = PeftModel.from_pretrained(base_model, adapter_local_path) |
|
|
self.model.eval() |
|
|
|
|
|
print("✅ Model + LoRA adapter loaded successfully.") |
|
|
|
|
|
def __call__(self, data): |
|
|
prompt = data.get("inputs", "") |
|
|
inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device) |
|
|
with torch.no_grad(): |
|
|
outputs = self.model.generate( |
|
|
**inputs, |
|
|
max_new_tokens=512, |
|
|
temperature=0.7, |
|
|
top_p=0.9, |
|
|
do_sample=True, |
|
|
pad_token_id=self.tokenizer.eos_token_id, |
|
|
eos_token_id=self.tokenizer.eos_token_id, |
|
|
) |
|
|
text = self.tokenizer.decode(outputs[0], skip_special_tokens=True) |
|
|
return {"generated_text": text} |
|
|
|
