llm_engine.py

#!/usr/bin/env python
# coding=utf-8

# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, time, json, re, gc, subprocess
import gradio as gr
import torch
import numpy as np
import argparse
import time
import sampling
import copy
from datetime import datetime
from huggingface_hub import hf_hub_download
from pynvml import *
from tokenizer_util import add_tokenizer_argument, get_tokenizer
import rwkv_world_tokenizer
from huggingface_hub import snapshot_download, hf_hub_download
hf_hub_download(repo_id="JoPmt/RWKV-5-3B-V2-Quant", filename="rwkv-5-world-3b-v2-20231118-ctx16k.Q4_0.bin", local_dir='~/app/Downloads')
model_path='~/app/Downloads/rwkv-5-world-3b-v2-20231118-ctx16k.Q4_0.bin'
from copy import deepcopy
from enum import Enum
from typing import Dict, List
from huggingface_hub import InferenceClient
from transformers.agents import PythonInterpreterTool
from transformers import AutoTokenizer
tokenizer=AutoTokenizer.from_pretrained("NousResearch/Hermes-2-Pro-Llama-3-8B",revision="pr/13")
tools=[PythonInterpreterTool()]
os.system("apt-get update && apt-get install cmake gcc g++")
os.system("git clone --recursive https://github.com/JoPmt/rwkv.cpp.git && cd rwkv.cpp && mkdir build && cd build && cmake .. -DRWKV_CUBLAS=ON -DRWKV_BUILD_SHARED_LIBRARY=ON -DGGML_CUDA=ON -DRWKV_BUILD_PYTHON_MODULE=ON -DRWKV_BUILD_TOOLS=ON -DRWKV_BUILD_EXTRAS=ON && cmake --build . --config Release && make RWKV_CUBLAS=1 GGML_CUDA=1")
import rwkv_cpp_model
import rwkv_cpp_shared_library

def find_lib():
    for root, dirs, files in os.walk("/"):
        for file in files:
            if file == "librwkv.so":
                return os.path.join(root, file)
    return None
library_path = find_lib()
rwkv_lib = rwkv_cpp_shared_library.RWKVSharedLibrary(library_path)
modal = rwkv_cpp_model.RWKVModel(rwkv_lib,model_path,thread_count=2)
print('Loading RWKV model')
tokenizer_decode, tokenizer_encode = get_tokenizer('auto', modal.n_vocab)
out_str = ''
prompt = out_str
token_count = 1200
temperature = 1.0
top_p = 0.7
presence_penalty = 0.1
count_penalty = 0.4
def generate_prompt(instruction, zput=""):
    instruction = instruction.strip().replace('\r\n','\n').replace('\n\n','\n')
    zput = zput.strip().replace('\r\n','\n').replace('\n\n','\n')
    if zput:
        return f"""Instruction: {instruction}
Input: {zput}
Response:"""
    else:
        return f"""User: hi
Assistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.
User: {instruction}
Assistant:"""
class MessageRole(str, Enum):
    USER = "user"
    ASSISTANT = "assistant"
    SYSTEM = "system"
    TOOL_CALL = "tool-call"
    TOOL_RESPONSE = "tool-response"
    @classmethod
    def roles(cls):
        return [r.value for r in cls]
def get_clean_message_list(message_list: List[Dict[str, str]], role_conversions: Dict[str, str] = {}):
    """
    Subsequent messages with the same role will be concatenated to a single message.

    Args:
        message_list (`List[Dict[str, str]]`): List of chat messages.
    """
    final_message_list = []
    message_list = deepcopy(message_list)  # Avoid modifying the original list
    for message in message_list:
        if not set(message.keys()) == {"role", "content"}:
            raise ValueError("Message should contain only 'role' and 'content' keys!")

        role = message["role"]
        if role not in MessageRole.roles():
            raise ValueError(f"Incorrect role {role}, only {MessageRole.roles()} are supported for now.")

        if role in role_conversions:
            message["role"] = role_conversions[role]

        if len(final_message_list) > 0 and message["role"] == final_message_list[-1]["role"]:
            final_message_list[-1]["content"] = "\n=======\n" + message["content"]
        else:
            final_message_list.append(message)
    return final_message_list
llama_role_conversions = {
    MessageRole.TOOL_RESPONSE: MessageRole.USER,
    MessageRole.TOOL_CALL: MessageRole.USER,
}
class HfEngine:
    def __init__(self, model: str = "JoPmt/JoPmt"):
        self.model = model
        self.client = modal
    def __call__(self, messages: List[Dict[str, str]], stop_sequences=[]) -> str:
        messages = get_clean_message_list(messages, role_conversions=llama_role_conversions)
        print(messages)
        pret=''
        prut=''
        for message in messages:
            print(message['content'])
            if message['role'].lower() == 'system':
                pret+=''+message['content']+''
            if message['role'].lower() == 'user':
                prut+=''+message['content']+''
        ##prompt = ins.format(question=''+pret+''+prut+'', system=pret)
        prompt=tokenizer.apply_chat_template(messages,tokenize=False,add_generation_prompt=True,)
        print(prompt)
        token_count=1200
        temperature=1.0
        top_p=0.7
        presencePenalty = 0.1
        countPenalty = 0.4
        token_ban=[]
        stop_token=[0]
        ctx=pret
        prompt=prut
        all_tokens = []
        out_last = 0
        out_str = ''
        occurrence = {}
        state = None
        ctx=generate_prompt(ctx,prompt)
        prompt_tokens = tokenizer_encode(ctx)
        prompt_token_count = len(prompt_tokens)
        init_logits, init_state = modal.eval_sequence_in_chunks(prompt_tokens, None, None, None, use_numpy=True)
        logits, state = init_logits.copy(), init_state.copy()
        out_str = ''
        occurrence = {}
        bof=[]
        for i in range(token_count):
          for n in occurrence:
            logits[n] -= (presencePenalty + occurrence[n] * countPenalty)
          token = sampling.sample_logits(logits, temperature, top_p)

          if token in stop_token:
            break
          all_tokens += [token]
      
          for xxx in occurrence:
            occurrence[xxx] *= 0.996

          if token not in occurrence:
            occurrence[token] = 1
          else:
            occurrence[token] += 1

          tmp = tokenizer_decode(all_tokens[out_last:])
          if '\ufffd' not in tmp:
            out_str += tmp
            out_last = i + 1
          ##yield out_str.strip()
          logits, state = modal.eval(token, state, state, logits, use_numpy=True)
        del state
        gc.collect()
        return out_str.strip()