components/LLM.py

"""
Contain Classes for LLM inference for RAG pipeline.
"""

### ** Make input output tokens as class properties **

from openai import OpenAI
from .utils import count_tokens
import json


class rLLM:
    def __init__(self, llm_name: str, api_key: str) -> None:
        self.llm_name = llm_name
        self.llm_client = OpenAI(
            api_key=api_key, base_url="https://api.together.xyz/v1"
        )
        with open("components/prompts.json", "r") as file:
            SysPrompt = json.load(file)["SysPrompt"]
        self.sys_prompt = SysPrompt

    def generate_rag_response(self, context: str, prompt: str, message_history):
        """
        Generates a natural language response for user query(prompt) based on provided
        context and message history, in Q&A style.
        """

        system_prompt = self.sys_prompt

        messages = [
            {"role": "system", "content": system_prompt},
        ]

        for message in message_history[-6:-1]:
            if message["role"] == "assistant":
                messages.append({"role": "assistant", "content": message["content"]})
            else:
                messages.append(message)

        messages.append(
            {"role": "user", "content": f"CONTEXT:\n{context}QUERY:\n{prompt}"},
        )

        stream = self.llm_client.chat.completions.create(
            model=self.llm_name,
            messages=messages,
            stream=True,
        )

        output = ""
        for chunk in stream:
            if chunk.choices[0].delta.content is not None:
                content = chunk.choices[0].delta.content
                output += content
                yield 0, content

        input_token_count = count_tokens(
            string="\n".join([i["content"] for i in messages])
        )

        output_token_count = count_tokens(string=output)

        yield 1, (output, input_token_count, output_token_count)

    def HyDE(self, query: str, message_history):
        """
        Rephare/rewrite the user query to include more semantics, hence improving the
        semantic search based retreival.
        """

        system_prompt = """You are an AI assistant specifically designed to generate hypothetical answers for semantic search. Your primary function is to create concise Maximum 100-150 words, informative, and relevant responses to user queries. Make sure to capture the original intent of the user query (by including keywords present in user query) as these responses will be used to generate embeddings for improved semantic search results.
"""

        messages = [
            {"role": "system", "content": system_prompt},
        ]

        for message in message_history[-6:-1]:
            if message["role"] == "assistant":
                messages.append({"role": "assistant", "content": message["content"]})
            else:
                messages.append(message)

        messages.append(
            {"role": "user", "content": f"\n\nQUERY:\n{query}"},
        )

        response = self.llm_client.chat.completions.create(
            model="meta-llama/Llama-3-8b-chat-hf",
            messages=messages,
            max_tokens=500,
        )
        response = response.choices[0].message.content
        return response

    ### NOT IN USE

    def generate_rag_chat_response(self, context: str, prompt: str, message_history):
        """
        NOT IN USE CURRENTLY
        Generates a natural language response for user query(prompt) based on provided
        context and message history, in Q&A style.
        """

        system_prompt = """You are a helpful legal compliance CHAT assistant designed to answer and resolve user query in chat format hence quick and *small* responses.

Instructions:
1. Use the provided CONTEXT to inform your responses, citing specific parts when relevant.
2. If unable to answer the QUERY, politely inform the user and suggest what additional information might help.
3. Give a small/chatty format response.
4. Try to give decisive responses that can help user to make informed decision. 
5. Format responses for readability, include bold words to give weightage.

Don't add phrases like "According to provided context.." etcectra. """

        messages = [
            {"role": "system", "content": system_prompt},
        ]

        for message in message_history[-6:-1]:
            if message["role"] == "assistant":
                messages.append({"role": "assistant", "content": message["content"]})
            else:
                messages.append(message)

        messages.append(
            {"role": "user", "content": f"CONTEXT:\n{context}\n\nQUERY:\n{prompt}"},
        )

        stream = self.llm_client.chat.completions.create(
            model=self.llm_name,
            messages=messages,
            stream=True,
        )

        output = ""
        for chunk in stream:
            if chunk.choices[0].delta.content is not None:
                content = chunk.choices[0].delta.content
                output += content
                yield 0, content

        input_token_count = count_tokens(
            string="\n".join([i["content"] for i in messages])
        )

        output_token_count = count_tokens(string=output)

        yield 1, (output, input_token_count, output_token_count)

    def rephrase_query(self, query: str, message_history):
        """
        NOT IN USE CURRENTLY
        Rephare/rewrite the user query to include more semantics, hence improving the
        semantic search based retreival.
        """

        system_prompt = """You are an AI assistant specifically designed to rewrite the user QUERY for semantic search. Your primary function is to create more comprehensive, semantically rich query *while maintaining the original intent of the user*. These responses will be used to generate embeddings for improved semantic search results.
Do not include any other comments or text, other than the repharased/rewritten query.
"""

        messages = [
            {"role": "system", "content": system_prompt},
        ]

        for message in message_history[-6:-1]:
            if message["role"] == "assistant":
                messages.append({"role": "assistant", "content": message["content"]})
            else:
                messages.append(message)

        messages.append(
            {"role": "user", "content": f"\n\nQUERY:\n{query}"},
        )

        response = self.llm_client.chat.completions.create(
            model="meta-llama/Llama-3-8b-chat-hf",
            messages=messages,
        )
        response = response.choices[0].message.content
        return response