app.py

import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
from threading import Thread

# Load the tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1")
model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1", torch_dtype=torch.float16)

# Move model to GPU if available, otherwise use CPU
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)

class StopOnTokens(StoppingCriteria):
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
        stop_ids = [29, 0]  # Define stop token IDs
        for stop_id in stop_ids:
            if input_ids[0][-1] == stop_id:
                return True
        return False

def predict(message, history):
    history_transformer_format = list(zip(history[:-1], history[1:])) + [[message, ""]]
    stop = StopOnTokens()

    # Format the messages for the model
    messages = "".join([f"\n<human>:{item[0]}\n<bot>:{item[1]}" for item in history_transformer_format])

    # Tokenize the input and move it to the correct device (GPU/CPU)
    model_inputs = tokenizer([messages], return_tensors="pt").to(device)
    
    # Create a streamer for output token generation
    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
    
    # Define generation parameters
    generate_kwargs = dict(
        model_inputs,
        streamer=streamer,
        max_new_tokens=1024,
        do_sample=True,
        top_p=0.95,
        top_k=1000,
        temperature=1.0,
        num_beams=1,
        stopping_criteria=StoppingCriteriaList([stop])
    )

    # Run the generation in a separate thread
    t = Thread(target=model.generate, kwargs=generate_kwargs)
    t.start()

    # Yield generated tokens as they are produced
    partial_message = ""
    for new_token in streamer:
        if new_token != '<':  # Ignore special tokens
            partial_message += new_token
            yield partial_message

# Gradio interface to interact with the model
gr.ChatInterface(predict).launch()




# import gradio as gr
# from huggingface_hub import InferenceClient

# """
# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
# """
# client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")


# def respond(
#     message,
#     history: list[tuple[str, str]],
#     system_message,
#     max_tokens,
#     temperature,
#     top_p,
# ):
#     messages = [{"role": "system", "content": system_message}]

#     for val in history:
#         if val[0]:
#             messages.append({"role": "user", "content": val[0]})
#         if val[1]:
#             messages.append({"role": "assistant", "content": val[1]})

#     messages.append({"role": "user", "content": message})

#     response = ""

#     for message in client.chat_completion(
#         messages,
#         max_tokens=max_tokens,
#         stream=True,
#         temperature=temperature,
#         top_p=top_p,
#     ):
#         token = message.choices[0].delta.content

#         response += token
#         yield response

# """
# For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
# """
# demo = gr.ChatInterface(
#     respond,
#     additional_inputs=[
#         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
#         gr.Slider(
#             minimum=0.1,
#             maximum=1.0,
#             value=0.95,
#             step=0.05,
#             label="Top-p (nucleus sampling)",
#         ),
#     ],
# )


# if __name__ == "__main__":
#     demo.launch()




# import gradio as gr

# def fake(message, history):
#     if message.strip():
#         # Instead of returning audio directly, return a message
#         return "Playing sample audio...", gr.Audio("https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav")
#     else:
#         return "Please provide the name of an artist", None

# with gr.Blocks() as demo:
#     chatbot = gr.Chatbot(placeholder="Play music by any artist!")
#     textbox = gr.Textbox(placeholder="Which artist's music do you want to listen to?", scale=7)
#     audio_player = gr.Audio()

#     def chat_interface(message, history):
#         response, audio = fake(message, history)
#         return history + [(message, response)], audio

#     textbox.submit(chat_interface, [textbox, chatbot], [chatbot, audio_player])

# demo.launch()

# import random
# def random_response(message, history):
#     return random.choice(["Yes", "No"])

# gr.ChatInterface(random_response).launch()



# import gradio as gr

# def yes_man(message, history):
#     if message.endswith("?"):
#         return "Yes"
#     else:
#         return "Ask me anything!"

# gr.ChatInterface(
#     yes_man,
#     chatbot=gr.Chatbot(placeholder="<strong>Ask me a yes or no question</strong><br>Ask me anything"),
#     textbox=gr.Textbox(placeholder="Ask me a yes or no question", container=False, scale=15),
#     title="Yes Man",
#     description="Ask Yes Man any question",
#     theme="soft",
#     examples=[{"text": "Hello"}, {"text": "Am I cool?"}, {"text": "Are tomatoes vegetables?"}],
#     cache_examples=True,
#     retry_btn=None,
#     undo_btn="Delete Previous",
#     clear_btn="Clear",
# ).launch()

# below code is not working
# import gradio as gr

# def count_files(files):
#     num_files = len(files)
#     return f"You uploaded {num_files} file(s)"

# with gr.Blocks() as demo:
#     with gr.Row():
#         chatbot = gr.Chatbot()
#         file_input = gr.Files(label="Upload Files")
#     file_input.change(count_files, inputs=file_input, outputs=chatbot)

# demo.launch()


# new code
# import os
# from langchain_openai import ChatOpenAI
# from langchain.schema import AIMessage, HumanMessage
# import openai
# import gradio as gr


# os.environ["OPENAI_API_KEY"] = "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"  # Replace with your key

# llm = ChatOpenAI(temperature=1.0, model='gpt-3.5-turbo')

# def predict(message, history):
#     history_langchain_format = []
#     for msg in history:
#         if msg['role'] == "user":
#             history_langchain_format.append(HumanMessage(content=msg['content']))
#         elif msg['role'] == "assistant":
#             history_langchain_format.append(AIMessage(content=msg['content']))
#     history_langchain_format.append(HumanMessage(content=message))
#     gpt_response = llm(history_langchain_format)
#     return gpt_response.content

# gr.ChatInterface(predict).launch()