Upload folder using huggingface_hub

README.md

@@ -1,12 +1,6 @@
 ---
-title: Gradio Server
-emoji: 👁
-colorFrom: blue
-colorTo: indigo
+title: gradio-server
+app_file: backend.py
 sdk: gradio
-sdk_version: 4.25.0
-app_file: app.py
-pinned: false
+sdk_version: 3.50.2
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,61 @@
+import gradio as gr
+import time
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from langchain.memory import ConversationBufferWindowMemory
+from peft import PeftModel
+import torch
+import re
+
+
+print("Initializing model")
+# Initialize the tokenizer and model
+base_model = "mistralai/Mistral-7B-Instruct-v0.2"
+tokenizer = AutoTokenizer.from_pretrained(base_model)
+tokenizer.add_special_tokens({"pad_token": "[PAD]"})
+base_model = AutoModelForCausalLM.from_pretrained(base_model)
+
+ft_model = PeftModel.from_pretrained(base_model, "nuratamton/story_sculptor_mistral")
+# ft_model = ft_model.merge_and_unload()
+ft_model.eval()
+
+
+# Set the device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ft_model.to(device)
+
+
+memory = ConversationBufferWindowMemory(k=10)
+
+def slow_echo(message, history):
+    message = chat_interface(message)
+    for i in range(len(message)):
+        time.sleep(0.05)
+        yield message[: i+1]
+
+
+def chat_interface(user_in):
+    if user_in.lower() == "quit":
+        return "Goodbye!"
+    #memory.save_context({"input": user_in}, {"output": ""})
+    memory_context = memory.load_memory_variables({})["history"]
+    user_input = f"[INST] Continue the game and maintain context and keep the story consistent throughout: {memory_context}{user_in}[/INST]"
+
+    encodings = tokenizer(user_input, return_tensors="pt", padding=True).to(device)
+    input_ids = encodings["input_ids"]
+    attention_mask = encodings["attention_mask"]
+
+    output_ids = ft_model.generate(input_ids, attention_mask = attention_mask, max_new_tokens=1000, num_return_sequences=1, do_sample=True, temperature=1.1, top_p=0.9, repetition_penalty=1.2)
+
+    generated_ids = output_ids[0, input_ids.shape[-1]:]
+
+    # Decode the output
+    response = tokenizer.decode(generated_ids, skip_special_tokens=True)
+    memory.save_context({"input": user_in}, {"output": response})
+    print(f"Game Agent: {response}")
+    # Your chatbot logic here
+    # response = "You said: " + user_in
+    return response
+
+
+iface = gr.ChatInterface(slow_echo).queue()
+iface.launch(share=True)

backend.py

@@ -0,0 +1,66 @@
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from peft import PeftModel
+from langchain.memory import ConversationBufferWindowMemory
+import gradio as gr
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+base_model = "mistralai/Mistral-7B-Instruct-v0.2"
+tokenizer = AutoTokenizer.from_pretrained(base_model, pad_token="[PAD]")
+model = AutoModelForCausalLM.from_pretrained(
+    base_model,
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True,
+)
+ft_model = PeftModel.from_pretrained(model, "nuratamton/story_sculptor_mistral").eval()
+memory = ConversationBufferWindowMemory(k=10)
+
+def generate_text(message):
+    user_in = message
+
+    if user_in.lower() in ["adventure", "mystery", "horror", "sci-fi"]:
+        memory.clear()
+
+    if user_in.lower() == "quit":
+        raise ValueError("User requested to quit")
+
+    memory_context = memory.load_memory_variables({})["history"]
+    user_input = f"{memory_context}[INST] Continue the game and maintain context: {user_in}[/INST]"
+
+    encodings = tokenizer(user_input, return_tensors="pt", padding=True).to(device)
+    input_ids, attention_mask = encodings["input_ids"], encodings["attention_mask"]
+    output_ids = ft_model.generate(
+        input_ids,
+        attention_mask=attention_mask,
+        max_new_tokens=1000,
+        num_return_sequences=1,
+        do_sample=True,
+        temperature=1.1,
+        top_p=0.9,
+        repetition_penalty=1.2,
+    )
+
+    generated_ids = output_ids[0, input_ids.shape[-1] :]
+    response = tokenizer.decode(generated_ids, skip_special_tokens=True)
+    memory.save_context({"input": user_in}, {"output": response})
+
+    response = response.replace("AI: ", "")
+    return response
+
+iface = gr.Interface(
+    fn=generate_text,
+    inputs="text",
+    outputs="text",
+    title="Text Generation",
+    description="Enter a message to generate text.",
+)
+
+iface.launch(share=True)

flask.py.save

@@ -0,0 +1,18 @@
+from flask import Flask, request, jsonify
+import gradio as gr
+
+app = Flask(__name__)
+
+def my_gradio_function(input_text):
+    # Your processing logic here
+    return "Processed: " + input_text
+
+@app.route("/process", methods=["POST"])
+def process():
+    input_text = request.json["input_text"]
+    output_text = my_gradio_function(input_text)
+    return jsonify({"output_text": output_text})
+
+if __name__ == "__main__":
+    app.run(port=5000)
+

flask.py.save.1

@@ -0,0 +1,18 @@
+from flask import Flask, request, jsonify
+import gradio as gr
+
+app = Flask(__name__)
+
+def my_gradio_function(input_text):
+    # Your processing logic here
+    return "Processed: " + input_text
+
+@app.route("/process", methods=["POST"])
+def process():
+    input_text = request.json["input_text"]
+    output_text = my_gradio_function(input_text)
+    return jsonify({"output_text": output_text})
+
+if __name__ == "__main__":
+    app.run(port=5000)
+

oldbacked.py

@@ -0,0 +1,83 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from peft import PeftModel
+from langchain.memory import ConversationBufferWindowMemory
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+
+# Add CORSMiddleware to the application
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+base_model = "mistralai/Mistral-7B-Instruct-v0.2"
+tokenizer = AutoTokenizer.from_pretrained(base_model, pad_token="[PAD]")
+model = AutoModelForCausalLM.from_pretrained(
+    base_model,
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True,
+)
+ft_model = PeftModel.from_pretrained(model, "nuratamton/story_sculptor_mistral").eval()
+memory = ConversationBufferWindowMemory(k=10)
+
+
+class UserRequest(BaseModel):
+    message: str
+
+
+@app.post("/generate/")
+async def generate_text(request: UserRequest):
+    user_in = request.message
+
+    if user_in.lower() in ["adventure", "mystery", "horror", "sci-fi"]:
+        memory.clear()
+
+    if user_in.lower() == "quit":
+        raise HTTPException(status_code=400, detail="User requested to quit")
+
+    memory_context = memory.load_memory_variables({})["history"]
+    user_input = f"{memory_context}[INST] Continue the game and maintain context: {user_in}[/INST]"
+
+    encodings = tokenizer(user_input, return_tensors="pt", padding=True).to(device)
+    input_ids, attention_mask = encodings["input_ids"], encodings["attention_mask"]
+    output_ids = ft_model.generate(
+        input_ids,
+        attention_mask=attention_mask,
+        max_new_tokens=1000,
+        num_return_sequences=1,
+        do_sample=True,
+        temperature=1.1,
+        top_p=0.9,
+        repetition_penalty=1.2,
+    )
+
+    generated_ids = output_ids[0, input_ids.shape[-1] :]
+    response = tokenizer.decode(generated_ids, skip_special_tokens=True)
+    memory.save_context({"input": user_in}, {"output": response})
+
+    response = response.replace("AI: ", "")
+    # response = response.replace("Human: ", "")
+
+    return {"response": response}
+
+
+@app.get("/")
+def read_root():
+    return {"message": "Hello from FastAPI"}
+

requirements.txt

@@ -0,0 +1 @@
+torch transformers peft langchain

server.py

@@ -0,0 +1,8 @@
+import gradio as gr
+
+def add_numbers(num1, num2):
+    return num1 + num2
+
+iface = gr.Interface(fn=add_numbers, inputs=["number", "number"], outputs="number")
+iface.launch(share=True)
+