Update app.py

app.py

@@ -1,85 +1,95 @@
-import os
-import spaces
-import torch
+from huggingface_hub import InferenceClient
 import gradio as gr
-from transformers import AutoTokenizer
 
-HF_TOKEN = os.environ.get("HF_TOKEN", None)
+client = InferenceClient(
+    "mistralai/Mistral-7B-Instruct-v0.3"
+)
 
-TITLE = "<h1><center>Learning Content Generator</center></h1>"
 
-PLACEHOLDER = """
-<center>
-<p>Enter the topic, description, and difficulty level to generate learning content.</p>
-</center>
-"""
+def format_prompt(message, history):
+  prompt = "<s>"
+  for user_prompt, bot_response in history:
+    prompt += f"[INST] {user_prompt} [/INST]"
+    prompt += f" {bot_response}</s> "
+  prompt += f"[INST] {message} [/INST]"
+  return prompt
 
-from huggingface_hub import snapshot_download
-from pathlib import Path
+def generate(
+    prompt, history, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,
+):
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
 
-mistral_models_path = Path.home().joinpath('mistral_models', '7B-Mathstral')
-mistral_models_path.mkdir(parents=True, exist_ok=True)
-
-# Update the repo_id to the new model
-snapshot_download(repo_id="mistralai/Mathstral-7B-v0.1", allow_patterns=["params.json", "consolidated.safetensors"], local_dir=mistral_models_path)
-
-from mistral_inference.transformer import Transformer
-from mistral_inference.generate import generate
-
-from mistral_common.protocol.instruct.messages import AssistantMessage, UserMessage
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
-
-# Force device to "cpu"
-device = "cpu"
-
-# Use a pretrained tokenizer from Hugging Face's transformers
-tokenizer = AutoTokenizer.from_pretrained("mistralai/Mathstral-7B-v0.1")
-
-# Load the model and move it to the CPU
-model = Transformer.from_folder(mistral_models_path).to(device=device)
-
-@spaces.GPU()
-def generate_learning_content(topic: str, description: str, difficulty: str, temperature: float = 0.3, max_new_tokens: int = 1024):
-    message = f"Generate learning content on the topic '{topic}' with the description: '{description}' and difficulty level: '{difficulty}'. Provide the content in paragraph format."
-    conversation = [UserMessage(content=message)]
-
-    completion_request = ChatCompletionRequest(messages=conversation)
-    
-    # Encode tokens and move to CPU
-    tokens = tokenizer.encode(completion_request.messages[0].content, return_tensors="pt").to(device)
-
-    # Generate output using the model on CPU
-    out_tokens, _ = generate(
-        [tokens], 
-        model, 
-        max_tokens=max_new_tokens, 
+    generate_kwargs = dict(
         temperature=temperature,
-        eos_id=tokenizer.eos_token_id
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
     )
-    
-    # Decode the output tokens to human-readable text
-    result = tokenizer.decode(out_tokens[0], skip_special_tokens=True)
-    
-    return result
 
-with gr.Blocks(theme="ocean") as demo:
-    gr.HTML(TITLE)
-    topic_input = gr.Textbox(label="Topic", placeholder="Enter the topic for learning content.")
-    description_input = gr.Textbox(label="Description", placeholder="Enter a brief description of the topic.")
-    difficulty_input = gr.Textbox(label="Difficulty Level", placeholder="Enter the difficulty level (easy, medium, hard).")
-    
-    temperature_slider = gr.Slider(minimum=0, maximum=1, step=0.1, value=0.3, label="Temperature")
-    tokens_slider = gr.Slider(minimum=128, maximum=8192, step=1, value=1024, label="Max New Tokens")
+    formatted_prompt = format_prompt(prompt, history)
+
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    output = ""
+
+    for response in stream:
+        output += response.token.text
+        yield output
+    return output
+
+
+additional_inputs=[
+    gr.Slider(
+        label="Temperature",
+        value=0.9,
+        minimum=0.0,
+        maximum=1.0,
+        step=0.05,
+        interactive=True,
+        info="Higher values produce more diverse outputs",
+    ),
+    gr.Slider(
+        label="Max new tokens",
+        value=256,
+        minimum=0,
+        maximum=1048,
+        step=64,
+        interactive=True,
+        info="The maximum numbers of new tokens",
+    ),
+    gr.Slider(
+        label="Top-p (nucleus sampling)",
+        value=0.90,
+        minimum=0.0,
+        maximum=1,
+        step=0.05,
+        interactive=True,
+        info="Higher values sample more low-probability tokens",
+    ),
+    gr.Slider(
+        label="Repetition penalty",
+        value=1.2,
+        minimum=1.0,
+        maximum=2.0,
+        step=0.05,
+        interactive=True,
+        info="Penalize repeated tokens",
+    )
+]
 
-    output = gr.Textbox(label="Generated Learning Content")
 
-    submit_button = gr.Button("Generate Content")
+gr.ChatInterface(
+    fn=generate,
+    chatbot=gr.Chatbot(show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel"),
+    additional_inputs=additional_inputs,
+    title="""Mistral 7B v0.3"""
+).launch(show_api=False)
 
-    submit_button.click(
-        fn=generate_learning_content,
-        inputs=[topic_input, description_input, difficulty_input, temperature_slider, tokens_slider],
-        outputs=output,
-    )
 
-if __name__ == "__main__":
-    demo.launch()
+gr.load("models/ehristoforu/dalle-3-xl-v2").launch()
+
+gr.load("models/microsoft/Phi-3-mini-4k-instruct").launch()