Update app.py

app.py

@@ -1,44 +1,30 @@
-import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
 import os
+import gradio as gr
+from transformers import pipeline
 
-token = os.getenv("HF_TOKEN")  # Safely fetch the token from environment
-# tokenizer = AutoTokenizer.from_pretrained(
-#     "meta-llama/Meta-Llama-3-70B-Instruct",
-#     token=token  # Use the token when loading the model
-# )
+# Load token from environment
+token = os.getenv("HF_TOKEN")
 
-# Load model and tokenizer
-tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B-Instruct", token=token)
-model = AutoModelForCausalLM.from_pretrained(
-    "meta-llama/Meta-Llama-3-70B-Instruct",
-    torch_dtype=torch.float16,
+# Use a pipeline as a high-level helper
+pipe = pipeline(
+    "text-generation",
+    model="meta-llama/Meta-Llama-3-8B-Instruct",
+    token=token,
+    torch_dtype="auto",
     device_map="auto"
 )
 
 # Inference function
-def generate_response(prompt, max_tokens=256, temperature=0.7):
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=max_tokens,
-        temperature=temperature,
-        do_sample=True,
-        top_p=0.95,
-        eos_token_id=tokenizer.eos_token_id
-    )
-    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+def generate_response(prompt):
+    messages = [{"role": "user", "content": prompt}]
+    response = pipe(messages, max_new_tokens=256, temperature=0.7)
+    return response[0]["generated_text"]
 
 # Gradio interface
 gr.Interface(
     fn=generate_response,
-    inputs=[
-        gr.Textbox(lines=4, label="Prompt"),
-        gr.Slider(50, 1024, step=10, value=256, label="Max Tokens"),
-        gr.Slider(0.1, 1.5, step=0.1, value=0.7, label="Temperature")
-    ],
+    inputs=gr.Textbox(lines=4, label="Prompt"),
     outputs=gr.Textbox(label="Generated Response"),
-    title="Meta LLaMA 3 70B Instruct",
-    description="Gradio demo for Meta-Llama-3-70B-Instruct"
+    title="Meta LLaMA 3 8B Instruct",
+    description="Gradio demo for Meta-Llama-3-8B-Instruct using Hugging Face Transformers pipeline"
 ).launch()