Create app.py

app.py

@@ -0,0 +1,54 @@
+import gradio as gr  
+import torch  
+from transformers import AutoTokenizer  
+from open_lm.utils.transformers.hf_config import OpenLMConfig  
+from open_lm.utils.transformers.hf_model import OpenLMforCausalLM  
+
+title = """# 🙋🏻‍♂️ Welcome to Tonic's DCLM 1B"""
+
+
+# Load the model and tokenizer  
+model_name = "TRI-ML/DCLM-1B-IT"  
+  
+# Load the configuration, tokenizer, and model separately  
+config = OpenLMConfig.from_pretrained(model_name)  
+tokenizer = AutoTokenizer.from_pretrained(model_name)  
+model = OpenLMforCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="cuda", config=config)  
+  
+# Define the prompt format  
+def create_prompt(instruction):  
+    PROMPT = '''Below is an instruction that describes a task.\n\nWrite a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:'''  
+    return PROMPT.format(instruction=instruction)  
+  
+# Define the respond function for Gradio  
+def respond(message, history, system_message, max_tokens, temperature, top_p):  
+    # Create the prompt  
+    prompt = create_prompt(message)  
+      
+    # Tokenize the input  
+    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(torch.device('cuda'))  
+      
+    # Generate the response  
+    output = model.generate(input_ids, max_length=max_tokens, top_p=top_p, do_sample=True, temperature=temperature)  
+      
+    # Decode the response  
+    response = tokenizer.decode(output[0][len(input_ids[0]):])  
+    response = response.split("<|endoftext|>")[0]  
+      
+    return response  
+  
+# Create Gradio ChatInterface  
+demo = gr.ChatInterface(
+    gr.markdown(title),
+    # gr.markdown(description),
+    respond,  
+    additional_inputs=[  
+        # gr.Textbox(value="You are a friendly Chatbot.", label="System message"),  
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),  
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),  
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")  
+    ],  
+)  
+  
+if __name__ == "__main__":  
+    demo.launch()