Update app.py

app.py

@@ -1,8 +1,6 @@
 
 import gradio as gr
-#from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline 
 from ctransformers import AutoModelForCausalLM, AutoTokenizer
-import torch
 import time
 from datetime import datetime
 import shutil
@@ -17,15 +15,12 @@ else:
     print(f"📂 Folder already exists: {folder_path}")
 
 # Define source file and destination folder
-source_file = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
-destination_folder = "./models"
+source_file1 = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+source_file2 = "config.json"
 
 # Copy the file
-shutil.copy(source_file, destination_folder)
-
-source_file = "config.json"
-# Copy the file
-shutil.copy(source_file, destination_folder)
+shutil.copy(source_file1, folder_path)
+shutil.copy(source_file2, folder_path)
 
 model_path = "./models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"  # Adjust if necessary
 
@@ -35,37 +30,9 @@ else:
     print(f"❌ Model not found at: {model_path}")
     print("🔍 Available files:", os.listdir("./models"))
 
-torch.random.manual_seed(0) 
-"""
-model = AutoModelForCausalLM.from_pretrained( 
-    "microsoft/Phi-3-mini-4k-instruct",
-    device_map="cpu",  
-    torch_dtype="auto",  
-    trust_remote_code=True,  
-)
-tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct") 
-"""
-
-"""
-model_name = "microsoft/phi-2"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="cpu")
-"""
-
-torch.set_default_device("cpu")
-
 print("model")
 print(datetime.fromtimestamp(time.time()))
-
-#model_name = "TheBloke/TinyLlama-1.1B-Chat-GGUF"  # Change if using a different version
-#model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"  # Change if using a different version
-model_name = "microsoft/phi-1_5"  # Change if using a different version
-#model = AutoModelForCausalLM.from_pretrained(destination_folder, model_type="llama")
-model = AutoModelForCausalLM.from_pretrained(model_name)
-
-print("tokenizer")
-print(datetime.fromtimestamp(time.time()))
-tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(folder_path, model_type="llama")
 
 print("done setup")
 print(datetime.fromtimestamp(time.time()))
@@ -74,45 +41,8 @@ def generate_letter(date, letter_time, purpose, place, sender, receiver):
     prompt = (f"Tuliskan surat resmi dengan detail sebagai berikut:\n"
               f"Tanggal: {date}\nWaktu: {letter_time}\nTujuan: {purpose}\nTempat: {place}\n"
               f"Pengirim: {sender}\nPenerima: {receiver}\n\nSurat:")
-    """prompt = (f"Write a formal letter with the following details:\n"
-              f"Date: {date}\nTime: {letter_time}\nPurpose: {purpose}\nPlace: {place}\n"
-              f"Sender: {sender}\nReceiver: {receiver}\n\nLetter:")"""
-    """
-    messages = [ 
-        {"role": "system", "content": "You are a helpful AI assistant."}, 
-        {"role": "user", "content": f"{prompt}"}, 
-    ] 
-    
-    pipe = pipeline( 
-    "text-generation", 
-    model=model, 
-    tokenizer=tokenizer, 
-    ) 
-    
-    generation_args = { 
-        "max_new_tokens": 500, 
-        "return_full_text": False, 
-        "temperature": 0.0, 
-        "do_sample": False, 
-    } 
-    
-    output = pipe(messages, **generation_args) 
-    return output[0]['generated_text']
-    """
-
-    print("tokenizer")
-    print(datetime.fromtimestamp(time.time()))
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    
-    print("model.generate")
-    print(datetime.fromtimestamp(time.time()))
-    output = model.generate(**inputs, max_length=100, temperature=0.7, top_p=0.9)
-    
-    print("return")
-    print(datetime.fromtimestamp(time.time()))
-    return tokenizer.decode(output[0], skip_special_tokens=True)
-    """response = model(prompt)
-    return response"""
+    response = model(prompt)
+    return response
 
 # Create the Gradio interface
 iface = gr.Interface(