eee

app.py

@@ -1,22 +1,14 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
 import gpt
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
 
-def respond(
-    message
-):
-    return gpt.get_response(message)
+print(gpt.get_response("test"))
 
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
-demo = gr.Interface(fn=respond, inputs="textbox", outputs="textbox")
+demo = gr.Interface(fn=gpt.get_response, inputs="textbox", outputs="textbox")
 
 
 if __name__ == "__main__":

gpt.py

@@ -103,17 +103,35 @@ class GPT(nn.Module):
     return logits, loss
 
 
+def load_compiled_model_state_dict(model, state_dict_path):
+    # Load the state dict
+    state_dict = torch.load(state_dict_path, map_location=torch.device('cpu'))
+    
+    # Create a new state dict without the '_orig_mod.' prefix
+    new_state_dict = {}
+    for key, value in state_dict.items():
+        if key.startswith('_orig_mod.'):
+            new_key = key[len('_orig_mod.'):]
+            new_state_dict[new_key] = value
+        else:
+            new_state_dict[key] = value
+    
+    # Load the new state dict into the model
+    model.load_state_dict(new_state_dict)
+    return model
+
 block_size = 512
 n_layers = 12
 n_heads = 12
 d_model = 768
 
-torch.set_float32_matmul_precision('high')
+torch.set_float32_matmul_precision('medium')
 
 my_GPT = GPT(enc.n_vocab, block_size, n_layers, n_heads, d_model, dropout=0.1) #enc.n_vocab
 my_GPT = my_GPT.to(device)
-my_GPT = torch.compile(my_GPT)
-my_GPT.load_state_dict(torch.load('latest_model_finetune.pth', map_location=torch.device('cpu')))
+#my_GPT = torch.compile(my_GPT, mode='reduce-overhead')
+my_GPT = load_compiled_model_state_dict(my_GPT, 'latest_model_finetune.pth')
+#my_GPT.load_state_dict(torch.load('latest_model_finetune.pth', map_location=torch.device('cpu')))
 my_GPT.eval()
 
 eot = enc._special_tokens['<|endoftext|>']
@@ -153,5 +171,6 @@ def get_response(in_text):
       break
     input_tokens.append(result)
     output_tokens.append(result)
+    yield enc.decode(output_tokens)
     
-  return enc.decode(output_tokens)
+  yield enc.decode(output_tokens)