Update main.py

main.py

@@ -5,8 +5,6 @@ from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.prompts import PromptTemplate
 from langchain.schema.output_parser import StrOutputParser
 
-# from langchain.llms import LlamaCpp
-
 app = Flask(__name__)
 
 n_gpu_layers = 0
@@ -15,7 +13,7 @@ n_batch = 1024
 callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
 
 llm = LlamaCpp(
-    model_path="phi-2.Q4_K_M.gguf",
+    model_path="Phi-3-mini-4k-instruct-q4.gguf",  # path to GGUF file
     temperature=0.1,
     n_gpu_layers=n_gpu_layers,
     n_batch=n_batch,
@@ -28,19 +26,15 @@ llm = LlamaCpp(
 def get_skills():
     cv_body = request.json.get('cv_body')
 
-    template = """Instruct:
-    Take a deep breath to deep understand, and don't this cv vc = {cv_body} . to answer this question and instructions </s> 
-    \nOutput:"""
-
-    prompt = PromptTemplate(template=template, input_variables=["text"])
-    chain = prompt | llm | StrOutputParser()
-
-    ans = chain.invoke({"question": "What are his best skills? write in points","cv_body":cv_body},
-                       config={
-                           # "callbacks": [ConsoleCallbackHandler()]
-                       })
+    # Simple inference example
+    output = llm(
+        f"<|user|>\n{cv_body}<|end|>\n<|assistant|>Can you list the skills mentioned in the CV?<|end|>",
+        max_tokens=256,  # Generate up to 256 tokens
+        stop=["<|end|>"], 
+        echo=True,  # Whether to echo the prompt
+    )
 
-    return jsonify({'skills': ans})
+    return jsonify({'skills': output})
 
 if __name__ == '__main__':
-    app.run()
+    app.run()