Update main.py

main.py

@@ -1,37 +1,46 @@
 from flask import Flask, request, jsonify
 from langchain_community.llms import LlamaCpp
-from llama_cpp import Llama
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
 
+# from langchain.llms import LlamaCpp
 
-
-import os 
 app = Flask(__name__)
 
 n_gpu_layers = 0
 n_batch = 1024
 
+callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
 
-llm = Llama(
-  model_path="Phi-3-mini-4k-instruct-q4.gguf",  # path to GGUF file
-  n_gpu_layers=0, # The number of layers to offload to GPU, if you have GPU acceleration available. Set to 0 if no GPU acceleration is available on your system.
+llm = LlamaCpp(
+    model_path="phi-2.Q4_K_M.gguf",
+    temperature=0.1,
+    n_gpu_layers=n_gpu_layers,
+    n_batch=n_batch,
+    callback_manager=callback_manager,
+    verbose=True,
+    n_ctx=2048
 )
 
-file_size = os.stat('Phi-3-mini-4k-instruct-q4.gguf')
-print("model size ====> :", file_size.st_size, "bytes")
-
-
 @app.route('/', methods=['POST'])
 def get_skills():
     cv_body = request.json.get('cv_body')
 
-    # Simple inference example
-    output = llm(
-        f"<|user|>\n{cv_body}<|end|>\n<|assistant|>Can you list the skills mentioned in the CV?<|end|>",
-        max_tokens=256,  # Generate up to 256 tokens
-        stop=["<|end|>"], 
-    )
+    template = """Instruct:
+    Take a deep breath to deep understand, and don't this cv vc = {cv_body} . to answer this question and instructions </s> 
+    \nOutput:"""
+
+    prompt = PromptTemplate(template=template, input_variables=["text"])
+    chain = prompt | llm | StrOutputParser()
+
+    ans = chain.invoke({"question": "What are his best skills? write in points","cv_body":cv_body},
+                       config={
+                           # "callbacks": [ConsoleCallbackHandler()]
+                       })
 
-    return jsonify({'skills': output['choices'][0]['text']})
+    return jsonify({'skills': ans})
 
 if __name__ == '__main__':
-    app.run()
+    app.run()