| import gradio as gr |
|
|
| from langchain_community.llms import LlamaCpp |
| from langchain.prompts import PromptTemplate |
| from langchain.chains import LLMChain |
| from langchain_core.callbacks import StreamingStdOutCallbackHandler |
|
|
|
|
| callbacks = [StreamingStdOutCallbackHandler()] |
|
|
| llm = LlamaCpp( |
| model_path="/content/drive/MyDrive/models/demo1/unsloth.Q5_K_M.gguf", |
| n_gpu_layers=40, |
| n_batch=512, |
| callbacks=callbacks, |
| verbose=True, |
| ) |
|
|
| template = """You are the Finiantial expert: |
| |
| ### Instruction: |
| {question} |
| |
| ### Input: |
| |
| |
| ### Response: |
| """ |
|
|
| prompt = PromptTemplate(template=template, input_variables=["question"]) |
|
|
| llm_chain_model = LLMChain(prompt=prompt, llm=llm) |
|
|
|
|
| def greet(question): |
| out_gen = llm_chain_model.run(question) |
| return out_gen |
|
|
| demo = gr.Interface(fn=greet, inputs="text", outputs="text") |
| demo.launch() |
