File size: 5,209 Bytes
dbaa85f 320ed98 9850ed3 dbaa85f ed7660a dbaa85f ed7660a dbaa85f ed7660a dbaa85f ed7660a dbaa85f 9850ed3 dbaa85f ed7660a 9850ed3 6c564a4 9850ed3 6c564a4 9850ed3 ed7660a dbaa85f ed7660a dbaa85f 9850ed3 dbaa85f 320ed98 dbaa85f ed7660a dbaa85f ed7660a dbaa85f ed7660a dbaa85f 434f63b 14d2221 dbaa85f 434f63b 79cba27 dbaa85f 320ed98 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
from typing import Optional
import langchain
from dotenv import load_dotenv
from langchain_core.prompts import PromptTemplate
from langchain import chains
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from pydantic import ValidationError
from langchain.agents import AgentExecutor
from langchain.agents.mrkl.base import ZeroShotAgent
from prompts import FORMAT_INSTRUCTIONS, QUESTION_PROMPT, QUESTION_PROMPT1, SUFFIX
from tools import make_tools , drug_tools
import os
from rmrkl import ChatZeroShotAgent, RetryAgentExecutor
from langchain_ollama import OllamaLLM
import base64
from io import BytesIO
from PIL import Image
from langchain_openai import ChatOpenAI , OpenAI
from langchain.agents import load_tools, initialize_agent, AgentType
from langchain.llms import OpenAI
def convert_to_base64(pil_image):
buffered = BytesIO()
pil_image.save(buffered, format="PNG")
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
return img_str
def _make_llm(model, temp, api_key, streaming: bool = False):
if model.startswith("claude") :
llm = OpenAI(
temperature=temp,
model_name=model,
max_tokens = 5000,
openai_api_key=api_key,
base_url=os.getenv("OPENAI_API_BASE")
)
elif model.startswith("gpt") or model.startswith("deepseek"):
if os.getenv("OPENAI_API_BASE"):
llm = ChatOpenAI(model=model,
temperature = 0.1,
timeout=1000,
openai_api_key=api_key,base_url = os.getenv("OPENAI_API_BASE")
)
else:
llm = ChatOpenAI(model=model,
temperature = 0.1,
timeout=1000,
openai_api_key=api_key
)
elif model.startswith("llama") :
llm = OllamaLLM(model=model,
temperature = 0.1,
)
else:
raise ValueError(f"Invalid model name: {model}")
return llm
class TeLLAgent:
def __init__(
self,
tools=None,
model1: str = "deepseek-ai/DeepSeek-R1",
model2: str = "gpt-4o-2024-11-20",
tools_model="gpt-4o-2024-11-20",
temp=0.1,
max_iterations=50,
verbose=True,
streaming: bool = True,
openai_api_key= None,
api_keys: str = {},
file_path: str= r"...",
image_path: str = r"..."
):
"""Initialize agent."""
self.file_path = file_path
self.image_path = image_path
load_dotenv()
try:
self.llm1 = _make_llm(model1, temp, openai_api_key, streaming)
self.llm2 = _make_llm(model2, temp, openai_api_key, streaming)
except ValidationError:
raise ValueError("Invalid OpenAI API key")
if tools is None:
api_keys["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
tools_llm = _make_llm(tools_model, temp, openai_api_key, streaming)
tools = make_tools(tools_llm, api_keys=api_keys, verbose=verbose, image_path = image_path, file_path = file_path)
if tools == 'drug':
api_keys["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
tools_llm = _make_llm(tools_model, temp, openai_api_key, streaming)
tools = drug_tools(tools_llm, api_keys=api_keys, verbose=verbose, image_path = image_path, file_path = file_path)
# Initialize agent
self.agent_executor1 = RetryAgentExecutor.from_agent_and_tools(
tools=tools,
agent=ChatZeroShotAgent.from_llm_and_tools(
self.llm1,
tools,
suffix=SUFFIX,
format_instructions=FORMAT_INSTRUCTIONS,
question_prompt=QUESTION_PROMPT1, return_intermediate_steps=True ,handle_parsing_errors=True
),
verbose=True,
max_iterations=1 , return_intermediate_steps=True, handle_parsing_errors=True
)
self.agent_executor2 = RetryAgentExecutor.from_agent_and_tools(
tools=tools,
agent=ChatZeroShotAgent.from_llm_and_tools(
self.llm2,
tools,
suffix=SUFFIX,
format_instructions=FORMAT_INSTRUCTIONS,
question_prompt=QUESTION_PROMPT ,handle_parsing_errors=True
),
verbose=True,
max_iterations=max_iterations ,handle_parsing_errors=True
)
def run(self, prompt):
prompt = prompt + ' ' + str(self.file_path) + ' ' + str(self.image_path)
outputs = self.agent_executor1.invoke( {"input": prompt})
try:
prompt = str(' ' +outputs["input"]+ ' ' + outputs["output"].split('Action')[0].split('Final Answer')[0].replace("*", "") )
outputs = self.agent_executor2.invoke( {"input":prompt })
except:
prompt = str(' ' + outputs["input"] + ' ' + outputs["intermediate_steps"][0][0].log.split('Action')[0].replace("*", ""))
outputs = self.agent_executor2.invoke( {"input": prompt})
return outputs['output'] |