Spaces:
No application file
No application file
File size: 2,998 Bytes
0bab228 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | from openai import OpenAI
import os
from dotenv import load_dotenv
import yaml
from langchain_openai import OpenAIEmbeddings
from pyprojroot import here
import shutil
load_dotenv()
class LoadConfig:
def __init__(self) -> None:
with open(here("configs/app_config.yml")) as cfg:
app_config = yaml.load(cfg, Loader=yaml.FullLoader)
# LLM configs
self.llm_engine = app_config["llm_config"]["engine"]
self.llm_system_role = app_config["llm_config"]["llm_system_role"]
self.persist_directory = str(here(
app_config["directories"]["persist_directory"]))
self.custom_persist_directory = str(here(
app_config["directories"]["custom_persist_directory"]))
# self.embedding_model = app_config["embedding_model_config"]["engine"]
self.embedding_model = OpenAIEmbeddings()
# Retrieval configs
self.data_directory = app_config["directories"]["data_directory"]
self.k = app_config["retrieval_config"]["k"]
self.embedding_model_engine = app_config["embedding_model_config"]["engine"]
self.chunk_size = app_config["splitter_config"]["chunk_size"]
self.chunk_overlap = app_config["splitter_config"]["chunk_overlap"]
# Summarizer config
self.max_final_token = app_config["summarizer_config"]["max_final_token"]
self.token_threshold = app_config["summarizer_config"]["token_threshold"]
self.summarizer_llm_system_role = app_config["summarizer_config"]["summarizer_llm_system_role"]
self.character_overlap = app_config["summarizer_config"]["character_overlap"]
self.final_summarizer_llm_system_role = app_config[
"summarizer_config"]["final_summarizer_llm_system_role"]
self.temperature = app_config["llm_config"]["temperature"]
# Memory
self.number_of_q_a_pairs = app_config["memory"]["number_of_q_a_pairs"]
# Load OpenAI credentials
self.load_openai_cfg()
# clean up the upload doc vectordb if it exists
self.create_directory(self.persist_directory)
self.remove_directory(self.custom_persist_directory)
def load_openai_cfg(self):
# openai.api_type = os.getenv("OPENAI_API_TYPE")
# openai.api_base = os.getenv("OPENAI_API_BASE")
# openai.api_version = os.getenv("OPENAI_API_VERSION")
OpenAI.api_key = os.getenv("OPENAI_API_KEY")
def create_directory(self, directory_path: str):
if not os.path.exists(directory_path):
os.makedirs(directory_path)
def remove_directory(self, directory_path: str):
if os.path.exists(directory_path):
try:
shutil.rmtree(directory_path)
print(
f"The directory '{directory_path}' has been successfully removed.")
except OSError as e:
print(f"Error: {e}")
else:
print(f"The directory '{directory_path}' does not exist.") |