Ross McNairn commited on
Commit ·
07dcd88
1
Parent(s): feb2d65
fmt
Browse files
hello_wordsmith/wordsmith.py
CHANGED
|
@@ -16,7 +16,6 @@ from llama_index.vector_stores.chroma import ChromaVectorStore
|
|
| 16 |
|
| 17 |
|
| 18 |
def initialize_chroma_db():
|
| 19 |
-
"""Initialize the ChromaDB client and collection"""
|
| 20 |
chroma_client = chromadb.EphemeralClient()
|
| 21 |
chroma_collection = chroma_client.create_collection("wordsmith")
|
| 22 |
vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
|
|
@@ -24,7 +23,6 @@ def initialize_chroma_db():
|
|
| 24 |
|
| 25 |
|
| 26 |
def setup_document_storage(vector_store):
|
| 27 |
-
"""Set up document storage and load data"""
|
| 28 |
package_directory = os.path.dirname(os.path.abspath(__file__))
|
| 29 |
dataset_path = os.path.join(package_directory, "public_wordsmith_dataset")
|
| 30 |
reader = SimpleDirectoryReader(input_dir=dataset_path)
|
|
@@ -35,7 +33,6 @@ def setup_document_storage(vector_store):
|
|
| 35 |
|
| 36 |
|
| 37 |
def initialize_llm():
|
| 38 |
-
"""Initialize the Large Language Model"""
|
| 39 |
llm = OpenAI(api_key=os.environ["OPENAI_API_KEY"], model="gpt-4")
|
| 40 |
return llm
|
| 41 |
|
|
@@ -77,7 +74,9 @@ _chat_template_messages = [
|
|
| 77 |
|
| 78 |
def configure_query_pipeline(index, llm):
|
| 79 |
"""Configure and set up the query pipeline"""
|
| 80 |
-
text_qa_chat_template = ChatPromptTemplate.from_messages(
|
|
|
|
|
|
|
| 81 |
query_pipeline = QueryPipeline()
|
| 82 |
|
| 83 |
retriever = index.as_retriever(similarity_top_k=5)
|
|
@@ -118,7 +117,9 @@ def main():
|
|
| 118 |
query_pipeline = configure_query_pipeline(index, llm)
|
| 119 |
ingestion_pipeline = IngestionPipeline(vector_store=vector_store)
|
| 120 |
rag_cli_instance = WordsmithRAGCLI(
|
| 121 |
-
ingestion_pipeline=ingestion_pipeline,
|
|
|
|
|
|
|
| 122 |
)
|
| 123 |
rag_cli_instance.cli()
|
| 124 |
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
def initialize_chroma_db():
|
|
|
|
| 19 |
chroma_client = chromadb.EphemeralClient()
|
| 20 |
chroma_collection = chroma_client.create_collection("wordsmith")
|
| 21 |
vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
def setup_document_storage(vector_store):
|
|
|
|
| 26 |
package_directory = os.path.dirname(os.path.abspath(__file__))
|
| 27 |
dataset_path = os.path.join(package_directory, "public_wordsmith_dataset")
|
| 28 |
reader = SimpleDirectoryReader(input_dir=dataset_path)
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
def initialize_llm():
|
|
|
|
| 36 |
llm = OpenAI(api_key=os.environ["OPENAI_API_KEY"], model="gpt-4")
|
| 37 |
return llm
|
| 38 |
|
|
|
|
| 74 |
|
| 75 |
def configure_query_pipeline(index, llm):
|
| 76 |
"""Configure and set up the query pipeline"""
|
| 77 |
+
text_qa_chat_template = ChatPromptTemplate.from_messages(
|
| 78 |
+
_chat_template_messages
|
| 79 |
+
)
|
| 80 |
query_pipeline = QueryPipeline()
|
| 81 |
|
| 82 |
retriever = index.as_retriever(similarity_top_k=5)
|
|
|
|
| 117 |
query_pipeline = configure_query_pipeline(index, llm)
|
| 118 |
ingestion_pipeline = IngestionPipeline(vector_store=vector_store)
|
| 119 |
rag_cli_instance = WordsmithRAGCLI(
|
| 120 |
+
ingestion_pipeline=ingestion_pipeline,
|
| 121 |
+
llm=llm,
|
| 122 |
+
query_pipeline=query_pipeline
|
| 123 |
)
|
| 124 |
rag_cli_instance.cli()
|
| 125 |
|