Update app.py

app.py

@@ -1,34 +1,29 @@
-import openai
-from llama_index.query_engine.retriever_query_engine import RetrieverQueryEngine
-from llama_index.callbacks.base import CallbackManager
-from llama_index import (
-    LLMPredictor,
-    ServiceContext,
-    StorageContext,
-    load_index_from_storage,
-)
-from langchain.chat_models import ChatOpenAI
-import chainlit as cl
+import os
+import textwrap
+
+from llama_index import VectorStoreIndex, SimpleDirectoryReader, Document
+from llama_index.vector_stores import DeepLakeVectorStore
+import deeplake
+
+os.environ["OPENAI_API_KEY"] = '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'
+os.environ["ACTIVELOOP_TOKEN"] = "eyJhbGciOiJIUzUxMiIsImlhdCI6MTY5MzM2NTU0OSwiZXhwIjoxNjk2MDQzOTM5fQ.eyJpZCI6ImN4Y3h4YWFhYWF6In0.5syabu1oVCO1tzkDTFxU8SxFbYtkdzoerPSYebYeOpLGCeO2YrIQClCN02Ob-wEfunsei5evahzSS-KvFz79wg"
+
 
 
+from llama_index import SimpleDirectoryReader, Document, StorageContext, OpenAIEmbedding, ServiceContext, PromptHelper, VectorStoreIndex
+from llama_index.vector_stores import PineconeVectorStore, QdrantVectorStore, SimpleVectorStore, DeepLakeVectorStore
+from transformers import BertTokenizerFast
 import openai
-from llama_index import SimpleDirectoryReader, Document, StorageContext, OpenAIEmbedding, ServiceContext, PromptHelper
+from llama_index.llms import OpenAI
+from llama_index import ServiceContext
+from llama_index.embeddings import OpenAIEmbedding
 from llama_index.node_parser import SimpleNodeParser
 from llama_index.text_splitter import TokenTextSplitter
-from llama_index.indices.vector_store import VectorStoreIndex
-from llama_index.vector_stores import SupabaseVectorStore
-from llama_index.llms import OpenAI
-from llama_index import GPTVectorStoreIndex, StorageContext
-from dotenv import load_dotenv
-import os
-from llama_index import get_response_synthesizer
-from llama_index.vector_stores import DeepLakeVectorStore
-
 
-import deeplake
+openai.api_key = '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'
+from llama_index import StorageContext, load_index_from_storage
+from llama_index import load_index_from_storage, load_indices_from_storage, load_graph_from_storage
 
-os.environ["OPENAI_API_KEY"] = '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'
-os.environ["ACTIVELOOP_TOKEN"] = "eyJhbGciOiJIUzUxMiIsImlhdCI6MTY5MzM2NTU0OSwiZXhwIjoxNjk2MDQzOTM5fQ.eyJpZCI6ImN4Y3h4YWFhYWF6In0.5syabu1oVCO1tzkDTFxU8SxFbYtkdzoerPSYebYeOpLGCeO2YrIQClCN02Ob-wEfunsei5evahzSS-KvFz79wg"
 
 
 dataset_path ="hub://cxcxxaaaaaz/text_embedding" # if we comment this out and don't pass the path then GPTDeepLakeIndex will create dataset in memory
@@ -37,16 +32,53 @@ from llama_index import VectorStoreIndex, SimpleDirectoryReader, Document
 from llama_index.vector_stores import DeepLakeVectorStore
 
 # Create an index over the documnts
-vector_store = DeepLakeVectorStore(dataset_path=dataset_path,reset=True
+vector_store = DeepLakeVectorStore(dataset_path=dataset_path
                                     )
-service_context = ServiceContext.from_defaults()
+
 storage_context = StorageContext.from_defaults(vector_store=vector_store)
-                                    
-index = VectorStoreIndex.from_documents([], storage_context=storage_context)
 
 
 
-from llama_index import get_response_synthesizer
+llm = OpenAI(model='gpt-3.5-turbo', temperature=0, max_tokens=3924)
+embed_model = OpenAIEmbedding()
+node_parser = SimpleNodeParser(
+  text_splitter=TokenTextSplitter(chunk_size=3924, chunk_overlap=10)
+)
+prompt_helper = PromptHelper(
+  context_window=4096,
+  num_output=256,
+  chunk_overlap_ratio=0.1,
+  chunk_size_limit=20
+)
+
+from deeplake.core.vectorstore import VectorStore
+import tiktoken
+from llama_index.callbacks import CallbackManager, TokenCountingHandler
+from llama_index import load_index_from_storage, load_indices_from_storage, load_graph_from_storage
+
+token_counter = TokenCountingHandler(
+    tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode
+)
+
+callback_manager = CallbackManager([token_counter])
+
+
+service_context = ServiceContext.from_defaults(
+  llm=llm,
+  embed_model=embed_model,
+  node_parser=node_parser,
+  prompt_helper=prompt_helper, callback_manager=callback_manager
+)
+from llama_index import set_global_service_context
+#set_global_service_context(service_context)
+
+
+
+
+
+index = VectorStoreIndex.from_documents([], vectorstore=vector_store, storage_context=storage_context, service_context=service_context,show_progress=True)
+
+