upload fine tune

fine-tune.py

@@ -0,0 +1,58 @@
+import os
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import DeepLake
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.document_loaders import SeleniumURLLoader
+from langchain import PromptTemplate
+from apscheduler.schedulers.background import BackgroundScheduler
+
+def embedding():
+    os.environ['OPENAI_API_KEY'] = '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'
+    os.environ["ACTIVELOOP_TOKEN"] = "eyJhbGciOiJIUzUxMiIsImlhdCI6MTY5NTE5MTAyNiwiZXhwIjoxNzU4MzQ5NDA3fQ.eyJpZCI6InV0a2Fyc2h0aXdhcmkifQ.PK_iz7uybeSmgqFvOYrICw-CQDbDY1aOjYhkMu-0Jle6gU33dCwxah7bmy39O0hPN4jYLu_RfLuU-XejyNvXrw"
+
+    #llm = ChatOpenAI(temperature=1.0, model='gpt-3.5-turbo-0613')
+
+    # URLs of articles to scrape
+    urls = ["https://modelwise.ai/product/","https://modelwise.ai/category/blog-articles/","https://modelwise.ai/company/"]
+
+    # Load documents using Selenium
+    loader = SeleniumURLLoader(urls=urls)
+    docs_not_splitted = loader.load()
+
+    # Split documents into smaller chunks
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    docs = text_splitter.split_documents(docs_not_splitted)
+
+    # Create OpenAIEmbeddings instance
+    embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+
+    # Specify your ActiveLoop organization ID
+    my_activeloop_org_id = "utkarshtiwari"
+    my_activeloop_dataset_name = "chatbot_modelwise"
+    dataset_path = f"hub://{my_activeloop_org_id}/{my_activeloop_dataset_name}"
+    db = DeepLake(dataset_path=dataset_path, embedding_function=embeddings)
+
+    # Add documents to the Deep Lake dataset
+    db.add_documents(docs)
+
+    template = """You are an exceptional customer support chatbot for the company Modelwise that gently answers questions related to the company.
+
+    You know the following context information.
+
+    {chunks_formatted}
+
+    Answer the following question from a customer. Use only information from the context. If you don't know the answer just ask the customer to contact Arnold and provide  his contact details. Do not make up any answer.
+
+    Question: {query}
+
+    Answer:"""
+
+    # Create a PromptTemplate instance
+    prompt = PromptTemplate(
+        input_variables=["chunks_formatted", "query"],
+        template=template,
+    )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(func=embedding, trigger="interval", seconds=60)
+scheduler.start()