Upload 3 files

app.py

@@ -14,7 +14,6 @@ load_dotenv()
 # Function to initialize APIs
 def initialize_apis():
     if "openai_api_key" in st.session_state and "cohere_api_key" in st.session_state:
-        openai.api_key = st.session_state["openai_api_key"]
         co = cohere.Client(st.session_state["cohere_api_key"])
         index = helpers.initialize_pinecone(
             st.session_state["api_key"], st.session_state["env"], "coherererank", 1536
@@ -86,5 +85,3 @@ if all(
                     st.warning(error)
             else:
                 st.warning("Please enter a query.")
-                
-                

helpers.py

@@ -1,8 +1,16 @@
 import random
 import time
+import os
 
 import faker
-import openai
+from openai import OpenAI
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
 import pinecone
 import tqdm
 from datasets import Dataset
@@ -79,10 +87,10 @@ def create_dataset(num_resumes=1000, chunk_size=800):
 
 def embed(docs: list[str]) -> list[list[float]]:
     print("Embedding documents...")
-    res = openai.Embedding.create(input=docs, engine=embed_model)
+    res = client.embeddings.create(input=docs, model="text-embedding-3-small")
     print("Documents embedded successfully!")
-    return [x["embedding"] for x in res["data"]]
-
+    # Assuming the new API response object exposes the embedding directly
+    return [x.embedding for x in res.data]
 
 
 def insert_to_pinecone(index, dataset, batch_size=100):
@@ -117,12 +125,12 @@ def insert_to_pinecone(index, dataset, batch_size=100):
 
     print("New data inserted to Pinecone successfully!")
 
-   
+
 def get_docs(index, query: str, top_k: int):
     print("Fetching documents from Pinecone...")
     xq = embed([query])[0]
     res = index.query(xq, top_k=top_k, include_metadata=True)
-    docs = {x["metadata"]["text"]: i for i, x in enumerate(res["matches"])}
+    docs = {x["metadata"]["text"]: i for i, x in enumerate(res.matches)}
     print("Documents fetched successfully!")
     return docs
 
@@ -131,7 +139,7 @@ def compare(index, co, query, top_k=25, top_n=3):
     # Get vec search results
     docs = get_docs(index, query, top_k=top_k)
     i2doc = {docs[doc]: doc for doc in docs.keys()}
-    
+
     # Re-rank
     rerank_docs = co.rerank(
         query=query,
@@ -139,18 +147,20 @@ def compare(index, co, query, top_k=25, top_n=3):
         top_n=top_n,
         model="rerank-english-v2.0",
     )
-    
+
     comparison_data = []
     # Compare order change
     for i, doc in enumerate(rerank_docs):
         rerank_i = docs[doc.document["text"]]
-        
-        comparison_data.append({
-            'Original Rank': i,
-            'Original Text': i2doc[i],
-            'Reranked Rank': rerank_i,
-            'Reranked Text': doc.document['text']
-        })
+
+        comparison_data.append(
+            {
+                "Original Rank": i,
+                "Original Text": i2doc[i],
+                "Reranked Rank": rerank_i,
+                "Reranked Text": doc.document["text"],
+            }
+        )
     return comparison_data
 
 

requirements.txt

@@ -63,7 +63,6 @@ jupyter_core==5.4.0
 langchain==0.0.325
 langsmith==0.0.53
 Levenshtein==0.23.0
-llama-index==0.8.53.post3
 loguru==0.7.2
 markdown-it-py==3.0.0
 MarkupSafe==2.1.3
@@ -76,7 +75,7 @@ mypy-extensions==1.0.0
 nest-asyncio==1.5.8
 nltk==3.8.1
 numpy==1.26.1
-openai==0.28.1
+openai==0.28.0
 openpyxl==3.1.2
 packaging==23.2
 pandas==2.1.2