Spaces:
Build error
Build error
File size: 4,022 Bytes
fcb4136 ba5532a fcb4136 62c16a7 fcb4136 62c16a7 fcb4136 e43f2c6 fcb4136 1b72b3a fcb4136 ba5532a fcb4136 e43f2c6 fcb4136 1b72b3a fcb4136 e43f2c6 fcb4136 1b72b3a fcb4136 e43f2c6 fcb4136 ae86b65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | import os
import pickle
import tempfile
from langchain.document_loaders.csv_loader import CSVLoader
from langchain.vectorstores import FAISS
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.document_loaders import TextLoader
from langchain.text_splitter import CharacterTextSplitter
class Embedder_txt:
def __init__(self):
self.PATH = "embeddings"
self.createEmbeddingsDir()
def createEmbeddingsDir(self):
"""
Creates a directory to store the embeddings vectors
"""
if not os.path.exists(self.PATH):
os.mkdir(self.PATH)
def storeDocEmbeds(self, file, filename):
"""
Stores document embeddings using Langchain and FAISS
"""
# Write the uploaded file to a temporary file
with tempfile.NamedTemporaryFile(mode="wb", delete=False) as tmp_file:
tmp_file.write(file)
tmp_file_path = tmp_file.name
# Load the data from the file using Langchain
# loader = CSVLoader(file_path=tmp_file_path, encoding="utf-8")
documents = []
loader = TextLoader(file_path=tmp_file_path)
documents.extend(loader.load())
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
data = loader.load_and_split()
texts = text_splitter.split_documents(documents)
# Create an embeddings object using Langchain
embeddings = OpenAIEmbeddings()
# Store the embeddings vectors using FAISS
vectors = FAISS.from_documents(texts, embeddings)
os.remove(tmp_file_path)
# Save the vectors to a pickle file
with open(f"{self.PATH}/{filename}.pkl", "wb") as f:
pickle.dump(vectors, f)
def getDocEmbeds(self, file, filename):
"""
Retrieves document embeddings
"""
# Check if embeddings vectors have already been stored in a pickle file
if not os.path.isfile(f"{self.PATH}/{filename}.pkl"):
# If not, store the vectors using the storeDocEmbeds function
self.storeDocEmbeds(file, filename)
# Load the vectors from the pickle file
with open(f"{self.PATH}/{filename}.pkl", "rb") as f:
vectors = pickle.load(f)
return vectors
class Embedder:
def __init__(self):
self.PATH = "embeddings"
self.createEmbeddingsDir()
def createEmbeddingsDir(self):
"""
Creates a directory to store the embeddings vectors
"""
if not os.path.exists(self.PATH):
os.mkdir(self.PATH)
def storeDocEmbeds(self, file, filename):
"""
Stores document embeddings using Langchain and FAISS
"""
# Write the uploaded file to a temporary file
with tempfile.NamedTemporaryFile(mode="wb", delete=False) as tmp_file:
tmp_file.write(file)
tmp_file_path = tmp_file.name
# Load the data from the file using Langchain
loader = CSVLoader(file_path=tmp_file_path, encoding="utf-8")
data = loader.load_and_split()
# Create an embeddings object using Langchain
embeddings = OpenAIEmbeddings()
# Store the embeddings vectors using FAISS
vectors = FAISS.from_documents(data, embeddings)
os.remove(tmp_file_path)
# Save the vectors to a pickle file
with open(f"{self.PATH}/{filename}.pkl", "wb") as f:
pickle.dump(vectors, f)
def getDocEmbeds(self, file, filename):
"""
Retrieves document embeddings
"""
# Check if embeddings vectors have already been stored in a pickle file
if not os.path.isfile(f"{self.PATH}/{filename}.pkl"):
# If not, store the vectors using the storeDocEmbeds function
self.storeDocEmbeds(file, filename)
# Load the vectors from the pickle file
with open(f"{self.PATH}/{filename}.pkl", "rb") as f:
vectors = pickle.load(f)
return vectors |