Update app.py

app.py

@@ -1,22 +1,11 @@
-from getpass import getpass
-ACCESS_TOKEN = getpass(token = "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")
-base_url = "https://github.com/Namitg02/Diabeteschatbot"
-
 from datasets import load_dataset
-dataset = load_dataset("text",prompt= base_url, stream=None)
-print(dataset[1])
-
 from langchain.docstore.document import Document as LangchainDocument
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from sentence_transformers import SentenceTransformer
 from langchain_community.embeddings import HuggingFaceEmbeddings
-#from langchain_community.vectorstores import faiss
 import faiss
 from langchain.prompts import PromptTemplate
-#from langchain.chains import ConversationalRetrievalChain
-#from transformers import pipeline
-#from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
-#from langchain_core.messages import SystemMessage
+
 import time
 from transformers import AutoTokenizer
 from transformers import AutoModelForCausalLM
@@ -28,62 +17,45 @@ tokenizer = AutoTokenizer.from_pretrained(llm_model)
 # pulling tokeinzer for text generation model
 
 
-datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
+#datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
 dataset = list(datasetiter)
 #dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
-#dataset = load_dataset("epfl-llm/guidelines", split='train')
-
+dataset = load_dataset("epfl-llm/guidelines", split='train')
 #Returns a list of dictionaries, each representing a row in the dataset.
 print(dataset[1])
-#dataset.features
 length = len(dataset)
+
 #Itemdetails = dataset.items()
 #print(Itemdetails)
-#splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25) # ["\n\n", "\n", " ", ""])
 
-#docs = splitter.create_documents(str(dataset))
-
-# Returns a list of documents
-#print(docs)
 embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 #embedding_model = HuggingFaceEmbeddings(model_name = "mixedbread-ai/mxbai-embed-large-v1")
 #all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
-#docs_text = [doc.text for doc in docs]
-#embed = embedding_model.embed_documents(docs_text)
-
-#embeddings = embedding_model.encode(docs)
 
 #doc_func = lambda x: x.text
 #dataset = list(map(doc_func, dataset))
 
-def embedder(dataset):
-    embeddings = embedding_model.encode(dataset["text"])
-    dataset = dataset.add_column('embeddings', embeddings)
-    return dataset
-updated_dataset = dataset.map(embedder)
-dataset['text'][:length]
-
-print(embeddings)
+#def embedder(dataset):
+#    embeddings = embedding_model.encode(dataset["text"])
+#    dataset = dataset.add_column('embeddings', embeddings)
+#    return dataset
+#updated_dataset = dataset.map(embedder)
+#dataset['text'][:length]
 
-#def embedder(dataset[i]):
-#    return embedding_model.encode(dataset[i])
-    
-#dataset = dataset.map(embedder, batched=True)
-print(dataset[1])
-print(dataset[2])
-#embeddings = embedding_model.encode(dataset)
+#print(embeddings)
 
-#embeddings = embedding_model.embed_documents(docs)
+#print(updated_dataset[1])
+#print(updated_dataset[2])
+#print(dataset[1])
 
-embedding_dim = embedding_model.get_sentence_embedding_dimension()
-print(dataset[1])
+#embedding_dim = embedding_model.get_sentence_embedding_dimension()
 #data = FAISS.from_embeddings(embed, embedding_model)
 #data = FAISS.from_texts(docs, embedding_model)
 
 # Returns a FAISS wrapper vector store. Input is a list of strings. from_documents method used documents to Return VectorStore
 # add_embeddings
-#data = dataset["clean_text"]
-data = dataset["text"]
+data = dataset["clean_text"]
+#data = updated_dataset["text"]
 
 #print(data)
 d = 384  # vectors dimension