Update app.py

app.py

@@ -13,73 +13,17 @@ from transformers import AutoTokenizer
 from transformers import AutoModelForCausalLM
 from transformers import TextIteratorStreamer
 from threading import Thread
-from torchtext.data import to_map_style_dataset
 
 llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 tokenizer = AutoTokenizer.from_pretrained(llm_model)
 # pulling tokeinzer for text generation model
 
-#import numpy as np
-
-datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
-
-def is_iterable_dataset(datasetiter):
-  return isinstance(datasetiter, torch.utils.data.IterableDataset)
-
-def is_map_style_dataset(datasetiter):
-  return isinstance(datasetiter, torch.utils.data.Dataset)
-
-if is_iterable_dataset(datasetiter):
-  print("The datasetiter dataset is iterable-style.")
-else:
-  print("The datasetiter dataset is map-style.")
-
-
-
-from torch.utils.data import Dataset, IterableDataset
-
-class MyIterableDataset(IterableDataset):
-    def __init__(self, iterable):
-        super().__init__()
-        self.iterable = iterable
-
-    def __iter__(self):
-        return iter(self.iterable)
-
-class MapStyleDataset(Dataset):
-    def __init__(self, iterable):
-        super().__init__()
-        self.data = list(iterable)
-
-    def __len__(self):
-        return len(self.data)
-
-    def __getitem__(self, idx):
-        return self.data[idx]
-
-
-# Create an iterable
-#iterable = "Namitg02/Test"
-
-# Convert the iterable to a MapStyle dataset
-map_style_dataset = MapStyleDataset(iterable)
-
-# Create a DataLoader for the MapStyle dataset
-data_loader = torch.utils.data.DataLoader(map_style_dataset, batch_size=2)
-
-
-
-
-
-#datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
-#dataset = to_map_style_dataset(datasetiter)
-
-
+dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
 #dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
 #dataset = load_dataset("epfl-llm/guidelines", split='train')
 #Returns a list of dictionaries, each representing a row in the dataset.
-print(map_style_dataset[1])
-length = len(map_style_dataset)
+print(dataset[1])
+length = len(dataset)
 
 #Itemdetails = dataset.items()
 #print(Itemdetails)
@@ -91,18 +35,18 @@ embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 #doc_func = lambda x: x.text
 #dataset = list(map(doc_func, dataset))
 
-def embedder(map_style_dataset):
-    embeddings = embedding_model.encode(map_style_dataset["text"])
-    map_style_dataset = map_style_dataset.add_column('embeddings', embeddings)
-    return map_style_dataset
-updated_dataset = map_style_dataset.map(embedder)
+def embedder(dataset):
+    embeddings = embedding_model.encode(dataset["text"])
+    dataset = dataset.add_column('embeddings', embeddings)
+    return dataset
+updated_dataset = dataset.map(embedder)
 dataset['text'][:length]
 
 #print(embeddings)
 
 print(updated_dataset[1])
 print(updated_dataset[2])
-print(map_style_dataset[1])
+print(dataset[1])
 
 embedding_dim = embedding_model.get_sentence_embedding_dimension()
 #data = FAISS.from_embeddings(embed, embedding_model)