Update app.py

app.py

@@ -3,22 +3,57 @@ from transformers import T5ForConditionalGeneration, T5Tokenizer
 from sentence_transformers import SentenceTransformer
 import faiss
 import torch
+import numpy as np
+import wikipediaapi
+
+# Initialize Wikipedia API
+wiki_wiki = wikipediaapi.Wikipedia('en')
+
+# Function to fetch content from Wikipedia
+def fetch_wikipedia_articles(titles):
+    corpus = []
+    for title in titles:
+        page = wiki_wiki.page(title)
+        if page.exists():
+            corpus.append(page.text)
+        else:
+            st.write(f"Page for '{title}' does not exist.")
+    return corpus
+
+# Initialize SentenceTransformer for embeddings
+embedder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+
+# List of Wikipedia articles to retrieve
+titles = [
+    "Crypto",
+    "Finance",
+    "Technology",
+    "Healthcare",
+    "Education"
+]
+
+# Fetch and create the corpus
+st.write("Fetching Wikipedia articles...")
+corpus = fetch_wikipedia_articles(titles)
+
+# Generate embeddings for the corpus
+st.write("Generating embeddings...")
+embeddings = embedder.encode(corpus, convert_to_tensor=True)
+
+# Convert embeddings to NumPy array
+embeddings_np = embeddings.cpu().numpy()
+
+# Initialize FAISS index and add embeddings
+faiss_index = faiss.IndexFlatL2(embeddings_np.shape[1])
+faiss_index.add(embeddings_np)
 
 # Load model and tokenizer
 model_name = "google/flan-t5-base"
 model = T5ForConditionalGeneration.from_pretrained(model_name)
 tokenizer = T5Tokenizer.from_pretrained(model_name)
 
-# Define your sentence transformer model for the RAG approach
-embedder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
-
-# Build FAISS index (assuming you have precomputed embeddings for your retrieval corpus)
-# embeddings = ...  # Your precomputed embeddings go here
-# faiss_index = faiss.IndexFlatL2(embeddings.shape[1])
-# faiss_index.add(embeddings)
-
-# Define the Streamlit interface
-st.title("Humanized Text Generator")
+# Streamlit interface
+st.title("Humanized AI Text Generator")
 
 # Text input from the user
 user_input = st.text_area("Enter your query here", max_chars=2000)
@@ -28,14 +63,16 @@ if st.button("Generate Humanized Text"):
     if user_input:
         # Convert user input to embedding for retrieval
         query_embedding = embedder.encode([user_input], convert_to_tensor=True)
-        
-        # Retrieve the top k related documents from your FAISS index
-        # _, top_k_indices = faiss_index.search(query_embedding.cpu().numpy(), k=5)
-        
-        # Dummy document context (replace this with actual retrieved docs)
-        # context = retrieve_documents(top_k_indices)
-        context = "Sample context related to the query."  # For demonstration
-        
+
+        # Retrieve top 5 related documents from FAISS index
+        _, top_k_indices = faiss_index.search(query_embedding.cpu().numpy(), k=5)
+
+        # Retrieve documents based on top_k_indices
+        def retrieve_documents(top_k_indices):
+            return " ".join([corpus[i] for i in top_k_indices[0]])
+
+        context = retrieve_documents(top_k_indices)
+
         # Concatenate query and context
         input_text = f"{user_input} {context}"
 
@@ -50,25 +87,3 @@ if st.button("Generate Humanized Text"):
         st.write(generated_text)
     else:
         st.write("Please enter a query.")
-
-import faiss
-import numpy as np
-
-# Load your corpus embeddings
-# embeddings = np.load("embeddings.npy")
-
-# Initialize FAISS index and add the embeddings
-faiss_index = faiss.IndexFlatL2(embeddings.shape[1])  # Use L2 distance
-faiss_index.add(embeddings)
-
-# When you have a query, encode it and retrieve the top documents
-query_embedding = embedder.encode([user_input], convert_to_tensor=True)
-_, top_k_indices = faiss_index.search(query_embedding.cpu().numpy(), k=5)
-
-# Retrieve documents based on the top_k_indices
-def retrieve_documents(top_k_indices):
-    # Here, you would map the indices to the actual documents in your corpus
-    # This is just a placeholder
-    documents = ["Doc 1", "Doc 2", "Doc 3", "Doc 4", "Doc 5"]
-    return " ".join([documents[i] for i in top_k_indices[0]])
-