import chromadb from chromadb.config import Settings from sentence_transformers import SentenceTransformer import fitz #chroma_db PyMuPDF from transformers import AutoTokenizer import os import re import gradio as gr import os from flask import send_file import google.generativeai as genai import tempfile import shutil download_counts={} ## === CLOUD STORAGE URL MAPPING === # Add direct download links for each PDF file ARTICLE_URLS = { "Davit Marikyan-2019-Computing And Informatics-unified_theory_of_acceptance_and_use_of_technology.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETDFRinNWmxIo6Sri8r0tFQB3MbD3M6CcdRv5a8T5jlY-g?e=vbCFca", "enard Omallah George-2015-Computing And Informatics-Role_of_E_Resources_for_Research_Managemt.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ETbZst6ntatPm-pVPsOVA0YBFUpuV40t0H7c70NzqHrdRg?e=pT6Abp", "Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/EYl1sNw6NqlEqUtZcCidDdMBkxQJEjEnv8VHh2giPHXL_A?e=azCxFs", "Mahmood-2025-Education-revalence and Effects of Smartphone Use on Academic Performance of Undergraduate Student Nurses: An Analytical Cross-Sectional Study.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/ERgGX0tf7jpOhfy5bDhWDzkBeaxL4x1RPWfyN0sHDGnuvA?e=WNUZst", "Arana_Cedeño-2022-Social Science-Wars and other current challenges for health and life.pdf": "https://1drv.ms/b/c/78b29f6d9bcf3843/Eab88y2jonpMg9aSTfgkKxIBTFSIQRUJljeeG9SYhqbMiw?e=aAz1Bv", "A Arunprakash-2022-Education-Investigating Digital Education, A Study on Fostering Accessibility and Equity in Learning Environments.pdf:": "https://1drv.ms/b/c/78b29f6d9bcf3843/EZ1cpWglJ4NFhQZnd83Q4z8B2u0ihaSGaZILBdvCRBuJJg?e=S7PoZK", "Anyanwu, D.-2021-Computing And Informatics-Cybersecurity in the Age of the Internet of Things A Review of Challenges.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EaXYy_YTWUFPmTJp_IQuPO0BWUVes_to3J_IXYE8ZXUywg?e=mK1con", "Ashish Makanadar-2020-Computing And Informatics-Digital surveillance capitalism and citiesdata, democracy and activism.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ET4Pk0ndottEtwKUnRE18XoBqt-qG9N229QUALcjIQT0BQ?e=oW3vDN", "Benedict Dellot-2017-Social Science-RSA The Age Of Automation Report.pdf" : "", "Carmen G. Gonzalez-2023-Social Science-Climate Change, Race, and Migration.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EfyivsAY9XxLgnOJsK_bk5IBzakhpkyVrInyv71rdUKwZg?e=xftDHo", "D Paris-2020-Education-Culturally Sustaining Pedagogy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EbcH1oNsQS9FjD0rZ4xMv8kBFdhJ5n6rMSGh-eEE-g7t6w?e=UyBpgA", "Ahmad Samadi-2021-Education-How Can Support and Stability Prevent Teacher Burnout and Support (1).pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWMuLDLZEo5FgkTwfUTT0LMBiPJ2-A0_5N52RPEET_UGgQ?e=56zG3D", "Lesley Bartlett-2023-Education-Debating the “Science of Reading” and its Impact on Policy.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ESlCuNhXdwFBoJbmU4DbZ4IB902hJ1cg_BdE11W_CMdxmA?e=LkroLE", "M. SHELLEY THOMAS-2024-Education-Trauma-Informed Practices in Schools Across Two.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/ERT6CkW2gHZKlXPVcRuhTzABZ-vcpXdwdwFy89eUds6zBA?e=tysA4K", "Manuel Au-Yong-Oliveira-2021-Social Science-The Role of AI and Automation on the FutURE.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EXi3c_Vib75GgygLr9U04voB8vP2jZPN5RH_66-8Sgvb0w?e=6ZehVo", "Myra Marx Fettee-2022-Social Science-Inequality, Intersectionality and the Politics of Discourse.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EUToraqEywhLgAOfKFWx_jAB5JFgUweNo3QeEBjfTptniw?e=UZFKUu", "Paula Braveman-2023-Social Science-The Social Determinants.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EeKtmqDtD-xCq-zKv56M4g4B8mvZtzV4fmh3aLZufjp4jw?e=Htlay0", "Ruhee D’Cunha-2021-Computing And Informatics-Challenges in the use of quantum computing hardware-efficient Ans¨atze.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EdCAR3vs_NxFsCLdXLTKOLkBMbTfI0yEtyjn7MozZwcU5Q?e=4FwmQq", "Samuel M. Wilson-2022-Social Science-The Anthropology of Online Communities.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/Ed8oce6arD1Cp74j88qGCh8B4nYqMPc8fEik1DdzPHBBPw?e=iQeOW9", "Ullrich K. H. Ecker-2017Social Science-Why rebuttals may not work: the psychology of misinformation.pdf" : "https://1drv.ms/b/c/78b29f6d9bcf3843/EWNyYK5cK-9Fo5qve_EU4ccB_r6z5_D6s9fvT-zaG6ByNA?e=gjo72C" # Add all your files here with their direct cloud storage URLs } # Initialize tokenizer for chunking tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2") def extract_text_from_pdf(pdf_path): doc = fitz.open(pdf_path) full_text = "" print(f"Processing {os.path.basename(pdf_path)} - {len(doc)} pages") for page_num in range(len(doc)): page = doc.load_page(page_num) text = page.get_text() # Skip obviously preliminary pages (title, declarations, etc.) if page_num < 3: # First few pages are usually preliminaries if any(word in text.lower() for word in ['declaration', 'dedication', 'acknowledgement', 'table of content']): print(f"Skipping preliminary page {page_num + 1}") continue full_text += text + "\n" full_text = re.sub(r'\s+', ' ', full_text).strip() print(f"Extracted {len(full_text)} characters") return full_text def enhanced_extract_text_from_pdf(pdf_path): """ Enhanced PDF text extraction that handles various PDF structures """ doc = fitz.open(pdf_path) text = "" for page_num, page in enumerate(doc): # Extract text with different methods if needed page_text = page.get_text("text") # Check if this page has substantial content (not just headers/footers) if len(page_text.strip()) > 100: # Avoid empty or nearly empty pages text += f"\n--- Page {page_num + 1} ---\n{page_text}\n" # Alternative: try different extraction methods if len(text.strip()) < 500: # If too little text extracted print(f"Warning: Only {len(text)} characters extracted - trying alternative method") text = "" for page in doc: text += page.get_text("blocks") # Try blocks method text = re.sub(r'\s+', ' ', text).strip() return text def parse_metadata_from_filename(filename): """ Extracts metadata from filenames in various formats. Handles: Author_Name-Year-Department-Title_Keywords.pdf Also handles: Other_Formats-With-Different-Structures.pdf """ # Remove the .pdf extension name_without_ext = os.path.splitext(filename)[0] # Default metadata metadata = { "source": filename, "author": "Unknown Author", "year": "Unknown Year", "department": "General", "title": name_without_ext.replace('_', ' ').title() # Fallback title } # Split by hyphens to get the components parts = name_without_ext.split('-') # Different parsing strategies based on number of parts if len(parts) >= 4: # Format: Author-Year-Department-Title (most structured) metadata["author"] = parts[0].replace('_', ' ').title() metadata["year"] = parts[1] metadata["department"] = parts[2].replace('_', ' ').title() metadata["title"] = ' '.join(parts[3:]).replace('_', ' ').title() elif len(parts) == 3: # Format: Author-Year-Title or other 3-part formats # Check if the second part looks like a year (4 digits) if parts[1].isdigit() and len(parts[1]) == 4: metadata["author"] = parts[0].replace('_', ' ').title() metadata["year"] = parts[1] metadata["title"] = parts[2].replace('_', ' ').title() else: # Not a year, so probably Author-Department-Title metadata["author"] = parts[0].replace('_', ' ').title() metadata["department"] = parts[1].replace('_', ' ').title() metadata["title"] = parts[2].replace('_', ' ').title() elif len(parts) == 2: # Format: Author-Title or Year-Title if parts[0].isdigit() and len(parts[0]) == 4: metadata["year"] = parts[0] metadata["title"] = parts[1].replace('_', ' ').title() else: metadata["author"] = parts[0].replace('_', ' ').title() metadata["title"] = parts[1].replace('_', ' ').title() elif len(parts) == 1: # No hyphens, just use the whole filename as title metadata["title"] = name_without_ext.replace('_', ' ').title() # Clean up author name - remove "email" prefix if present if metadata["author"].lower().startswith("email"): metadata["author"] = metadata["author"][5:].strip() # Clean up any remaining underscores in all fields for key in ["author", "department", "title"]: if isinstance(metadata[key], str): metadata[key] = metadata[key].replace('_', ' ') return metadata def chunk_text(text, chunk_size=1000, overlap=100): """ Smart chunking that prioritizes main content """ tokens = tokenizer.encode(text) chunks = [] # Skip very short chunks (headers/footers) for i in range(0, len(tokens), chunk_size - overlap): chunk_tokens = tokens[i:i + chunk_size] if len(chunk_tokens) < 100: # Skip very short chunks continue chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True) # Skip chunks that are mostly preliminary content if is_preliminary_content(chunk_text): continue chunks.append(chunk_text) return chunks def is_preliminary_content(text): """ Identify and skip preliminary pages content """ preliminary_keywords = [ 'declaration', 'dedication', 'acknowledgement', 'table of content', 'abstract', 'chapter one', 'page', '©', 'all rights reserved' ] text_lower = text.lower() return any(keyword in text_lower for keyword in preliminary_keywords) def semantic_chunk_text(text, chunk_size=512): """ Chunk at sentence boundaries for better coherence """ import nltk nltk.download('punkt', quiet=True) from nltk.tokenize import sent_tokenize sentences = sent_tokenize(text) chunks = [] current_chunk = "" for sentence in sentences: if len(current_chunk) + len(sentence) < chunk_size: current_chunk += " " + sentence else: if current_chunk.strip(): chunks.append(current_chunk.strip()) current_chunk = sentence if current_chunk.strip(): chunks.append(current_chunk.strip()) return chunks def process_and_index_with_chunks(directory_path): """ Process documents and store them as chunks in ChromaDB """ documents = [] metadatas = [] ids = [] embeddings_list = [] for filename in os.listdir(directory_path): if filename.endswith(".pdf"): file_path = os.path.join(directory_path, filename) print(f"Processing {filename}...") # Extract full text text = extract_text_from_pdf(file_path) # Get metadata file_metadata = parse_metadata_from_filename(filename) file_metadata["source_file"] = filename file_metadata["full_text"] = text # Keep full text in metadata # Split into chunks chunks = chunk_text(text) print(f" Split into {len(chunks)} chunks") # Create embedding for each chunk and add to collection for i, chunk in enumerate(chunks): chunk_id = f"{filename}_chunk_{i}" embedding = model.encode(chunk).tolist() documents.append(chunk) embeddings_list.append(embedding) metadatas.append(file_metadata) # Same metadata for all chunks ids.append(chunk_id) # Add to ChromaDB collection.add( documents=documents, embeddings=embeddings_list, metadatas=metadatas, ids=ids ) print(f"✅ Indexed {len(documents)} chunks from {len(os.listdir(directory_path))} documents") def reindex_with_larger_chunks(): """ Re-index with larger chunk sizes for more content """ print("🔄 Re-indexing with larger chunks (1000 tokens)...") # Clear existing collection try: chroma_client.delete_collection("mk_library_doc_ceelt") except: pass global collection collection = chroma_client.create_collection(name="mk_library_doc_ceelt") # Process with larger chunks process_and_index_with_chunks("pdf_store") # This will use the updated chunk_size print("🎉 Re-indexed with larger chunks!") # --- RUN THIS ONCE TO POPULATE YOUR DATABASE --- def smart_chunk_text(text, chunk_size=1000, overlap=100): """ Smarter chunking that tries to preserve complete paragraphs """ # Split into paragraphs first paragraphs = [p for p in text.split('\n\n') if p.strip()] chunks = [] current_chunk = "" for paragraph in paragraphs: if len(current_chunk) + len(paragraph) < chunk_size: current_chunk += "\n\n" + paragraph else: if current_chunk.strip(): chunks.append(current_chunk.strip()) current_chunk = paragraph if current_chunk.strip(): chunks.append(current_chunk.strip()) return chunks model = SentenceTransformer('all-MiniLM-L6-v2') chroma_client = chromadb.PersistentClient(path="chroma_db") # === GEMINI API CONFIGURATION === GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "") if GEMINI_API_KEY: try: genai.configure(api_key=GEMINI_API_KEY) # Choose one of these working models: gemini_model = genai.GenerativeModel('models/gemini-1.5-pro-latest') # ✅ Best option # OR gemini_model = genai.GenerativeModel('models/gemini-1.5-flash-latest') # ✅ Faster option # OR gemini_model = genai.GenerativeModel('models/gemini-2.0-flash') # ✅ Good balance print("✅ Gemini API configured successfully") except Exception as e: print(f"❌ Gemini configuration failed: {e}") gemini_model = None else: gemini_model = None print("🔶 Gemini API not configured - using local responses") # Create (or get) a collection. Think of it as a table for your library data. collection = chroma_client.create_collection(name="mk_library_mor") # 🚨 RE-INDEXING STEP (RUN ONCE - THEN COMMENT OUT) print("Starting re-indexing with full text...") #reindex_all_documents() # This will recreate the collection with full text print("Re-indexing completed!") # Re-process just this problematic PDF def reprocess_specific_pdf(filename): pdf_path = os.path.join("pdf_store", filename) if os.path.exists(pdf_path): print(f"Re-processing: {filename}") # Remove existing entry from ChromaDB try: collection.delete(ids=[filename]) print(f"Removed old entry for {filename}") except: print(f"No existing entry to remove for {filename}") # Extract with enhanced method text = extract_text_from_pdf(pdf_path) print(f"Extracted text length: {len(text)}") if len(text) > 1000: # Get metadata metadata = parse_metadata_from_filename(filename) metadata["full_text"] = text # Create embedding and add to collection embedding = model.encode(text).tolist() doc_snippet = text[:1000] + "..." if len(text) > 1000 else text collection.add( documents=[doc_snippet], embeddings=[embedding], metadatas=[metadata], ids=[filename] ) print(f"Successfully re-indexed {filename}") return True else: print(f"Warning: Very little text extracted ({len(text)} chars)") return False return False # Run this for the problematic PDF reprocess_specific_pdf("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf") # Add this debug function to check what text was actually extracted def debug_pdf_text(pdf_filename): file_path = os.path.join("pdf_store", pdf_filename) if os.path.exists(file_path): full_text = extract_text_from_pdf(file_path) print(f"=== TEXT EXTRACTED FROM {pdf_filename} ===") print(full_text[:1000]) # First 1000 chars return full_text return None # Test with the specific PDF debug_pdf_text("Kamau M-2020-Education-Strategies Employed by Mount Kenya University to Achieve Competitive Advantage.pdf") pdf_store = "pdf_store" process_and_index_with_chunks(pdf_store) def semantic_search(query, n_results=5, year_filter=None, department_filter=None, author_filter=None): """Search that returns individual chunks""" query_embedding = model.encode([query]).tolist() # Build filter where_filter = {} if year_filter and year_filter != "All": where_filter["year"] = {"$gte": int(year_filter)} if department_filter and department_filter != "All": where_filter["department"] = {"$eq": department_filter} if author_filter and author_filter != "All": where_filter["author"] = {"$eq": author_filter} # Query ChromaDB results = collection.query( query_embeddings=query_embedding, n_results=n_results, where=where_filter if where_filter else None, include=['metadatas', 'documents', 'distances'] ) # Format results output = [] for meta, doc_text, distance in zip(results['metadatas'][0], results['documents'][0], results['distances'][0]): similarity_score = 1 - (distance / 2) output.append({ "title": meta.get('title', 'Research Document'), "author": meta.get('author', 'Unknown Author'), "year": meta.get('year', ''), "department": meta.get('department', 'General Studies'), "source": meta.get('source_file', ''), "content": doc_text, # This is now the actual chunk content "relevance": f"{similarity_score:.1%}", "chunk": True # Flag that this is a chunk }) return output def prepare_context(relevant_docs): """Prepare context from relevant documents""" context = "Based on the following research documents:\n\n" for i, doc in enumerate(relevant_docs, 1): context += f"Document {i}: {doc['title']} by {doc['author']} ({doc['year']})\n" context += f"Content: {doc['content'][:250]}...\n\n" return context def generate_contextual_response(question, relevant_docs): """Smart response with adaptive content length""" if not relevant_docs: return "🔍 **I couldn't find specific research on this topic.**" response = "**📚 Research Findings:**\n\n" for i, doc in enumerate(relevant_docs[:3], 1): content = doc['content'] # Show more content for highly relevant results if float(doc['relevance'].strip('%')) > 70: # Highly relevant preview = content[:800] + "..." if len(content) > 800 else content else: # Moderately relevant preview = content[:400] + "..." if len(content) > 400 else content response += f"**{i}. {doc['title']}**\n" response += f" 👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n" response += f" 📖 {preview}\n" response += f" 🎯 Relevance: {doc['relevance']}\n\n" response += "**📋 Source References:**\n" for doc in relevant_docs[:3]: response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n" return response def generate_local_response(question, relevant_docs): """Enhanced response formatting for better readability""" if not relevant_docs: return "🔍 **I couldn't find specific research on this topic.**\n\n**Try:**\n• Using broader search terms\n• Adjusting the filters\n• Asking about general research areas" # Start with a more natural introduction response = "**📚 I found some relevant research for you:**\n\n" for i, doc in enumerate(relevant_docs[:3], 1): # Create a cleaner, more readable snippet content = doc['content'] # Remove excessive whitespace and formatting issues content = re.sub(r'\s+', ' ', content).strip() # Create a better preview - focus on the actual content if len(content) > 120: # Try to find a complete sentence sentences = content.split('.') if len(sentences) > 1 and len(sentences[0]) > 20: preview = sentences[0] + '.' else: preview = content[:120] + '...' else: preview = content response += f"**{i}. {doc['title']}**\n" response += f" 👤 *{doc['author']}* ({doc['year']}) - {doc['department']}\n" response += f" 📖 {preview}\n\n" # Add more natural follow-up suggestions response += "**💡 You might want to ask:**\n" response += "• 'Can you tell me more about the first study?'\n" response += "• 'What methodology was used in this research?'\n" response += "• 'What were the main findings or conclusions?'\n" response += "• 'Are there similar studies on this topic?'" return response def create_text_snippet(text, max_words=10, query_terms=None): """ Creates a clean text snippet from the full text. - Shows the beginning of the content (not metadata like declarations) - Highlights query terms if provided - Limits to a specific number of words """ # Remove extra whitespace and make lowercase for processing clean_text = ' '.join(text.split()) # Try to find the actual content (skip declarations, acknowledgements, etc.) # Look for common academic document sections to find the main content content_starters = [ "abstract", "chapter", "introduction", "background", "this study", "research", "the purpose", "objective" ] # Find where the actual content begins content_start = 0 lower_text = clean_text.lower() for starter in content_starters: pos = lower_text.find(starter) if pos != -1 and (content_start == 0 or pos < content_start): content_start = pos # If we found a content start, use that section if content_start > 0: snippet_text = clean_text[content_start:] else: snippet_text = clean_text # Truncate to max_words words = snippet_text.split() if len(words) > max_words: snippet = ' '.join(words[:max_words]) + '...' else: snippet = snippet_text # Optional: Highlight query terms if provided if query_terms: for term in query_terms: if term.lower() in snippet.lower(): # Simple highlighting with HTML snippet = snippet.replace(term, f"{term}") snippet = snippet.replace(term.lower(), f"{term.lower()}") snippet = snippet.replace(term.upper(), f"{term.upper()}") return snippet def call_gemini_api(question, context): """More specific prompt for research objectives""" prompt = f"""As a research assistant, analyze this context to find the SPECIFIC RESEARCH OBJECTIVES. QUESTION: {question} CONTEXT EXCERPTS: {context} Instructions: 1. Look for sections titled: "Objectives", "Research Objectives", "Study Objectives" 2. If no specific objectives section, look for research goals or aims mentioned in introduction 3. If found, list the specific objectives clearly 4. If not found, state that objectives could not be located in the provided excerpts Provide a structured response:""" try: response = gemini_model.generate_content(prompt) return response.text except Exception as e: raise Exception(f"Gemini API call failed: {str(e)}") def chat_with_research(question, chat_history, year_filter="All", department_filter="All", author_filter="All"): if not question.strip(): return chat_history, "" try: # Find relevant research relevant_docs = semantic_search( question, n_results=3, year_filter=year_filter, department_filter=department_filter, author_filter=author_filter ) if not relevant_docs: response = "🔍 **I couldn't find specific research on this topic.**\n\n" response += "**Try:**\n• Using different keywords\n• Adjusting the filters\n• Asking about broader research areas" chat_history.append((question, response)) return chat_history, "" # === NEW: AI-GENERATED SUMMARY SECTION === if GEMINI_API_KEY and gemini_model: try: # Prepare context for AI summary context = "Research Context:\n" for i, doc in enumerate(relevant_docs[:3], 1): context += f"\nDocument {i}: {doc['title']} by {doc['author']} ({doc['year']})\n" context += f"Content: {doc['content'][:300]}...\n" # Generate AI summary summary_prompt = f"""Based on the following research excerpts, provide a concise summary (about 150 words) that directly answers this question: {question} {context} Instructions: - Provide a direct, comprehensive answer to the question - short in-text citation, APA 7 format - Write in a natural, conversational tone - Keep it around 150 words - Style: Professional academic tone, no references, direct answer only - Focus on the key insights from the research""" ai_response = gemini_model.generate_content(summary_prompt) summary = ai_response.text # Format the response with summary first, then references response = f"**🤖 AI Research Summary:**\n\n{summary}\n\n" response += "**📚 Source References:**\n" for doc in relevant_docs: response += f"• {doc['title']} by {doc['author']} ({doc['year']})\n" except Exception as e: print(f"AI summary failed: {e}") # Fallback to regular response response = generate_contextual_response(question, relevant_docs) else: # Local mode without AI summary response = generate_contextual_response(question, relevant_docs) except Exception as e: response = f"⚠️ **I encountered a technical issue**\n\nPlease try again or ask a different question.\n\n*Error: {str(e)}*" chat_history.append((question, response)) return chat_history, "" # Function to get unique values for dropdowns from the collection's metadata def get_unique_metadata_values(metadata_field): # Get all metadata (be cautious with very large collections) all_metadata = collection.get(include=['metadatas'])['metadatas'] # Extract the specific field, handling missing keys values = [meta.get(metadata_field, '') for meta in all_metadata] # Get unique, non-empty values and sort them unique_values = sorted(list(set(filter(None, values)))) return ["All"] + unique_values # Add "All" option # Fetch unique values for our filters (run this once when the app starts) unique_departments = get_unique_metadata_values('department') unique_authors = get_unique_metadata_values('author') # For years, we might just want a list of decades or a slider. Using a dropdown for simplicity. unique_years = sorted(list(set(meta.get('year', '2000') for meta in collection.get(include=['metadatas'])['metadatas']))) unique_years = ["All"] + unique_years def run_advanced_search(query, num_results, year_filter, department_filter, author_filter): results = semantic_search( query, n_results=num_results, year_filter=year_filter, department_filter=department_filter, author_filter=author_filter ) # Group results by document source grouped_results = {} for res in results: source = res['source'] if source not in grouped_results: grouped_results[source] = { 'title': res['title'], 'author': res['author'], 'year': res['year'], 'department': res['department'], 'source': res['source'], 'chunks': [], 'best_relevance': 0.0 # Store as float for comparison } grouped_results[source]['chunks'].append(res['content']) # Convert relevance percentage to float for comparison current_rel = grouped_results[source]['best_relevance'] new_rel = float(res['relevance'].strip('%')) / 100 # Convert "85.0%" to 0.85 if new_rel > current_rel: grouped_results[source]['best_relevance'] = new_rel # Convert back to percentage string for display for source in grouped_results: grouped_results[source]['relevance'] = f"{grouped_results[source]['best_relevance'] * 100:.1f}%" # Convert to list and sort by relevance unique_docs = list(grouped_results.values()) unique_docs.sort(key=lambda x: x['best_relevance'], reverse=True) # Now generate HTML output output_html = """
""" if not unique_docs: output_html += "

No results found matching your criteria.

" return output_html for res in unique_docs: # Create preview from chunks if res['chunks']: preview_text = " ".join([chunk[:200] for chunk in res['chunks'][:2]]) if len(preview_text) > 400: preview_text = preview_text[:400] + "..." if len(res['chunks']) > 2: preview_text += f" [+{len(res['chunks'])-2} more relevant sections]" else: preview_text = "No content preview available" cloud_url = ARTICLE_URLS.get(res['source'], "") if cloud_url: download_btn = f"""
📥 Download PDF
""" else: download_btn = """

⚠️ Download not available

""" output_html += f"""

{res['title']}

👤 Author: {res['author']} 📅 Year: {res['year']} 🏫 Department: {res['department']}
{download_btn}
🎯 Relevance: {res['relevance']}

📖 Preview:

{preview_text}

""" output_html += "
" return output_html # Create the advanced interface with dropdowns iface = gr.Interface( fn=run_advanced_search, inputs=[ gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."), gr.Dropdown(choices=unique_years, label="Published After Year", value="All"), gr.Dropdown(choices=unique_departments, label="Department", value="All"), gr.Dropdown(choices=unique_authors, label="Author", value="All") ], outputs=gr.HTML(label="Filtered Search Results"), title="🏛️ Mount Kenya University - Advanced Library Search", description="Find relevant resources using semantic search. Filter by year, department, or author to narrow down results." ) # Create the advanced interface with dropdowns AND result count control iface = gr.Interface( fn=run_advanced_search, inputs=[ gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture..."), gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results"), # Slider option # gr.Number(value=10, label="Number of Results", precision=0), # Number input option gr.Dropdown(choices=unique_years, label="Published After Year", value="All"), gr.Dropdown(choices=unique_departments, label="Department", value="All"), gr.Dropdown(choices=unique_authors, label="Author", value="All") ], outputs=gr.HTML(label="Filtered Search Results"), title="🏛️ Mount Kenya University - Advanced Library Search", description="Find relevant resources using semantic search. Choose how many results to see and filter by year, department, or author." ) # --- NEW: Create Tabbed Interface --- with gr.Blocks(title="MKU Smart Library Search", theme=gr.themes.Default()) as demo: gr.Markdown("# 🏛️ Mount Kenya University - Smart Library Search") gr.Markdown("Explore our academic resources through semantic search or chat with our research database.") file_download = gr.File(visible=False, label="Download PDF") # ====== HIDDEN COMPONENTS FOR DOCUMENT TRACKING ====== current_pdf_title = gr.Textbox(value="", visible=False) current_pdf_filename = gr.Textbox(value="", visible=False) # ====== END HIDDEN COMPONENTS ====== # Tab 1: Your Existing Semantic Search with gr.Tab("🔍 Semantic Search"): gr.Markdown("### Search our library collection with advanced filters") with gr.Row(): with gr.Column(): search_query = gr.Textbox(label="Your Research Question", placeholder="e.g., impact of climate change on agriculture...") num_results = gr.Slider(minimum=1, maximum=50, value=10, step=1, label="Number of Results") year_filter = gr.Dropdown(choices=unique_years, label="Published After Year", value="All") department_filter = gr.Dropdown(choices=unique_departments, label="Department", value="All") author_filter = gr.Dropdown(choices=unique_authors, label="Author", value="All") search_btn = gr.Button("Search", variant="primary") with gr.Column(): search_output = gr.HTML(label="Search Results") # Connect your existing function search_btn.click( fn=run_advanced_search, inputs=[search_query, num_results, year_filter, department_filter, author_filter], outputs=search_output ) #tab2 chat interface with gr.Tab("💬 Chat with Research"): gr.Markdown("## 🤖 Research Discussion Assistant") # === CHAT INTERFACE === with gr.Row(): with gr.Column(scale=3): chatbot = gr.Chatbot( label="Research Conversation", height=400, value=[ ("👋", "Hello! I can help you explore research papers.") ] ) with gr.Column(scale=1): gr.Markdown("### 💡 Chat Tips") gr.Markdown(""" **Use filters to:** - Focus on recent research - Explore specific departments - Find authors' work - Narrow down results """) # === MESSAGE INPUT === with gr.Row(): msg = gr.Textbox( label="Your research question", placeholder="e.g., 'What are recent findings about AI in education?'", scale=4 ) submit_btn = gr.Button("Send", variant="primary", scale=1) clear_btn = gr.Button("🔄 Clear Conversation") # === EVENT HANDLERS === submit_btn.click( chat_with_research, inputs=[msg, chatbot, year_filter, department_filter, author_filter], outputs=[chatbot, msg] ) msg.submit( chat_with_research, inputs=[msg, chatbot, year_filter, department_filter, author_filter], outputs=[chatbot, msg] ) clear_btn.click(lambda: [], None, chatbot) if __name__ == "__main__": # Display configuration status if GEMINI_API_KEY: print("✅ Hugging Face API enabled") else: print("🔶 Local mode - API token not set") print("💡 Get token: https://huggingface.co/settings/tokens") demo.launch( share=True, server_name="0.0.0.0", server_port=7860 )