Create app.py

app.py

@@ -0,0 +1,420 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import torch
+import json
+import os
+import glob
+from pathlib import Path
+from datetime import datetime
+import edge_tts
+import asyncio
+import base64
+import requests
+import plotly.graph_objects as go
+from gradio_client import Client
+from collections import defaultdict
+from bs4 import BeautifulSoup
+from audio_recorder_streamlit import audio_recorder
+import streamlit.components.v1 as components
+
+# Page configuration
+st.set_page_config(
+    page_title="Video Search & Research Assistant",
+    page_icon="🎥",
+    layout="wide",
+    initial_sidebar_state="auto",
+)
+
+# Initialize session state
+if 'search_history' not in st.session_state:
+    st.session_state['search_history'] = []
+if 'last_voice_input' not in st.session_state:
+    st.session_state['last_voice_input'] = ""
+if 'transcript_history' not in st.session_state:
+    st.session_state['transcript_history'] = []
+if 'should_rerun' not in st.session_state:
+    st.session_state['should_rerun'] = False
+
+# Custom styling
+st.markdown("""
+<style>
+    .main { background: linear-gradient(to right, #1a1a1a, #2d2d2d); color: #fff; }
+    .stMarkdown { font-family: 'Helvetica Neue', sans-serif; }
+    .stButton>button { margin-right: 0.5rem; }
+</style>
+""", unsafe_allow_html=True)
+
+# Initialize components
+speech_component = components.declare_component("speech_recognition", path="mycomponent")
+
+class VideoSearch:
+    def __init__(self):
+        self.text_model = SentenceTransformer('all-MiniLM-L6-v2')
+        self.load_dataset()
+        
+    def fetch_dataset_rows(self):
+        """Fetch dataset from Hugging Face API with debug and caching"""
+        try:
+            st.info("Fetching from Hugging Face API...")
+            url = "https://datasets-server.huggingface.co/first-rows?dataset=omegalabsinc%2Fomega-multimodal&config=default&split=train"
+            
+            response = requests.get(url, timeout=30)
+            st.write(f"Response status: {response.status_code}")
+            
+            if response.status_code == 200:
+                data = response.json()
+                
+                if 'rows' in data:
+                    # Extract actual row data from the nested structure
+                    processed_rows = []
+                    for row_data in data['rows']:
+                        if 'row' in row_data:  # Access the nested 'row' data
+                            processed_rows.append(row_data['row'])
+                    
+                    df = pd.DataFrame(processed_rows)
+                    
+                    # Debug output
+                    st.write("DataFrame columns after processing:", list(df.columns))
+                    st.write("Number of rows:", len(df))
+                    
+                    return df
+                else:
+                    st.error("No 'rows' found in API response")
+                    st.write("Raw API Response:", data)
+                    return self.load_example_data()
+            else:
+                st.error(f"API request failed with status code: {response.status_code}")
+                return self.load_example_data()
+                
+        except Exception as e:
+            st.error(f"Error fetching dataset: {str(e)}")
+            return self.load_example_data()
+
+    def load_example_data(self):
+        """Load example data as fallback"""
+        example_data = [
+            {
+                "video_id": "cd21da96-fcca-4c94-a60f-0b1e4e1e29fc",
+                "youtube_id": "IO-vwtyicn4",
+                "description": "This video shows a close-up of an ancient text carved into a surface, with the text appearing to be in a cursive script.",
+                "views": 45489,
+                "start_time": 1452,
+                "end_time": 1458,
+                "video_embed": [0.014160037972033024, -0.003111184574663639, -0.016604168340563774],
+                "description_embed": [-0.05835828185081482, 0.02589797042310238, 0.11952091753482819]
+            },
+            {
+                "video_id": "a8ebde7d-d717-4c1e-8be4-bdb4bc0c544f",
+                "youtube_id": "mo4rEyF7gTE",
+                "description": "This video shows a close-up view of a classical architectural structure, featuring stone statues with ornate details.",
+                "views": 4468,
+                "start_time": 318,
+                "end_time": 324,
+                "video_embed": [0.015160037972033024, -0.004111184574663639, -0.017604168340563774],
+                "description_embed": [-0.06835828185081482, 0.03589797042310238, 0.12952091753482819]
+            },
+            {
+                "video_id": "d1be64a6-22e2-4fbd-a176-20749e7c3d8a",
+                "youtube_id": "IO-vwtyicn4",
+                "description": "This video shows a weathered ancient painting depicting figures in classical style with vibrant colors preserved.",
+                "views": 45489,
+                "start_time": 1698,
+                "end_time": 1704,
+                "video_embed": [0.016160037972033024, -0.005111184574663639, -0.018604168340563774],
+                "description_embed": [-0.07835828185081482, 0.04589797042310238, 0.13952091753482819]
+            }
+        ]
+        return pd.DataFrame(example_data)
+    
+    def prepare_features(self):
+        """Prepare and cache embeddings"""
+        try:
+            if 'video_embed' not in self.dataset.columns:
+                st.warning("Using example data embeddings")
+                self.dataset = self.load_example_data()
+            
+            # Debug: Show raw data types and first row
+            st.write("Data Types:", self.dataset.dtypes)
+            st.write("\nFirst row of embeddings:")
+            st.write("video_embed type:", type(self.dataset['video_embed'].iloc[0]))
+            st.write("video_embed content:", self.dataset['video_embed'].iloc[0])
+            st.write("\ndescription_embed type:", type(self.dataset['description_embed'].iloc[0]))
+            st.write("description_embed content:", self.dataset['description_embed'].iloc[0])
+
+            # Convert string representations of embeddings back to numpy arrays
+            def safe_eval_list(s):
+                try:
+                    # Clean the string representation
+                    if isinstance(s, str):
+                        s = s.replace('[', '').replace(']', '').strip()
+                        # Split by whitespace and/or commas
+                        numbers = [float(x.strip()) for x in s.split() if x.strip()]
+                        return numbers
+                    elif isinstance(s, list):
+                        return [float(x) for x in s]
+                    else:
+                        st.error(f"Unexpected type for embedding: {type(s)}")
+                        return None
+                except Exception as e:
+                    st.error(f"Error parsing embedding: {str(e)}")
+                    st.write("Problematic string:", s)
+                    return None
+
+            # Process embeddings with detailed error reporting
+            video_embeds = []
+            text_embeds = []
+            
+            for idx in range(len(self.dataset)):
+                try:
+                    video_embed = safe_eval_list(self.dataset['video_embed'].iloc[idx])
+                    desc_embed = safe_eval_list(self.dataset['description_embed'].iloc[idx])
+                    
+                    if video_embed is not None and desc_embed is not None:
+                        video_embeds.append(video_embed)
+                        text_embeds.append(desc_embed)
+                    else:
+                        st.warning(f"Skipping row {idx} due to parsing failure")
+                except Exception as e:
+                    st.error(f"Error processing row {idx}: {str(e)}")
+                    st.write("Row data:", self.dataset.iloc[idx])
+
+            if video_embeds and text_embeds:
+                try:
+                    self.video_embeds = np.array(video_embeds)
+                    self.text_embeds = np.array(text_embeds)
+                    st.success(f"Successfully processed {len(video_embeds)} embeddings")
+                    st.write("Video embeddings shape:", self.video_embeds.shape)
+                    st.write("Text embeddings shape:", self.text_embeds.shape)
+                except Exception as e:
+                    st.error(f"Error converting to numpy arrays: {str(e)}")
+            else:
+                st.warning("No valid embeddings found, using random embeddings")
+                num_rows = len(self.dataset)
+                self.video_embeds = np.random.randn(num_rows, 384)
+                self.text_embeds = np.random.randn(num_rows, 384)
+            
+        except Exception as e:
+            st.error(f"Error preparing features: {str(e)}")
+            import traceback
+            st.write("Traceback:", traceback.format_exc())
+            # Create random embeddings as fallback
+            num_rows = len(self.dataset)
+            self.video_embeds = np.random.randn(num_rows, 384)
+            self.text_embeds = np.random.randn(num_rows, 384)
+
+    def load_dataset(self):
+        try:
+            self.dataset = self.fetch_dataset_rows()
+            if self.dataset is not None:
+                self.prepare_features()
+            else:
+                self.create_dummy_data()
+        except Exception as e:
+            st.error(f"Error loading dataset: {e}")
+            self.create_dummy_data()
+    
+    def prepare_features(self):
+        try:
+            self.video_embeds = np.array([json.loads(e) if isinstance(e, str) else e 
+                                        for e in self.dataset.video_embed])
+            self.text_embeds = np.array([json.loads(e) if isinstance(e, str) else e 
+                                       for e in self.dataset.description_embed])
+        except Exception as e:
+            st.error(f"Error preparing features: {e}")
+            num_rows = len(self.dataset)
+            self.video_embeds = np.random.randn(num_rows, 384)
+            self.text_embeds = np.random.randn(num_rows, 384)
+    
+    def search(self, query, top_k=5):
+        query_embedding = self.text_model.encode([query])[0]
+        
+        video_sims = cosine_similarity([query_embedding], self.video_embeds)[0]
+        text_sims = cosine_similarity([query_embedding], self.text_embeds)[0]
+        
+        combined_sims = 0.5 * video_sims + 0.5 * text_sims
+        top_indices = np.argsort(combined_sims)[-top_k:][::-1]
+        
+        results = []
+        for idx in top_indices:
+            results.append({
+                'video_id': self.dataset.iloc[idx]['video_id'],
+                'youtube_id': self.dataset.iloc[idx]['youtube_id'],
+                'description': self.dataset.iloc[idx]['description'],
+                'start_time': self.dataset.iloc[idx]['start_time'],
+                'end_time': self.dataset.iloc[idx]['end_time'],
+                'relevance_score': float(combined_sims[idx]),
+                'views': self.dataset.iloc[idx]['views']
+            })
+        return results
+
+def perform_arxiv_search(query, vocal_summary=True, extended_refs=False):
+    """Perform Arxiv search with audio summaries"""
+    try:
+        client = Client("awacke1/Arxiv-Paper-Search-And-QA-RAG-Pattern")
+        refs = client.predict(query, 20, "Semantic Search", 
+                            "mistralai/Mixtral-8x7B-Instruct-v0.1",
+                            api_name="/update_with_rag_md")[0]
+        response = client.predict(query, "mistralai/Mixtral-8x7B-Instruct-v0.1",
+                                True, api_name="/ask_llm")
+        
+        result = f"### 🔎 {query}\n\n{response}\n\n{refs}"
+        st.markdown(result)
+        
+        if vocal_summary:
+            audio_file = asyncio.run(generate_speech(response[:500]))
+            if audio_file:
+                st.audio(audio_file)
+                os.remove(audio_file)
+        
+        return result
+    except Exception as e:
+        st.error(f"Error in Arxiv search: {e}")
+        return None
+
+async def generate_speech(text, voice="en-US-AriaNeural"):
+    """Generate speech using Edge TTS"""
+    if not text.strip():
+        return None
+    
+    try:
+        communicate = edge_tts.Communicate(text, voice)
+        audio_file = f"speech_{datetime.now().strftime('%Y%m%d_%H%M%S')}.mp3"
+        await communicate.save(audio_file)
+        return audio_file
+    except Exception as e:
+        st.error(f"Error generating speech: {e}")
+        return None
+
+def process_audio_input(audio_bytes):
+    """Process audio input from recorder"""
+    if audio_bytes:
+        # Save temporary file
+        audio_path = f"temp_audio_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
+        with open(audio_path, "wb") as f:
+            f.write(audio_bytes)
+        
+        # Here you would typically use a speech-to-text service
+        # For now, we'll just acknowledge the recording
+        st.success("Audio recorded successfully!")
+        
+        # Cleanup
+        if os.path.exists(audio_path):
+            os.remove(audio_path)
+        
+        return True
+    return False
+
+def main():
+    st.title("🎥 Video Search & Research Assistant")
+    
+    # Initialize search
+    search = VideoSearch()
+    
+    # Create main tabs
+    tab1, tab2, tab3 = st.tabs(["🔍 Video Search", "🎙️ Voice & Audio", "📚 Arxiv Research"])
+    
+    with tab1:
+        st.subheader("Search Video Dataset")
+        
+        # Text search
+        query = st.text_input("Enter your search query:")
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            search_button = st.button("🔍 Search")
+        with col2:
+            num_results = st.slider("Number of results:", 1, 10, 5)
+            
+        if search_button and query:
+            results = search.search(query, num_results)
+            st.session_state['search_history'].append({
+                'query': query,
+                'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+                'results': results
+            })
+            
+            for i, result in enumerate(results, 1):
+                with st.expander(f"Result {i}: {result['description'][:100]}...", expanded=i==1):
+                    cols = st.columns([2, 1])
+                    
+                    with cols[0]:
+                        st.markdown(f"**Full Description:**")
+                        st.write(result['description'])
+                        st.markdown(f"**Time Range:** {result['start_time']}s - {result['end_time']}s")
+                        st.markdown(f"**Views:** {result['views']:,}")
+                    
+                    with cols[1]:
+                        st.markdown(f"**Relevance Score:** {result['relevance_score']:.2%}")
+                        if result['youtube_id']:
+                            st.video(f"https://youtube.com/watch?v={result['youtube_id']}&t={result['start_time']}")
+                        
+                        # Generate audio summary
+                        if st.button(f"🔊 Generate Audio Summary", key=f"audio_{i}"):
+                            summary = f"Video summary: {result['description'][:200]}"
+                            audio_file = asyncio.run(generate_speech(summary))
+                            if audio_file:
+                                st.audio(audio_file)
+                                os.remove(audio_file)
+
+    with tab2:
+        st.subheader("Voice Input & Audio Recording")
+        
+        col1, col2 = st.columns(2)
+        with col1:
+            st.write("🎙️ Speech Recognition")
+            voice_input = speech_component()
+            
+            if voice_input and voice_input != st.session_state['last_voice_input']:
+                st.session_state['last_voice_input'] = voice_input
+                st.markdown("**Transcribed Text:**")
+                st.write(voice_input)
+                
+                if st.button("🔍 Search Videos"):
+                    results = search.search(voice_input, num_results)
+                    for i, result in enumerate(results, 1):
+                        with st.expander(f"Result {i}", expanded=i==1):
+                            st.write(result['description'])
+                            if result['youtube_id']:
+                                st.video(f"https://youtube.com/watch?v={result['youtube_id']}&t={result['start_time']}")
+        
+        with col2:
+            st.write("🎵 Audio Recorder")
+            audio_bytes = audio_recorder()
+            if audio_bytes:
+                process_audio_input(audio_bytes)
+
+    with tab3:
+        st.subheader("Arxiv Research")
+        arxiv_query = st.text_input("🔍 Research Query:")
+        
+        col1, col2 = st.columns(2)
+        with col1:
+            vocal_summary = st.checkbox("Generate Audio Summary", value=True)
+        with col2:
+            extended_refs = st.checkbox("Include Extended References", value=False)
+            
+        if st.button("🔍 Search Arxiv") and arxiv_query:
+            perform_arxiv_search(arxiv_query, vocal_summary, extended_refs)
+
+    # Sidebar for history and settings
+    with st.sidebar:
+        st.subheader("⚙️ Settings & History")
+        
+        if st.button("🗑️ Clear History"):
+            st.session_state['search_history'] = []
+            st.experimental_rerun()
+        
+        st.markdown("### Recent Searches")
+        for entry in reversed(st.session_state['search_history'][-5:]):
+            st.markdown(f"**{entry['timestamp']}**: {entry['query']}")
+
+        st.markdown("### Voice Settings")
+        st.selectbox("TTS Voice:", 
+                    ["en-US-AriaNeural", "en-US-GuyNeural", "en-GB-SoniaNeural"],
+                    key="tts_voice")
+
+if __name__ == "__main__":
+    main()