Update app.py

app.py

@@ -6,31 +6,17 @@ from langchain_community.vectorstores.faiss import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 import os
-import json
 from langchain_groq import ChatGroq
 from langchain.chains.qa_with_sources.retrieval import RetrievalQAWithSourcesChain
 from langchain.prompts import PromptTemplate
 from bs4 import SoupStrainer
-from youtube_transcript_api import YouTubeTranscriptApi, TranscriptsDisabled, NoTranscriptFound
-import yt_dlp
-import re
-from googleapiclient.discovery import build
-from googleapiclient.errors import HttpError
-from google_auth_oauthlib.flow import InstalledAppFlow
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
+import PyPDF2
 
 # Load environment variables (optional)
 load_dotenv()
 
 # Hardcoded Groq API key 
 GROQ_API_KEY = "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"
-# YouTube API key (to be set in Hugging Face Spaces secrets, optional if using OAuth)
-YOUTUBE_API_KEY = os.getenv("YOUTUBE_API_KEY")
-
-# Path to store OAuth credentials
-CREDENTIALS_FILE = "youtube_credentials.json"
-CLIENT_SECRETS_FILE = "client_secrets.json"
 
 # Custom CSS
 st.markdown("""
@@ -138,20 +124,20 @@ if "llm" not in st.session_state:
         max_tokens=512  # Keep reduced to minimize resource usage
     )
 
-# Sidebar for URL and YouTube input
+# Sidebar for URL and PDF input
 with st.sidebar:
     st.header("Enter Web URL")
     url = st.text_input("URL", placeholder="e.g., https://mahatirtusher.com/astronomy-mythology/")
     process_url_clicked = st.button("Process URL")
 
-    st.header("Enter YouTube URL")
-    youtube_url = st.text_input("YouTube URL", placeholder="e.g., https://www.youtube.com/watch?v=DJO_9auJhJQ")
-    process_youtube_clicked = st.button("Process YouTube Video")
+    st.header("Upload PDF File")
+    pdf_file = st.file_uploader("Upload a PDF", type=["pdf"])
+    process_pdf_clicked = st.button("Process PDF")
 
 # Main content container
 main_container = st.container()
 
-# Custom prompt for detailed answers (for web URLs only)
+# Custom prompt for detailed answers
 qa_prompt = PromptTemplate(
     template="""You are an expert assistant tasked with providing detailed, extensive, and comprehensive answers. Use the provided context to answer the question thoroughly, including explanations, examples, and additional relevant information. If the context is limited, expand on the topic with your knowledge to ensure a complete response. In case of explaining anything, break the topic and explain step by step. Sometimes use your own reasoning and knowledge to explain anything to the users. If the users ask any question in Bengali, you too will answer it in fine and detailed Bengali.
 
@@ -163,17 +149,9 @@ Answer with sources: """
 )
 
 # Function to summarize content
-def summarize_content(content, llm, is_youtube=False):
-    if is_youtube:
-        # Extensive summary for YouTube videos (15-20 sentences)
-        summary_prompt = f"""You are an expert summarizer tasked with providing a very detailed and extensive summary of the following YouTube video transcript. Capture all key points, main ideas, and significant details in 15-20 sentences. Include specific examples, quotes, or moments from the transcript to make the summary comprehensive and vivid. Ensure the summary is well-organized, flowing naturally from one point to the next, and provides a thorough overview of the video's content.
-
-Transcript: {content}
-
-Extensive Summary: """
-    else:
-        # Shorter summary for web URLs (5-10 sentences)
-        summary_prompt = f"""Summarize the following content in 5-10 sentences, capturing the main points and key details in easy expression:
+def summarize_content(content, llm):
+    # Shorter summary for web URLs and PDFs (5-10 sentences)
+    summary_prompt = f"""Summarize the following content in 5-10 sentences, capturing the main points and key details in easy expression:
 
 {content}
 
@@ -181,228 +159,21 @@ Summary: """
     summary = llm.invoke(summary_prompt).content
     return summary
 
-# Function to extract YouTube video ID from URL
-def get_video_id(url):
-    if "youtube.com/watch?v=" in url:
-        return url.split("v=")[1].split("&")[0]
-    elif "youtu.be/" in url:
-        return url.split("youtu.be/")[1].split("?")[0]
-    return None
-
-# Function to fetch YouTube transcript
-def fetch_youtube_transcript(video_id):
+# Function to extract text from PDF
+def extract_text_from_pdf(pdf_file):
     try:
-        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
-        # Try English variants first
-        for lang in ['en', 'en-US', 'en-GB']:
-            try:
-                transcript = transcript_list.find_transcript([lang]).fetch()
-                full_text = " ".join([item['text'] for item in transcript])
-                return full_text
-            except NoTranscriptFound:
-                continue
-        
-        # If no English transcript, try any available transcript and translate to English
-        for transcript in transcript_list:
-            if transcript.is_translatable:
-                translated_transcript = transcript.translate('en').fetch()
-                return " ".join([item['text'] for item in translated_transcript])
-        
-        return None
-    except TranscriptsDisabled:
-        return None
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page in pdf_reader.pages:
+            page_text = page.extract_text()
+            if page_text:
+                text += page_text + "\n"
+        return text
     except Exception as e:
-        st.error(f"Error fetching transcript with youtube-transcript-api: {str(e)}")
-        return None
-
-# Function to get YouTube API credentials
-def get_youtube_credentials():
-    creds = None
-    if os.path.exists(CREDENTIALS_FILE):
-        creds = Credentials.from_authorized_user_file(CREDENTIALS_FILE, scopes=['https://www.googleapis.com/auth/youtube.force-ssl'])
-    
-    if not creds or not creds.valid:
-        if creds and creds.expired and creds.refresh_token:
-            creds.refresh(Request())
-        else:
-            if os.path.exists(CLIENT_SECRETS_FILE):
-                st.warning("Attempting to authenticate with YouTube Data API. This may not work in Hugging Face Spaces due to redirect URI limitations.")
-                flow = InstalledAppFlow.from_client_secrets_file(
-                    CLIENT_SECRETS_FILE,
-                    scopes=['https://www.googleapis.com/auth/youtube.force-ssl']
-                )
-                # This will fail in Hugging Face Spaces because it can't open a browser
-                creds = flow.run_local_server(port=0)
-                with open(CREDENTIALS_FILE, 'w') as token_file:
-                    token_file.write(creds.to_json())
-            else:
-                st.warning(
-                    f"{CLIENT_SECRETS_FILE} not found. To use OAuth 2.0 for YouTube Data API:\n"
-                    "1. Go to https://console.developers.google.com/.\n"
-                    "2. Create a project, enable YouTube Data API v3, and create OAuth 2.0 credentials.\n"
-                    "3. Download the credentials as 'client_secrets.json'.\n"
-                    "4. Run the app locally: pip install -r requirements.txt && streamlit run app.py\n"
-                    "5. Authenticate via the browser prompt to generate youtube_credentials.json.\n"
-                    "6. Upload youtube_credentials.json to your Hugging Face Space via the Files tab."
-                )
-                return None
-    
-    return creds
-
-# Function to fetch captions using YouTube Data API (with OAuth 2.0 or API key fallback)
-def fetch_youtube_captions_api(video_id, api_key=None):
-    # First, try OAuth 2.0 if credentials are available
-    creds = get_youtube_credentials()
-    if creds:
-        try:
-            youtube = build('youtube', 'v3', credentials=creds)
-            captions = youtube.captions().list(
-                part='snippet',
-                videoId=video_id
-            ).execute()
-
-            caption_id = None
-            for item in captions.get('items', []):
-                if item['snippet']['language'] == 'en':
-                    caption_id = item['id']
-                    break
-                elif item['snippet']['language'] in ['en-US', 'en-GB']:
-                    caption_id = item['id']
-                    break
-
-            if not caption_id:
-                st.warning("No English captions found via YouTube Data API.")
-                return None
-
-            # Download captions using OAuth 2.0 credentials
-            caption_content = youtube.captions().download(
-                id=caption_id,
-                tfmt='srt'
-            ).execute()
-
-            # The response is a binary string, decode it
-            caption_text = caption_content.decode('utf-8')
-            # Parse SRT format to extract text
-            lines = caption_text.split('\n')
-            text_lines = []
-            for line in lines:
-                if line.strip() and not line.isdigit() and not re.match(r'\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}', line):
-                    text_lines.append(line.strip())
-            
-            return " ".join(text_lines)
-
-        except HttpError as e:
-            st.error(f"Error fetching captions with YouTube Data API (OAuth 2.0): {str(e)}")
-            return None
-
-    # Fallback to API key if OAuth fails or credentials are not available
-    if not api_key:
-        st.warning("YOUTUBE_API_KEY not set and OAuth 2.0 credentials not available. Skipping YouTube Data API fallback.")
-        return None
-    try:
-        youtube = build('youtube', 'v3', developerKey=api_key)
-        captions = youtube.captions().list(
-            part='snippet',
-            videoId=video_id
-        ).execute()
-
-        caption_id = None
-        for item in captions.get('items', []):
-            if item['snippet']['language'] == 'en':
-                caption_id = item['id']
-                break
-            elif item['snippet']['language'] in ['en-US', 'en-GB']:
-                caption_id = item['id']
-                break
-
-        if not caption_id:
-            st.warning("No English captions found via YouTube Data API.")
-            return None
-
-        # Note: Downloading captions requires OAuth 2.0 authentication
-        st.warning(
-            "English captions are available for this video but cannot be fetched with an API key alone. "
-            "Downloading captions requires OAuth 2.0 authentication, which is not supported in Hugging Face Spaces without user interaction. "
-            "To fetch captions:\n"
-            "- Follow the instructions above to generate youtube_credentials.json locally and upload it.\n"
-            "- Or try a video with transcripts available (e.g., https://www.youtube.com/watch?v=dQw4w9WgXcQ)."
-        )
+        st.error(f"Error extracting text from PDF: {str(e)}")
         return None
 
-    except HttpError as e:
-        st.error(f"Error fetching captions with YouTube Data API (API Key): {str(e)}")
-        return None
-
-# Function to extract subtitles using yt-dlp with cookies
-def extract_subtitles_with_ytdlp(video_url):
-    ydl_opts = {
-        'writesubtitles': True,
-        'writeautomaticsub': True,
-        'subtitleslangs': ['all', '-live_chat'],
-        'skip_download': True,
-        'subtitlesformat': 'vtt',
-        'outtmpl': 'subtitle.%(ext)s',
-        'http_headers': {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-        },
-        'cookiefile': 'cookies.txt',
-    }
-    try:
-        if not os.path.exists('cookies.txt'):
-            st.error(
-                "cookies.txt file not found. Please upload a valid cookies.txt file to the root directory of your Space. "
-                "To generate it:\n"
-                "1. Open Chrome and log in to YouTube.\n"
-                "2. Install the 'Export Cookies' extension (or use a tool like 'cookies.txt' for Firefox).\n"
-                "3. Export cookies for 'youtube.com' and save as 'cookies.txt'.\n"
-                "4. Upload the file to your Space via the Files tab.\n"
-                "Alternative: If this fails, test locally to rule out Spaces IP restrictions."
-            )
-            return None
-
-        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-            info = ydl.extract_info(video_url, download=False)
-            available_subs = info.get('subtitles', {})
-            auto_subs = info.get('automatic_captions', {})
-
-            subtitle_langs = list(available_subs.keys()) or list(auto_subs.keys())
-            if not subtitle_langs:
-                st.warning("No subtitles or auto-captions available in any language.")
-                return None
-
-            ydl.params['subtitleslangs'] = subtitle_langs
-            ydl.download([video_url])
-
-        subtitle_file = None
-        for lang in subtitle_langs:
-            possible_file = f"subtitle.{lang}.vtt"
-            if os.path.exists(possible_file):
-                subtitle_file = possible_file
-                break
-
-        if not subtitle_file:
-            st.warning("No subtitle files were downloaded.")
-            return None
-
-        with open(subtitle_file, 'r', encoding='utf-8') as f:
-            subtitle_text = f.read()
-
-        os.remove(subtitle_file)
-
-        lines = subtitle_text.split('\n')
-        text_lines = []
-        for line in lines:
-            if line.strip() and not line.startswith('WEBVTT') and not line.startswith('Kind:') and not line.startswith('Language:') and not re.match(r'\d{2}:\d{2}:\d{2}\.\d{3} --> \d{2}:\d{2}:\d{2}\.\d{3}', line):
-                text_lines.append(line.strip())
-
-        return " ".join(text_lines)
-    except Exception as e:
-        st.error(f"Error fetching captions with yt-dlp: {str(e)}")
-        return None
-
-# Function to process and chunk text (for web URLs only)
+# Function to process and chunk text
 def process_content(text, embeddings, source):
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=1000,
@@ -416,7 +187,7 @@ def process_content(text, embeddings, source):
     vectorstore = FAISS.from_documents(docs, embeddings)
     return vectorstore
 
-# Function to create QA chain (for web URLs only)
+# Function to create QA chain
 def create_qa_chain(vectorstore, llm):
     if vectorstore is None:
         st.error("Vector store is not initialized. Cannot create QA chain.")
@@ -468,68 +239,41 @@ if process_url_clicked:
                     st.error(f"Error processing URL: {str(e)}")
                     st.stop()
 
-# Process YouTube Video
-if process_youtube_clicked:
+# Process PDF File
+if process_pdf_clicked:
     with main_container:
-        if not youtube_url.strip():
-            st.error("Please provide a valid YouTube URL.")
+        if not pdf_file:
+            st.error("Please upload a PDF file.")
         else:
-            with st.spinner("Processing YouTube Video..."):
+            with st.spinner("Processing PDF..."):
                 try:
-                    video_id = get_video_id(youtube_url)
-                    if not video_id:
-                        st.error("Invalid YouTube URL. Please provide a URL like https://www.youtube.com/watch?v=VIDEO_ID.")
-                        st.stop()
-
-                    transcript_text = None
-                    st.text("Fetching Transcript...Started...✅✅✅")
-                    transcript_text = fetch_youtube_transcript(video_id)
-
-                    if not transcript_text:
-                        st.warning("Transcripts are disabled or unavailable. Attempting to fetch closed captions...")
-                        st.text("Fetching Closed Captions with yt-dlp...Started...✅✅✅")
-                        transcript_text = extract_subtitles_with_ytdlp(youtube_url)
-
-                        if not transcript_text:
-                            st.text("Fetching Captions via YouTube Data API...Started...✅✅✅")
-                            transcript_text = fetch_youtube_captions_api(video_id, YOUTUBE_API_KEY)
-
-                        if not transcript_text:
-                            st.error(
-                                "No transcripts or closed captions available. "
-                                "Possible reasons:\n"
-                                "1. Captions are not enabled for this video.\n"
-                                "2. YouTube detected this request as a bot (even with cookies.txt).\n"
-                                "Solutions:\n"
-                                "- Ensure captions are enabled for the video by checking the video settings on YouTube (gear icon > Subtitles/CC > Enable if available).\n"
-                                "- Regenerate and upload a fresh cookies.txt file (see instructions above).\n"
-                                "- Set up OAuth 2.0 credentials by following the instructions above to download captions directly.\n"
-                                "- Try a different video (e.g., https://www.youtube.com/watch?v=dQw4w9WgXcQ, which has transcripts available).\n"
-                                "- Test locally to rule out Hugging Face Spaces IP restrictions by running: pip install -r requirements.txt && streamlit run app.py"
-                            )
-                            st.stop()
+                    st.text("Extracting Text from PDF...Started...✅✅✅")
+                    pdf_text = extract_text_from_pdf(pdf_file)
 
-                    if not transcript_text.strip():
-                        st.error("Transcript or captions are empty. Try a different video.")
+                    if not pdf_text:
+                        st.error("No text could be extracted from the PDF. Try a different file.")
                         st.stop()
 
-                    st.session_state.url_content = transcript_text
-                    # No vector store for YouTube videos since we're not doing QA
-                    st.session_state.vectorstore = None
-                    st.session_state.index_created = False
-                    st.session_state.content_type = "youtube"
+                    # Initialize embeddings only when needed
+                    if "embeddings" not in st.session_state:
+                        st.session_state.embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+                    st.session_state.url_content = pdf_text
+                    embeddings = st.session_state.embeddings
+                    st.session_state.vectorstore = process_content(st.session_state.url_content, embeddings, source=pdf_file.name)
+                    st.session_state.index_created = True
+                    st.session_state.content_type = "pdf"
                     st.session_state.summary = None
-                    st.text("YouTube video processed successfully! ✅✅✅")
+                    st.text("PDF processed successfully! ✅✅✅")
                 except Exception as e:
-                    st.error(f"Error processing YouTube video: {str(e)}")
+                    st.error(f"Error processing PDF: {str(e)}")
                     st.stop()
 
 # Summary button
 with main_container:
     if st.session_state.url_content and st.button("Generate Summary"):
         with st.spinner("Generating summary..."):
-            is_youtube = st.session_state.content_type == "youtube"
-            st.session_state.summary = summarize_content(st.session_state.url_content, st.session_state.llm, is_youtube=is_youtube)
+            st.session_state.summary = summarize_content(st.session_state.url_content, st.session_state.llm)
 
 # Display summary if generated
 if st.session_state.summary:
@@ -537,8 +281,8 @@ if st.session_state.summary:
         st.header("Summary of the Content")
         st.write(st.session_state.summary)
 
-# Query input with Ask button (only for web URLs)
-if st.session_state.url_content and st.session_state.content_type == "web":
+# Query input with Ask button
+if st.session_state.url_content and st.session_state.index_created:
     with main_container:
         st.header("Ask a Question")
         query = st.text_input("Question", placeholder="e.g., What is the article about?")