Update app.py

app.py

@@ -3,6 +3,44 @@ import tempfile
 import gradio as gr
 import re
 import sys
+import shutil
+import importlib.util
+
+def check_requirements():
+    """Check if all required packages are installed and return status"""
+    requirements_status = []
+    
+    packages = [
+        ('gradio', 'gradio'),
+        ('yt-dlp', 'yt_dlp'),
+        ('openai-whisper', 'whisper'),
+        ('torch', 'torch'),
+        ('torchaudio', 'torchaudio'),
+        ('numpy', 'numpy'),
+        ('regex', 'regex'),
+    ]
+    
+    for package_name, import_name in packages:
+        try:
+            spec = importlib.util.find_spec(import_name)
+            if spec is None:
+                requirements_status.append(f"❌ {package_name}: Not found")
+                continue
+            
+            module = importlib.import_module(import_name)
+            version = getattr(module, '__version__', 'Unknown version')
+            requirements_status.append(f"✅ {package_name}: {version}")
+            
+        except ImportError as e:
+            requirements_status.append(f"❌ {package_name}: Import error - {str(e)}")
+        except Exception as e:
+            requirements_status.append(f"⚠️  {package_name}: Found but error - {str(e)}")
+    
+    # Add Python info
+    requirements_status.append(f"\n🐍 Python: {sys.version}")
+    requirements_status.append(f"📁 Python executable: {sys.executable}")
+    
+    return "\n".join(requirements_status)
 
 # Try to import required packages with error handling
 try:
@@ -12,30 +50,46 @@ except ImportError as e:
     YT_DLP_AVAILABLE = False
     print(f"yt-dlp import error: {e}")
 
+# Try multiple whisper import methods
+WHISPER_AVAILABLE = False
+WHISPER_TYPE = None
+
 try:
     import whisper
     WHISPER_AVAILABLE = True
+    WHISPER_TYPE = "openai-whisper"
+    print("Using OpenAI Whisper")
 except ImportError as e:
-    WHISPER_AVAILABLE = False
-    print(f"whisper import error: {e}")
+    print(f"OpenAI Whisper import error: {e}")
+    try:
+        from transformers import pipeline
+        WHISPER_AVAILABLE = True
+        WHISPER_TYPE = "transformers"
+        print("Using Transformers Whisper")
+    except ImportError as e2:
+        print(f"Transformers Whisper import error: {e2}")
 
 print(f"Python version: {sys.version}")
+print(f"Python executable: {sys.executable}")
 print(f"yt-dlp available: {YT_DLP_AVAILABLE}")
-print(f"whisper available: {WHISPER_AVAILABLE}")
+print(f"whisper available: {WHISPER_AVAILABLE} (type: {WHISPER_TYPE})")
 
-def get_cookies_path():
-    """Get the path to cookies.txt file"""
-    # Check if cookies.txt exists in the current directory
-    if os.path.exists('cookies.txt'):
-        return 'cookies.txt'
-    # Check in the same directory as the script
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    cookies_path = os.path.join(script_dir, 'cookies.txt')
-    if os.path.exists(cookies_path):
-        return cookies_path
-    return None
+# Additional diagnostics
+if YT_DLP_AVAILABLE:
+    try:
+        from yt_dlp import YoutubeDL
+        print(f"yt-dlp version: {YoutubeDL().__class__.__module__}")
+    except:
+        pass
+
+if WHISPER_AVAILABLE and WHISPER_TYPE == "openai-whisper":
+    try:
+        import whisper
+        print(f"whisper version: {whisper.__version__}")
+    except:
+        pass
 
-def download_audio(url):
+def download_audio(url, cookies_file_path=None):
     """Download audio from YouTube URL and return the file path"""
     if not YT_DLP_AVAILABLE:
         raise Exception("yt-dlp is not available. Please check the installation.")
@@ -45,150 +99,82 @@ def download_audio(url):
         temp_dir = tempfile.mkdtemp()
         output_path = os.path.join(temp_dir, "audio")
         
-        # Get cookies path
-        cookies_path = get_cookies_path()
-        
-        # Base yt-dlp options
         ydl_opts = {
             'format': 'bestaudio[ext=m4a]/bestaudio/best',
             'outtmpl': output_path + '.%(ext)s',
             'quiet': True,
             'no_warnings': True,
-            'extract_flat': False,
-            'ignoreerrors': False,
-            # Add user agent to avoid bot detection
-            'http_headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-            },
-            # Add additional options to avoid bot detection
+            # Anti-bot detection measures
+            'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+            'referer': 'https://www.youtube.com/',
             'extractor_retries': 3,
             'fragment_retries': 3,
-            'retry_sleep_functions': {'http': lambda n: min(2 ** n, 30)},
-            # Add geo bypass options
-            'geo_bypass': True,
-            'geo_bypass_country': 'US',
-            # Add more headers
-            'http_headers': {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                'Accept-Language': 'en-us,en;q=0.5',
-                'Accept-Encoding': 'gzip,deflate',
-                'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
-                'Connection': 'keep-alive',
-                'Upgrade-Insecure-Requests': '1',
-            }
+            'retry_sleep_functions': {'http': lambda n: 2 ** n},
         }
         
-        # Add cookies if available
-        if cookies_path:
-            ydl_opts['cookiefile'] = cookies_path
-            print(f"Using cookies from: {cookies_path}")
+        # Add cookies file if provided
+        if cookies_file_path and os.path.exists(cookies_file_path):
+            ydl_opts['cookiefile'] = cookies_file_path
+            print(f"Using cookies file: {cookies_file_path}")
         else:
-            print("No cookies.txt found - proceeding without cookies")
+            print("No cookies file provided - may encounter bot detection")
+            # Additional headers without cookies
+            ydl_opts.update({
+                'headers': {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    'Accept-Language': 'en-us,en;q=0.5',
+                    'Accept-Encoding': 'gzip,deflate',
+                    'DNT': '1',
+                    'Connection': 'keep-alive',
+                    'Upgrade-Insecure-Requests': '1',
+                }
+            })
         
         with YoutubeDL(ydl_opts) as ydl:
-            try:
-                # Extract info first to check if video is available
-                info_dict = ydl.extract_info(url, download=False)
-                
-                # Check if video is available
-                if info_dict.get('availability') == 'private':
-                    raise Exception("Video is private and cannot be accessed")
-                elif info_dict.get('availability') == 'premium_only':
-                    raise Exception("Video requires premium subscription")
-                elif info_dict.get('availability') == 'subscriber_only':
-                    raise Exception("Video is only available to channel subscribers")
-                elif info_dict.get('availability') == 'needs_auth':
-                    raise Exception("Video requires authentication - try updating cookies")
-                elif info_dict.get('live_status') == 'is_live':
-                    raise Exception("Cannot download live streams")
-                elif info_dict.get('live_status') == 'was_live':
-                    print("Note: This was a live stream, trying to download recorded version...")
-                
-                # Download the audio
-                ydl.download([url])
-                
-            except Exception as extract_error:
-                # If extract_info fails, try direct download as fallback
-                print(f"Info extraction failed: {extract_error}")
-                print("Attempting direct download...")
-                ydl.download([url])
+            info_dict = ydl.extract_info(url, download=True)
+            filename = ydl.prepare_filename(info_dict)
             
             # Find the downloaded file
             for ext in ['.m4a', '.webm', '.mp4', '.mp3']:
                 potential_file = output_path + ext
                 if os.path.exists(potential_file):
-                    print(f"Successfully downloaded: {potential_file}")
                     return potential_file
             
             raise FileNotFoundError(f"Downloaded audio file not found")
                 
     except Exception as e:
-        error_msg = str(e)
-        if "Sign in to confirm your age" in error_msg:
-            raise Exception("❌ Video is age-restricted. Please use a different video or update your cookies with an authenticated session.")
-        elif "Private video" in error_msg:
-            raise Exception("❌ Video is private and cannot be accessed.")
-        elif "This video is unavailable" in error_msg or "Video unavailable" in error_msg:
-            raise Exception("❌ Video is unavailable. This could be due to:\n• Geographic restrictions\n• Content removed by uploader\n• Copyright issues\n• Try a different video")
-        elif "This content isn't available" in error_msg:
-            raise Exception("❌ Content not available in your region or has been restricted. Try:\n• Using a VPN\n• Different video\n• Updating cookies")
-        elif "blocked" in error_msg.lower():
-            raise Exception("❌ Access blocked. Try using updated cookies or a different video.")
-        elif "HTTP Error 403" in error_msg:
-            raise Exception("❌ Access forbidden. Video may be region-locked or require authentication.")
-        elif "HTTP Error 404" in error_msg:
-            raise Exception("❌ Video not found. It may have been deleted or the URL is incorrect.")
+        if "403" in str(e) or "Forbidden" in str(e):
+            raise Exception(f"YouTube blocked the request (403 Forbidden). Please upload your cookies.txt file to bypass bot detection. Original error: {str(e)}")
         else:
-            raise Exception(f"❌ Download failed: {error_msg}")
-
-def test_video_access(url):
-    """Test if a video is accessible without downloading"""
-    try:
-        cookies_path = get_cookies_path()
-        ydl_opts = {
-            'quiet': True,
-            'no_warnings': True,
-            'extract_flat': False,
-            'skip_download': True,
-        }
-        
-        if cookies_path:
-            ydl_opts['cookiefile'] = cookies_path
-            
-        with YoutubeDL(ydl_opts) as ydl:
-            info_dict = ydl.extract_info(url, download=False)
-            
-            status = "✅ Video accessible"
-            details = []
-            
-            if info_dict.get('title'):
-                details.append(f"Title: {info_dict['title'][:50]}...")
-            if info_dict.get('duration'):
-                details.append(f"Duration: {info_dict['duration']} seconds")
-            if info_dict.get('availability'):
-                details.append(f"Availability: {info_dict['availability']}")
-            if info_dict.get('age_limit'):
-                details.append(f"Age limit: {info_dict['age_limit']}+")
-                
-            return status + "\n" + "\n".join(details)
-            
-    except Exception as e:
-        return f"❌ Video access test failed: {str(e)}"
+            raise Exception(f"Failed to download audio: {str(e)}")
 
 def transcribe_audio(file_path):
     """Transcribe audio file using Whisper"""
     if not WHISPER_AVAILABLE:
-        raise Exception("OpenAI Whisper is not available. Please check the installation.")
+        raise Exception("OpenAI Whisper is not available. Please install it using: pip install openai-whisper")
     
     try:
-        # Use the smallest model to reduce memory usage
-        model = whisper.load_model("tiny")
-        result = model.transcribe(file_path)
-        return result["text"]
+        if WHISPER_TYPE == "openai-whisper":
+            # Use OpenAI Whisper
+            model = whisper.load_model("tiny")
+            result = model.transcribe(file_path)
+            return result["text"]
+        
+        elif WHISPER_TYPE == "transformers":
+            # Use Transformers Whisper
+            from transformers import pipeline
+            transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-tiny")
+            result = transcriber(file_path)
+            return result["text"]
+        
+        else:
+            raise Exception("No compatible Whisper installation found")
+            
     except Exception as e:
         raise Exception(f"Failed to transcribe audio: {str(e)}")
 
+
 def extract_stock_info_simple(text):
     """Extract stock information using simple pattern matching"""
     try:
@@ -261,95 +247,76 @@ def cleanup_file(file_path):
     except:
         pass
 
-def system_test():
-    """Test system components"""
-    results = []
-    
-    # Test yt-dlp
-    if YT_DLP_AVAILABLE:
-        results.append("✅ yt-dlp: Available")
-        try:
-            ydl = YoutubeDL({'quiet': True})
-            results.append("✅ yt-dlp: Can create YoutubeDL instance")
-        except Exception as e:
-            results.append(f"❌ yt-dlp: Cannot create instance - {e}")
-    else:
-        results.append("❌ yt-dlp: Not available")
+def process_cookies_file(cookies_file):
+    """Process uploaded cookies file and return the path"""
+    if cookies_file is None:
+        return None
     
-    # Test Whisper
-    if WHISPER_AVAILABLE:
-        results.append("✅ Whisper: Available (Type: openai-whisper)")
-        try:
-            import whisper
-            results.append("✅ Whisper: OpenAI Whisper can be imported")
-        except Exception as e:
-            results.append(f"❌ Whisper: Cannot import - {e}")
-    else:
-        results.append("❌ Whisper: Not available")
-    
-    # Test file operations
     try:
-        temp_file = tempfile.NamedTemporaryFile(delete=False)
-        temp_file.write(b"test")
-        temp_file.close()
-        os.remove(temp_file.name)
-        results.append("✅ File operations: Working")
+        # Create a temporary file for cookies
+        temp_cookies_path = tempfile.mktemp(suffix='.txt')
+        
+        # Copy the uploaded file to temp location
+        shutil.copy2(cookies_file.name, temp_cookies_path)
+        
+        return temp_cookies_path
     except Exception as e:
-        results.append(f"❌ File operations: Failed - {e}")
-    
-    # Test cookies
-    cookies_path = get_cookies_path()
-    if cookies_path:
-        results.append(f"✅ Cookies: Found at {cookies_path}")
-    else:
-        results.append("⚠️ Cookies: Not found (may cause bot detection issues)")
-    
-    return "\n".join(results)
+        print(f"Error processing cookies file: {e}")
+        return None
 
-def process_video(url, progress=gr.Progress()):
+def process_video(url, cookies_file, progress=gr.Progress()):
     """Main function to process YouTube video"""
     
     # Check if required packages are available
     if not YT_DLP_AVAILABLE:
-        return "Error: yt-dlp is not installed properly. Please check the requirements.", ""
+        return "Error: yt-dlp is not installed properly. Please install it using: pip install yt-dlp", "", "❌ Error: Missing yt-dlp"
     
     if not WHISPER_AVAILABLE:
-        return "Error: OpenAI Whisper is not installed properly. Please check the requirements.", ""
+        return "Error: OpenAI Whisper is not installed properly. Please install it using: pip install openai-whisper", "", "❌ Error: Missing Whisper"
     
     if not url or not url.strip():
-        return "Please provide a valid YouTube URL", ""
+        return "Please provide a valid YouTube URL", "", "❌ Error: Invalid URL"
     
     audio_path = None
+    cookies_temp_path = None
+    
     try:
         # Validate URL
         if not any(domain in url.lower() for domain in ['youtube.com', 'youtu.be']):
-            return "Please provide a valid YouTube URL", ""
+            return "Please provide a valid YouTube URL", "", "❌ Error: Invalid URL"
+        
+        # Process cookies file if provided
+        progress(0.05, desc="Processing cookies...")
+        cookies_temp_path = process_cookies_file(cookies_file)
+        
+        status_msg = "✅ Cookies loaded" if cookies_temp_path else "⚠️ No cookies (may encounter bot detection)"
         
         # Download audio
-        progress(0.1, desc="Downloading audio...")
-        audio_path = download_audio(url)
+        progress(0.2, desc="Downloading audio...")
+        audio_path = download_audio(url, cookies_temp_path)
         
         # Transcribe audio
-        progress(0.5, desc="Transcribing audio...")
+        progress(0.6, desc="Transcribing audio...")
         transcript = transcribe_audio(audio_path)
         
         if not transcript.strip():
-            return "No speech detected in the video", ""
+            return "No speech detected in the video", "", "❌ No speech detected"
         
         # Extract stock information
-        progress(0.8, desc="Extracting stock information...")
+        progress(0.9, desc="Extracting stock information...")
         stock_details = extract_stock_info_simple(transcript)
         
         progress(1.0, desc="Complete!")
-        return transcript, stock_details
+        return transcript, stock_details, "✅ Processing completed successfully"
         
     except Exception as e:
         error_msg = f"Error processing video: {str(e)}"
-        return error_msg, ""
+        return error_msg, "", f"❌ Error: {str(e)}"
     
     finally:
         # Clean up temporary files
         cleanup_file(audio_path)
+        cleanup_file(cookies_temp_path)
 
 # Create Gradio interface
 with gr.Blocks(
@@ -357,9 +324,14 @@ with gr.Blocks(
     theme=gr.themes.Soft(),
     css="""
     .gradio-container {
-        max-width: 1200px;
+        max-width: 1400px;
         margin: auto;
     }
+    .status-box {
+        padding: 10px;
+        border-radius: 5px;
+        margin: 10px 0;
+    }
     """
 ) as demo:
     
@@ -369,75 +341,81 @@ with gr.Blocks(
     Extract stock recommendations and trading information from YouTube videos using AI transcription.
     
     **How it works:**
-    1. Downloads audio from YouTube video
-    2. Transcribes using OpenAI Whisper
-    3. Extracts stock-related information
+    1. Upload your cookies.txt file (optional but recommended to avoid bot detection)
+    2. Paste YouTube video URL
+    3. Downloads audio from YouTube video
+    4. Transcribes using OpenAI Whisper
+    5. Extracts stock-related information
     
     **⚠️ Disclaimer:** This is for educational purposes only. Always do your own research!
     """)
     
-    # Add system test section
-    with gr.Accordion("🧪 System Status", open=False):
-        system_status = gr.Textbox(
-            value=system_test(),
-            label="System Test Results",
-            lines=10,
-            interactive=False
-        )
-        test_btn = gr.Button("🔄 Re-run System Test")
-        test_btn.click(fn=system_test, outputs=system_status)
-    
     with gr.Row():
         with gr.Column(scale=1):
+            # Requirements check button
+            gr.Markdown("### 🔍 System Check")
+            check_req_btn = gr.Button(
+                "Check Requirements", 
+                variant="secondary",
+                size="sm"
+            )
+            
+            requirements_output = gr.Textbox(
+                label="📋 Requirements Status",
+                lines=10,
+                interactive=False,
+                visible=False
+            )
+            
+            # Cookies file upload
+            cookies_input = gr.File(
+                label="🍪 Upload Cookies File (cookies.txt)",
+                file_types=[".txt"],
+                file_count="single"
+            )
+            
+            gr.Markdown("""
+            **How to get cookies.txt to fix 403 Forbidden errors:**
+            1. Install browser extension: "Get cookies.txt LOCALLY" 
+            2. Visit YouTube in your browser (while logged in)
+            3. Click the extension icon and export cookies for youtube.com
+            4. Upload the downloaded cookies.txt file here
+            
+            **Alternative extensions:**
+            - "cookies.txt" (Chrome/Firefox)
+            - "Export Cookies" (Chrome)
+            
+            ⚠️ **Important**: Without cookies, you'll likely get 403 Forbidden errors
+            """)
+            
             url_input = gr.Textbox(
                 label="📺 YouTube URL",
                 placeholder="https://www.youtube.com/watch?v=...",
                 lines=2
             )
             
-            with gr.Row():
-                process_btn = gr.Button(
-                    "🚀 Extract Stock Information", 
-                    variant="primary",
-                    size="lg"
-                )
-                test_btn = gr.Button(
-                    "🔍 Test Video Access",
-                    variant="secondary"
-                )
+            process_btn = gr.Button(
+                "🚀 Extract Stock Information", 
+                variant="primary",
+                size="lg"
+            )
             
-            test_result = gr.Textbox(
-                label="📋 Video Access Test",
-                lines=4,
-                visible=False
+            # Status display
+            status_output = gr.Textbox(
+                label="📊 Status",
+                lines=1,
+                interactive=False
             )
             
             gr.Markdown("""
             ### 💡 Tips:
-            - **First try "Test Video Access"** to check if video is available
+            - **MUST upload cookies.txt** to avoid 403 Forbidden errors
             - Works best with financial YouTube channels
             - Ensure video has clear audio
             - English content works best
-            - If you get bot detection errors, try updating cookies.txt
-            
-            ### 🎯 Recommended Financial Channels:
-            - Ben Felix, The Plain Bagel, Two Cents, Graham Stephan
-            - Make sure videos are public and not age-restricted
+            - Try shorter videos first (under 10 minutes)
             """)
     
-    # Add test button functionality
-    def test_and_show(url):
-        if not url:
-            return "Please enter a YouTube URL first", gr.update(visible=False)
-        result = test_video_access(url)
-        return result, gr.update(visible=True)
-    
-    test_btn.click(
-        fn=test_and_show,
-        inputs=[url_input],
-        outputs=[test_result, test_result]
-    )
-    
     with gr.Row():
         with gr.Column():
             transcript_output = gr.Textbox(
@@ -456,10 +434,19 @@ with gr.Blocks(
             )
     
     # Event handlers
+    def show_requirements():
+        status = check_requirements()
+        return gr.update(value=status, visible=True)
+    
+    check_req_btn.click(
+        fn=show_requirements,
+        outputs=[requirements_output]
+    )
+    
     process_btn.click(
         fn=process_video,
-        inputs=[url_input],
-        outputs=[transcript_output, stock_info_output],
+        inputs=[url_input, cookies_input],
+        outputs=[transcript_output, stock_info_output, status_output],
         show_progress=True
     )
     
@@ -472,6 +459,48 @@ with gr.Blocks(
         inputs=[url_input],
         label="Click to try example"
     )
+    
+    gr.Markdown("""
+    ### 🔧 Installation & Troubleshooting:
+    
+    **Step 1: Click "Check Requirements" button above to see what's missing**
+    
+    **If you get "Whisper Missing" error:**
+    ```bash
+    pip install openai-whisper
+    ```
+    
+    **If you get "yt-dlp Missing" error:**
+    ```bash
+    pip install yt-dlp
+    ```
+    
+    **Install all requirements at once:**
+    ```bash
+    pip install gradio==4.44.0 yt-dlp==2023.12.30 openai-whisper==20231117 torch==2.1.0 torchaudio==2.1.0 numpy==1.24.3 regex==2023.8.8
+    ```
+    
+    **Alternative Whisper installation:**
+    ```bash
+    pip install transformers torch torchaudio
+    ```
+    
+    **If using virtual environment:**
+    ```bash
+    # Create and activate virtual environment first
+    python -m venv myenv
+    # Windows: myenv\\Scripts\\activate
+    # Mac/Linux: source myenv/bin/activate
+    # Then install packages
+    pip install -r requirements.txt
+    ```
+    
+    **Other Issues:**
+    - **Bot Detection Error**: Upload your cookies.txt file
+    - **No Audio Found**: Check if video has audio track
+    - **Transcription Failed**: Video might be too long or audio quality poor
+    - **No Stock Info**: Video might not contain financial content
+    """)
 
 if __name__ == "__main__":
     demo.launch()