Update: fix bot detection

.env.example

@@ -5,6 +5,9 @@ SERVER_PORT=8000
 # API Configuration
 API_V1_STR=/api
 
+# YouTube Configuration
+YOUTUBE_API_KEY=your_youtube_api_key_here
+
 # Project Configuration
 PROJECT_NAME="YouTube to MP3 Converter"
 

app/core/config.py

@@ -11,6 +11,9 @@ class Settings(BaseSettings):
     SERVER_HOST: str
     SERVER_PORT: int
 
+    # YouTube API configuration
+    YOUTUBE_API_KEY: str
+
     # Hugging Face configuration
     HF_SPACE_NAME: str = "bivav/video-mp3"
     HF_STATIC_DIR: str = "/data/audio"

app/services/youtube_service.py

@@ -22,24 +22,39 @@ class YouTubeService:
             # Add user agent and other headers to avoid restrictions
             "http_headers": {
                 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                "Accept-Language": "en-us,en;q=0.5",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Accept-Encoding": "gzip, deflate, br",
                 "Referer": "https://www.youtube.com/",
+                "Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120"',
+                "Sec-Ch-Ua-Mobile": "?0",
+                "Sec-Ch-Ua-Platform": '"Windows"',
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "none",
+                "Sec-Fetch-User": "?1",
+                "X-YouTube-API-Key": settings.YOUTUBE_API_KEY,  # Add API key to headers
+            },
+            # Add API key to extractor args
+            "extractor_args": {
+                "youtube": {
+                    "api_key": [settings.YOUTUBE_API_KEY],
+                }
             },
             # Add retries and throttling with better Docker-optimized settings
-            "retries": 3,
-            "fragment_retries": 3,
+            "retries": 5,
+            "fragment_retries": 5,
             "skip_unavailable_fragments": True,
-            "ignoreerrors": False,  # Changed to False to better handle errors in Docker
-            "no_warnings": False,  # Show warnings for better Docker logs
-            "sleep_interval": 2,  # Increased sleep to avoid rate limiting
-            "max_sleep_interval": 6,
+            "ignoreerrors": False,
+            "no_warnings": False,
+            "sleep_interval": 5,
+            "max_sleep_interval": 10,
             # Add geo bypass
             "geo_bypass": True,
             "geo_bypass_country": "US",
             # Add network settings
-            "socket_timeout": 30,  # Timeout for network operations
-            "extractor_retries": 3,  # Retries for extractor errors
+            "socket_timeout": 30,
+            "extractor_retries": 5,
             # Add progress hooks for better logging
             "progress_hooks": [self._progress_hook],
             # Force IPv4 to avoid IPv6 issues in some Docker networks
@@ -97,7 +112,7 @@ class YouTubeService:
         opts = {**self.ydl_opts, "outtmpl": str(output_path.with_suffix(""))}
 
         try:
-            # Run youtube-dl in a thread pool to avoid blocking
+            # First try with default options
             await asyncio.get_event_loop().run_in_executor(
                 None, self._download_and_convert, url, opts
             )
@@ -107,36 +122,44 @@ class YouTubeService:
 
             return filename
         except Exception as e:
-            print(
-                f"Error in convert_to_mp3: {str(e)}"
-            )  # Better error logging for Docker
+            print(f"Error in convert_to_mp3: {str(e)}")
             raise Exception(f"Error converting video: {str(e)}")
 
     def _download_and_convert(self, url: str, opts: dict):
-        """
-        Download and convert the video using yt-dlp
-        """
-        try:
-            with yt_dlp.YoutubeDL(opts) as ydl:
-                error = ydl.download([url])
-                if error:  # yt-dlp returns non-zero on error
-                    raise Exception(f"yt-dlp returned error code: {error}")
-        except Exception as e:
-            print(f"First download attempt failed: {str(e)}")
-            # Try one more time with different format
+        attempts = [
+            {"format": "bestaudio/best", "msg": "Trying with best audio quality..."},
+            {
+                "format": "worstaudio/worst",
+                "msg": "Trying with lowest quality as fallback...",
+            },
+            {
+                "format": "bestaudio/best",
+                "extract_flat": True,
+                "msg": "Trying with flat extraction...",
+            },
+        ]
+
+        last_error = None
+        for attempt in attempts:
             try:
-                opts["format"] = (
-                    "worstaudio/worst"  # Try with lowest quality as fallback
-                )
-                with yt_dlp.YoutubeDL(opts) as ydl_fallback:
-                    error = ydl_fallback.download([url])
-                    if error:
-                        raise Exception(
-                            f"Fallback download failed with error code: {error}"
-                        )
-            except Exception as fallback_error:
-                print(f"Fallback download attempt failed: {str(fallback_error)}")
-                raise  # Re-raise the last error
+                print(attempt["msg"])
+                current_opts = {**opts, "format": attempt["format"]}
+                if "extract_flat" in attempt:
+                    current_opts["extract_flat"] = attempt["extract_flat"]
+
+                with yt_dlp.YoutubeDL(current_opts) as ydl:
+                    error = ydl.download([url])
+                    if not error:  # yt-dlp returns 0 on success
+                        return  # Success!
+                    raise Exception(f"yt-dlp returned error code: {error}")
+            except Exception as e:
+                print(f"Download attempt failed: {str(e)}")
+                last_error = e
+                # Wait before next attempt
+                asyncio.get_event_loop().run_in_executor(None, asyncio.sleep, 2)
+
+        # If we get here, all attempts failed
+        raise last_error or Exception("All download attempts failed")
 
 
 youtube_service = YouTubeService()