Update scraper.py

scraper.py

@@ -8,7 +8,15 @@ from dejavu import Dejavu
 djv = Dejavu(dburl="sqlite:///fingerprints.db")
 
 # ============================================================
-# ADD ARTIST NAMES — scraper will find top songs automatically
+# PLAYLISTS — songs will be scraped from these directly
+# ============================================================
+PLAYLISTS = [
+    "https://www.youtube.com/playlist?list=PLr7xQC-cXWL9EZ3dqpu8E_Xf_4nhS6xEJ",
+    "https://www.youtube.com/playlist?list=PLxA687tYuMWjrFhZTNBtk13YUL2TkwUnU",
+]
+
+# ============================================================
+# ARTISTS — scraper will find top songs automatically
 # ============================================================
 ARTISTS = [
     "Arijit Singh",
@@ -31,11 +39,10 @@ ARTISTS = [
     "Ed Sheeran",
     "The Weeknd",
     "Drake",
-    # Add more artists...
 ]
 
-SONGS_PER_ARTIST = 25        # 25 songs x 200 artists = 5000 total
-BATCH_SIZE = 500             # 500 songs per daily run
+SONGS_PER_ARTIST = 25
+BATCH_SIZE = 50
 PROGRESS_FILE = "scraper_progress.json"
 
 # ============================================================
@@ -51,12 +58,11 @@ def save_progress(progress):
         json.dump(progress, f, indent=2)
 
 def build_queue(progress):
-    """Search YouTube for each artist and build a queue of video URLs."""
     if progress["queue"]:
         print(f"📋 Resuming existing queue ({len(progress['queue'])} songs left)")
         return progress
 
-    print("🔍 Building song queue from artist names...")
+    print("🔍 Building song queue from playlists and artist names...")
     queue = []
     done_set = set(progress["done"])
 
@@ -66,27 +72,58 @@ def build_queue(progress):
         'extract_flat': True,
     }
 
+    # ── Playlists ──────────────────────────────────────────
+    for playlist_url in PLAYLISTS:
+        try:
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                result = ydl.extract_info(playlist_url, download=False)
+                playlist_title = result.get('title', playlist_url)
+                entries = result.get('entries', [])
+                added = 0
+                for entry in entries:
+                    if not entry or not entry.get('id'):
+                        continue
+                    url = f"https://www.youtube.com/watch?v={entry['id']}"
+                    title = entry.get('title', 'Unknown')
+                    if url not in done_set:
+                        queue.append({"url": url, "title": title, "artist": "playlist"})
+                        added += 1
+                print(f"  ✅ Playlist '{playlist_title}': {added} songs added")
+        except Exception as e:
+            print(f"  ❌ Failed to load playlist {playlist_url}: {e}")
+
+    # ── Artists ────────────────────────────────────────────
     for artist in ARTISTS:
         search_query = f"ytsearch{SONGS_PER_ARTIST}:{artist} songs"
         try:
             with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                 result = ydl.extract_info(search_query, download=False)
                 entries = result.get('entries', [])
+                added = 0
                 for entry in entries:
+                    if not entry or not entry.get('id'):
+                        continue
                     url = f"https://www.youtube.com/watch?v={entry['id']}"
                     title = entry.get('title', artist)
                     if url not in done_set:
                         queue.append({"url": url, "title": title, "artist": artist})
-            print(f"  ✅ Found songs for: {artist}")
+                        added += 1
+                print(f"  ✅ Artist '{artist}': {added} songs added")
         except Exception as e:
             print(f"  ❌ Failed to search {artist}: {e}")
 
     progress["queue"] = queue
     save_progress(progress)
-    print(f"📋 Total queue built: {len(queue)} songs")
+    print(f"\n📋 Total queue built: {len(queue)} songs")
     return progress
 
 def download_and_fingerprint(url, title):
+    wav_path = '/tmp/song.wav'
+
+    # Clean up any leftover file from previous run
+    if os.path.exists(wav_path):
+        os.remove(wav_path)
+
     ydl_opts = {
         'format': 'bestaudio/best',
         'outtmpl': '/tmp/song.%(ext)s',
@@ -97,24 +134,24 @@ def download_and_fingerprint(url, title):
         'quiet': True,
         'no_warnings': True,
     }
+
     try:
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
             ydl.extract_info(url, download=True)
 
-        wav_path = '/tmp/song.wav'
         if os.path.exists(wav_path):
             djv.fingerprint_file(wav_path, song_name=title)
             print(f"  ✅ Fingerprinted: {title}")
             os.remove(wav_path)
             return True
         else:
-            print(f"  ❌ WAV not found: {title}")
+            print(f"  ❌ WAV not found after download: {title}")
             return False
+
     except Exception as e:
         print(f"  ❌ Failed: {title} → {e}")
-        # Clean up if file exists
-        if os.path.exists('/tmp/song.wav'):
-            os.remove('/tmp/song.wav')
+        if os.path.exists(wav_path):
+            os.remove(wav_path)
         return False
 
 def run_batch():
@@ -128,7 +165,6 @@ def run_batch():
         print("🎉 All songs have been fingerprinted!")
         return
 
-    # Take next BATCH_SIZE from queue
     batch = queue[:BATCH_SIZE]
     remaining = queue[BATCH_SIZE:]
 
@@ -147,7 +183,6 @@ def run_batch():
         else:
             failed += 1
 
-    # Update progress
     progress["queue"] = remaining
     progress["done"] = done
     save_progress(progress)