feat: role_backfiller auto mode - fetches all VODs from server

local_worker/role_backfiller.py

@@ -19,18 +19,37 @@ headers = {
     "Content-Type": "application/json"
 }
 
+def get_all_streams():
+    """Fetch all streams from the backend"""
+    try:
+        res = requests.get(f"{API_URL}/api/streams", headers=headers, timeout=10)
+        if res.status_code == 200:
+            data = res.json()
+            # API returns { streams: [...] } or just [...]
+            if isinstance(data, list):
+                return data
+            return data.get("streams", [])
+    except Exception as e:
+        print(f"[Error] Не удалось получить список стримов: {e}")
+    return []
+
+def extract_vod_id(stream):
+    """Extract VOD ID from stream's twitch_stream_id field (e.g. 'vod-2154382910' -> '2154382910')"""
+    twitch_id = stream.get("twitch_stream_id", "")
+    if twitch_id.startswith("vod-"):
+        return twitch_id[4:]
+    return None
+
 def download_vod_roles(vod_id):
     """Download chat log of a past VOD and extract only roles"""
-    roles_dict = {}  # Store latest role per user
+    roles_dict = {}
     url = "https://gql.twitch.tv/gql"
     gql_headers = {
         "Client-Id": "kimne78kx3ncx6brgo4mv6wki5h1ko",
         "User-Agent": "Mozilla/5.0"
     }
-    
+
     current_offset = 0
-    print(f"\n[Roles] Запуск сверхбыстрого парсера ролей для VOD {vod_id}...")
-    
     total_messages = 0
 
     while True:
@@ -47,58 +66,49 @@ def download_vod_roles(vod_id):
                 }
             }
         }
-            
+
         try:
             res = requests.post(url, json=payload, headers=gql_headers, timeout=10)
             if res.status_code != 200:
-                print(f"[Roles] Ошибка GQL: {res.status_code}")
                 break
-                
+
             data = res.json()
             if isinstance(data, list):
                 data = data[0]
-                
+
             video = data.get("data", {}).get("video", {})
             if not video:
                 break
-                
-            comments_edge = video.get("comments") or {}
-            edges = comments_edge.get("edges") or []
+
+            edges = (video.get("comments") or {}).get("edges") or []
             if not edges:
                 break
-            
+
             for edge in edges:
                 if not edge:
                     continue
                 node = edge.get("node")
                 if not node:
                     continue
-                    
                 commenter = node.get("commenter")
                 if not commenter:
                     continue
-                
                 user = commenter.get("login")
                 if not user:
                     continue
-                    
+
                 display_name = commenter.get("displayName", user)
-                
                 message = node.get("message")
                 if not message:
                     continue
-                
+
                 timestamp = node.get("createdAt")
-                
-                # Badges parse
                 user_badges = message.get("userBadges") or []
                 badges = [b.get("setID") for b in user_badges if b]
                 is_mod = "moderator" in badges or "broadcaster" in badges
                 is_sub = "subscriber" in badges or "founder" in badges
                 is_vip = "vip" in badges
-                
-                # We always overwrite with the latest role from the VOD
-                # because the chat goes forward in time
+
                 roles_dict[user.lower()] = {
                     "username": user,
                     "displayName": display_name,
@@ -108,10 +118,9 @@ def download_vod_roles(vod_id):
                     "timestamp": timestamp
                 }
                 total_messages += 1
-                
-            print(f"-> Обработано {total_messages} сообщений. Найдено уникальных пользователей: {len(roles_dict)}...", end="\r")
-            
-            # Progress offset
+
+            print(f"   -> Обработано {total_messages} сообщений, найдено {len(roles_dict)} уникальных пользователей...", end="\r")
+
             last_offset = edges[-1].get("node", {}).get("contentOffsetSeconds")
             if last_offset is not None:
                 next_offset = last_offset + 1
@@ -121,37 +130,72 @@ def download_vod_roles(vod_id):
             else:
                 break
         except Exception as e:
-            print(f"\n[Roles] Ошибка при загрузке: {e}")
+            print(f"\n   [!] Ошибка при загрузке: {e}")
             break
-            
-    print(f"\n[Roles] Парсинг завершен. Всего уникальных пользователей: {len(roles_dict)}")
+
     return list(roles_dict.values())
 
+def upload_roles(roles, vod_id):
+    """Send roles to backend in batches"""
+    batch_size = 500
+    for i in range(0, len(roles), batch_size):
+        batch = roles[i:i+batch_size]
+        try:
+            res = requests.post(f"{API_URL}/api/log/roles", json={"roles": batch}, headers=headers, timeout=10)
+            if res.status_code == 200:
+                print(f"   -> Обновлено пользователей: {i + len(batch)} / {len(roles)}", end="\r")
+            else:
+                print(f"\n   [!] Ошибка отправки: {res.status_code}")
+        except Exception as e:
+            print(f"\n   [!] Ошибка сети: {e}")
+    print()
+
 if __name__ == "__main__":
     print("=========================================================")
-    print("      Ultra-Fast Role Backfiller                        ")
+    print("      Ultra-Fast Role Backfiller (AUTO MODE)            ")
     print("=========================================================")
-    
-    vod_id = input("Введите ID Twitch VOD (например, 2154382910): ").strip()
-    if not vod_id:
-        print("ID VOD не введен. Выход.")
+    print(f"Backend: {API_URL}")
+    print()
+
+    # Fetch all streams
+    print("[1] Получение списка стримов с сервера...")
+    streams = get_all_streams()
+
+    if not streams:
+        print("[Error] Список стримов пустой или не удалось получить!")
+        sys.exit(1)
+
+    # Filter only VODs (twitch_stream_id starts with "vod-")
+    vod_streams = [s for s in streams if str(s.get("twitch_stream_id", "")).startswith("vod-")]
+
+    print(f"[+] Найдено стримов всего: {len(streams)}")
+    print(f"[+] Из них архивных VOD: {len(vod_streams)}")
+    print()
+
+    if not vod_streams:
+        print("[!] Нет VOD-стримов для обработки. Выход.")
         sys.exit(0)
-        
-    roles = download_vod_roles(vod_id)
-    if roles:
-        print("\n[Sync] Отправка ролей на сервер (пакетами по 500 пользователей)...")
-        batch_size = 500
-        for i in range(0, len(roles), batch_size):
-            batch = roles[i:i+batch_size]
-            try:
-                url = f"{API_URL}/api/log/roles"
-                res = requests.post(url, json={"roles": batch}, headers=headers, timeout=10)
-                if res.status_code == 200:
-                    print(f"-> Обновлено пользователей: {i + len(batch)} / {len(roles)}", end="\r")
-                else:
-                    print(f"\n-> Ошибка отправки пакета: {res.status_code}")
-            except Exception as e:
-                print(f"\n-> Ошибка сети при отправке пакета: {e}")
-        print("\n-> Восстановление ролей успешно завершено!")
-    else:
-        print("\n-> Нет ролей для обновления.")
+
+    # Process each VOD
+    for idx, stream in enumerate(vod_streams, 1):
+        vod_id = extract_vod_id(stream)
+        title = stream.get("title", "Без названия")
+        print(f"[{idx}/{len(vod_streams)}] VOD {vod_id} — «{title}»")
+
+        if not vod_id:
+            print("   -> Пропуск: нет VOD ID")
+            continue
+
+        roles = download_vod_roles(vod_id)
+        print(f"\n   -> Найдено пользователей с бейджами: {len(roles)}")
+
+        if roles:
+            upload_roles(roles, vod_id)
+            print(f"   -> ✅ Готово!")
+        else:
+            print(f"   -> Чат пустой или VOD недоступен, пропуск.")
+        print()
+
+    print("=========================================================")
+    print("       Восстановление ролей завершено!                  ")
+    print("=========================================================")