unstable update

LoadBalancer.py

@@ -3,6 +3,7 @@ from indexer import indexer
 import re
 from tvdb import fetch_and_cache_json
 from threading import Event, Thread
+import asyncio
 import time
 import logging
 from utils import convert_to_gb
@@ -10,8 +11,6 @@ from api import InstancesAPI
 
 CACHE_DIR = os.getenv("CACHE_DIR")
 
-download_progress = {}
-
 class LoadBalancer:
     def __init__(self, cache_dir, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
         self.version = "0.0.0.1 Alpha"
@@ -33,8 +32,8 @@ class LoadBalancer:
         if not os.path.exists(self.CACHE_DIR):
             os.makedirs(self.CACHE_DIR)
 
-        # run services
-        self.file_structure=indexer()
+        # Initialize file structure and start prefetching
+        self.file_structure = indexer()
         self.start_prefetching()
 
         # Start polling and file checking in separate threads
@@ -42,6 +41,42 @@ class LoadBalancer:
         polling_thread.daemon = True
         polling_thread.start()
 
+    def start_prefetching(self):
+        """Start the metadata prefetching in a separate thread."""
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.run_in_executor(None, self.prefetch_metadata)
+        
+    async def prefetch_metadata(self):
+        """Prefetch metadata for all items in the file structure."""
+        tasks = []
+        for item in self.file_structure:
+            if 'contents' in item:
+                for sub_item in item['contents']:
+                    original_title = sub_item['path'].split('/')[-1]
+                    media_type = 'series' if item['path'].startswith('tv') else 'movie'
+                    title = original_title
+                    year = None
+
+                    # Extract year from the title if available
+                    match = re.search(r'\((\d{4})\)', original_title)
+                    if match:
+                        year_str = match.group(1)
+                        if year_str.isdigit() and len(year_str) == 4:
+                            title = original_title[:match.start()].strip()
+                            year = int(year_str)
+                    else:
+                        parts = original_title.rsplit(' ', 1)
+                        if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
+                            title = parts[0].strip()
+                            year = int(parts[-1])
+
+                    # Schedule the fetch and cache task
+                    tasks.append(fetch_and_cache_json(original_title, title, media_type, year))
+        
+        # Run all tasks concurrently
+        await asyncio.gather(*tasks)
+
     def register_instance(self, instance_url):
         if instance_url not in self.instances:
             self.instances.append(instance_url)
@@ -113,10 +148,6 @@ class LoadBalancer:
         logging.info("Stopping polling.")
         self.stop_event.set()
 
-    def start_prefetching(self):
-        """Start the metadata prefetching in a separate thread."""
-        self.prefetch_metadata()
-
     def update_instances_health(self, instance, cache_size):
         self.instances_health[instance] = {"used":cache_size["cache_size"],
                                            "total": "50 GB"}
@@ -241,31 +272,6 @@ class LoadBalancer:
         """Generate a film ID based on the title."""
         return title.replace(" ", "_").lower()
 
-    def prefetch_metadata(self):
-        """Prefetch metadata for all items in the file structure."""
-        for item in self.file_structure:
-            if 'contents' in item:
-                for sub_item in item['contents']:
-                    original_title = sub_item['path'].split('/')[-1]
-                    media_type = 'series' if item['path'].startswith('tv') else 'movie'
-                    title = original_title
-                    year = None
-
-                    # Extract year from the title if available
-                    match = re.search(r'\((\d{4})\)', original_title)
-                    if match:
-                        year_str = match.group(1)
-                        if year_str.isdigit() and len(year_str) == 4:
-                            title = original_title[:match.start()].strip()
-                            year = int(year_str)
-                    else:
-                        parts = original_title.rsplit(' ', 1)
-                        if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
-                            title = parts[0].strip()
-                            year = int(parts[-1])
-
-                    fetch_and_cache_json(original_title, title, media_type, year)
-
     def get_all_tv_shows(self):
         """Get all TV shows from the indexed cache structure JSON file."""
         tv_shows = {}

tvdb.py

@@ -5,7 +5,9 @@ import urllib.parse
 from datetime import datetime, timedelta
 from dotenv import load_dotenv
 import json
-from tvdbApiClient import fetch_and_cache_seasons
+import asyncio
+import aiofiles
+from tvdbApiClient import fetch_and_cache_seasons, save_to_json
 
 load_dotenv()
 THETVDB_API_KEY = os.getenv("THETVDB_API_KEY")
@@ -37,7 +39,7 @@ def get_thetvdb_token():
         authenticate_thetvdb()
     return THETVDB_TOKEN
 
-def fetch_and_cache_json(original_title, title, media_type, year=None):
+async def fetch_and_cache_json(original_title, title, media_type, year=None):
     if year:
         search_url = f"{THETVDB_API_URL}/search?query={urllib.parse.quote(title)}&type={media_type}&year={year}"
     else:
@@ -74,7 +76,7 @@ def fetch_and_cache_json(original_title, title, media_type, year=None):
                 extended_url = f"{THETVDB_API_URL}/movies/{tvdb_id}/extended?meta=translations"
             elif media_type == 'series':
                 extended_url = f"{THETVDB_API_URL}/series/{tvdb_id}/extended?meta=translations"
-                fetch_and_cache_seasons(tvdb_id)
+                await fetch_and_cache_seasons(tvdb_id)  # Using await for the async function
             else:
                 print(f"Unsupported media type: {media_type}")
                 return
@@ -86,8 +88,19 @@ def fetch_and_cache_json(original_title, title, media_type, year=None):
             
             # Cache the extended JSON response
             json_cache_path = os.path.join(CACHE_DIR, f"{urllib.parse.quote(original_title)}.json")
-            with open(json_cache_path, 'w') as f:
-                json.dump(extended_data, f)
+            await save_to_json(extended_data, json_cache_path)
                 
     except requests.RequestException as e:
         print(f"Error fetching data: {e}")
+
+def main():
+    # Replace with your series ID and other parameters
+    original_title = "The Listner (2009)"
+    title = "The Listner"
+    media_type = "series"  # or "movie"
+    year = 2009
+
+    asyncio.run(fetch_and_cache_json(original_title, title, media_type, year))
+
+if __name__ == "__main__":
+    main()

tvdbApiClient.py

@@ -3,10 +3,11 @@ import os
 import logging
 from pathlib import Path
 import tvdb_v4_official
+import aiofiles
 
 THETVDB_API_KEY = os.getenv("THETVDB_API_KEY")
 CACHE_DIR = os.getenv("CACHE_DIR")
-SAVE_DIR = os.path.join(CACHE_DIR,"metadata")
+SAVE_DIR = os.path.join(CACHE_DIR, "metadata")
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Initialize TVDB client
@@ -41,17 +42,17 @@ def filter_episode_data(episode):
         "year": episode.get("year")
     }
 
-def save_to_json(data, path):
-    """Save data to a JSON file."""
+async def save_to_json(data, path):
+    """Save data to a JSON file asynchronously."""
     try:
-        with open(path, 'w', encoding='utf-8') as f:
-            json.dump(data, f, indent=4, ensure_ascii=False)
+        async with aiofiles.open(path, 'w', encoding='utf-8') as f:
+            await f.write(json.dumps(data, indent=4, ensure_ascii=False))
         logging.info(f"Data saved to {path}")
     except IOError as e:
         logging.error(f"Error saving data to {path}: {e}")
 
-def fetch_and_cache_seasons(series_id):
-    """Fetch and cache episodes for a given series ID."""
+async def fetch_and_cache_seasons(series_id):
+    """Fetch and cache episodes for a given series ID asynchronously."""
     series_info = get_series_info(series_id)
     if not series_info:
         logging.error("Series info could not be fetched.")
@@ -78,20 +79,21 @@ def fetch_and_cache_seasons(series_id):
             logging.error(f"Error fetching season info for {season_key}: {e}")
 
     # Create folder for the series
-    series_folder = Path(SAVE_DIR) / series_id
+    series_folder = Path(SAVE_DIR) / str(series_id)
     series_folder.mkdir(parents=True, exist_ok=True)
     
     # Save episodes for each season in separate JSON files
     for season_key, episodes in sorted(all_seasons.items()):
         episodes_sorted = sorted(episodes, key=lambda e: e.get('number'))
         season_file = series_folder / f"{season_key}.json"
-        save_to_json(episodes_sorted, season_file)
+        await save_to_json(episodes_sorted, season_file)
 
-def main(series_id):
-    """Main function to fetch and cache episodes."""
-    fetch_and_cache_seasons(series_id)
+async def main(series_id):
+    """Main function to fetch and cache episodes asynchronously."""
+    await fetch_and_cache_seasons(series_id)
 
 if __name__ == "__main__":
+    import asyncio
     # Replace with your series ID
     SERIES_ID = "315103"
-    main(SERIES_ID)
+    asyncio.run(main(SERIES_ID))