init

.gitignore

@@ -0,0 +1,10 @@
+#.env
+.env
+# cache
+tmp
+# pycache
+__pycache__
+# stream-test.py
+stream-test.py
+#test
+test.py

README.md

@@ -1,12 +1,308 @@
 ---
 title: Load Balancer
-emoji: 🐨
-colorFrom: indigo
-colorTo: pink
+emoji: 🚀
+colorFrom: purple
+colorTo: red
 sdk: gradio
-sdk_version: 4.39.0
+sdk_version: 4.36.1
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+## Scripts
+```
+app.py         ->  main script that run flask server
+hf_scrapper.py ->  script for interacting with huggingface
+indexer.py     ->  script to index the repo structure
+tvdb.py        ->  script to interact with TheTVDB
+```
+## Film and TV API
+
+This API provides endpoints for accessing and managing film and TV show data, including downloading, caching, and retrieving metadata.
+
+## Table of Contents
+
+
+- [Base URL](#base-url)
+- [Endpoints](#endpoints)
+  - [Film Endpoints](#film-endpoints)
+  - [TV Show Endpoints](#tv-show-endpoints)
+  - [Cache Endpoints](#cache-endpoints)
+  - [Metadata Endpoints](#metadata-endpoints)
+  - [Miscellaneous Endpoints](#miscellaneous-endpoints)
+- [Error Handling](#error-handling)
+- [Running the Server](#running-the-server)
+
+
+## Base URL
+
+All endpoints are accessed through the base URL:
+
+```markdown
+http://<server-address>:7860
+```
+
+Replace `<server-address>` with your server's address.
+
+## Endpoints
+
+### Film Endpoints
+
+#### `GET /api/film`
+
+**Description:** Starts the download of a film if it's not already cached.
+
+**Query Parameters:**
+- `title` (string): The title of the film.
+
+**Responses:**
+- `200 OK`: Download started successfully.
+  ```json
+  {
+    "status": "Download started",
+    "film_id": "film_id_here"
+  }
+  ```
+- `400 Bad Request`: Title parameter is required.
+  ```json
+  {
+    "error": "Title parameter is required"
+  }
+  ```
+- `404 Not Found`: Movie not found.
+
+#### `GET /api/film/store`
+
+**Description:** Retrieves the JSON data for the film store.
+
+**Responses:**
+- `200 OK`: Returns the film store JSON data.
+  ```json
+  {
+    "film_title": "cache_path_here"
+  }
+  ```
+- `404 Not Found`: Film store JSON not found.
+
+#### `GET /api/film/metadata`
+
+**Description:** Retrieves metadata for a film by title.
+
+**Query Parameters:**
+- `title` (string): The title of the film.
+
+**Responses:**
+- `200 OK`: Returns the metadata JSON for the film.
+  ```json
+  {
+    "title": "Film Title",
+    "year": 2024,
+    "metadata": { ... }
+  }
+  ```
+- `400 Bad Request`: No title provided.
+  ```json
+  {
+    "error": "No title provided"
+  }
+  ```
+- `404 Not Found`: Metadata not found.
+
+### TV Show Endpoints
+
+#### `GET /api/tv`
+
+**Description:** Starts the download of a TV show episode if it's not already cached.
+
+**Query Parameters:**
+- `title` (string): The title of the TV show.
+- `season` (string): The season number.
+- `episode` (string): The episode number.
+
+**Responses:**
+- `200 OK`: Download started successfully.
+  ```json
+  {
+    "status": "Download started",
+    "episode_id": "episode_id_here"
+  }
+  ```
+- `400 Bad Request`: Title, season, and episode parameters are required.
+  ```json
+  {
+    "error": "Title, season, and episode parameters are required"
+  }
+  ```
+- `404 Not Found`: TV show or episode not found.
+
+#### `GET /api/tv/store`
+
+**Description:** Retrieves the JSON data for the TV store.
+
+**Responses:**
+- `200 OK`: Returns the TV store JSON data.
+  ```json
+  {
+    "show_title": {
+      "season": {
+        "episode": "cache_path_here"
+      }
+    }
+  }
+  ```
+- `404 Not Found`: TV store JSON not found.
+
+#### `GET /api/tv/metadata`
+
+**Description:** Retrieves metadata for a TV show by title.
+
+**Query Parameters:**
+- `title` (string): The title of the TV show.
+
+**Responses:**
+- `200 OK`: Returns the metadata JSON for the TV show.
+  ```json
+  {
+    "title": "TV Show Title",
+    "seasons": [ ... ],
+    "metadata": { ... }
+  }
+  ```
+- `400 Bad Request`: No title provided.
+  ```json
+  {
+    "error": "No title provided"
+  }
+  ```
+- `404 Not Found`: Metadata not found.
+
+### Cache Endpoints
+
+#### `GET /api/cache/size`
+
+**Description:** Retrieves the total size of the cache.
+
+**Responses:**
+- `200 OK`: Returns the cache size in a human-readable format.
+  ```json
+  {
+    "cache_size": "10.5 MB"
+  }
+  ```
+
+#### `POST /api/cache/clear`
+
+**Description:** Clears the entire cache.
+
+**Responses:**
+- `200 OK`: Cache cleared successfully.
+  ```json
+  {
+    "status": "Cache cleared"
+  }
+  ```
+
+### Metadata Endpoints
+
+#### `GET /api/filmid`
+
+**Description:** Retrieves the film ID by title.
+
+**Query Parameters:**
+- `title` (string): The title of the film.
+
+**Responses:**
+- `200 OK`: Returns the film ID.
+  ```json
+  {
+    "film_id": "film_id_here"
+  }
+  ```
+- `400 Bad Request`: Title parameter is required.
+  ```json
+  {
+    "error": "Title parameter is required"
+  }
+  ```
+
+#### `GET /api/episodeid`
+
+**Description:** Retrieves the episode ID by title, season, and episode.
+
+**Query Parameters:**
+- `title` (string): The title of the TV show.
+- `season` (string): The season number.
+- `episode` (string): The episode number.
+
+**Responses:**
+- `200 OK`: Returns the episode ID.
+  ```json
+  {
+    "episode_id": "episode_id_here"
+  }
+  ```
+- `400 Bad Request`: Title, season, and episode parameters are required.
+  ```json
+  {
+    "error": "Title, season, and episode parameters are required"
+  }
+  ```
+
+### Miscellaneous Endpoints
+
+#### `GET /api/film/all`
+
+**Description:** Retrieves a list of all films.
+
+**Responses:**
+- `200 OK`: Returns a list of film paths.
+  ```json
+  [
+    "film_path_1",
+    "film_path_2"
+  ]
+  ```
+
+#### `GET /api/tv/all`
+
+**Description:** Retrieves a list of all TV shows.
+
+**Responses:**
+- `200 OK`: Returns a list of TV shows with their episodes.
+  ```json
+  {
+    "show_title": [
+      {
+        "season": "season_number",
+        "episode": "episode_title"
+      }
+    ]
+  }
+  ```
+
+## Error Handling
+
+All endpoints return standard HTTP status codes:
+- `200 OK` for successful requests.
+- `400 Bad Request` for invalid requests.
+- `404 Not Found` for missing resources.
+
+Errors are returned in the following format:
+```json
+{
+  "error": "Error message here"
+}
+```
+
+## Running the Server
+
+To run the server, ensure you have all required dependencies installed and use the following command:
+
+```bash
+python app.py
+```
+
+The server will start on `http://0.0.0.0:7860` by default.
+
+---

app.py

@@ -0,0 +1,377 @@
+from flask import Flask, jsonify, request, send_from_directory
+from flask_cors import CORS
+import os
+import json
+import threading
+import urllib.parse
+from hf_scrapper import download_film, download_episode, get_system_proxies, get_download_progress
+from indexer import indexer
+from tvdb import fetch_and_cache_json
+import re
+
+app = Flask(__name__)
+CORS(app)
+
+# Constants and Configuration
+CACHE_DIR = os.getenv("CACHE_DIR")
+INDEX_FILE = os.getenv("INDEX_FILE")
+TOKEN = os.getenv("TOKEN")
+FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
+TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
+REPO = os.getenv("REPO")
+download_threads = {}
+
+# Ensure CACHE_DIR exists
+if not os.path.exists(CACHE_DIR):
+    os.makedirs(CACHE_DIR)
+
+for path in [FILM_STORE_JSON_PATH, TV_STORE_JSON_PATH]:
+    if not os.path.exists(path):
+        with open(path, 'w') as json_file:
+            json.dump({}, json_file)
+
+# Index the file structure
+indexer()
+
+# Load the file structure JSON
+if not os.path.exists(INDEX_FILE):
+    raise FileNotFoundError(f"{INDEX_FILE} not found. Please make sure the file exists.")
+
+with open(INDEX_FILE, 'r') as f:
+    file_structure = json.load(f)
+
+# Function Definitions
+
+def load_json(file_path):
+    """Load JSON data from a file."""
+    with open(file_path, 'r') as file:
+        return json.load(file)
+
+def find_movie_path(json_data, title):
+    """Find the path of the movie in the JSON data based on the title."""
+    for directory in json_data:
+        if directory['type'] == 'directory' and directory['path'] == 'films':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory':
+                    for item in sub_directory['contents']:
+                        if item['type'] == 'file' and title.lower() in item['path'].lower():
+                            return item['path']
+    return None
+
+def find_tv_path(json_data, title):
+    """Find the path of the TV show in the JSON data based on the title."""
+    for directory in json_data:
+        if directory['type'] == 'directory' and directory['path'] == 'tv':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
+                    return sub_directory['path']
+    return None
+
+def get_tv_structure(json_data,title):
+    """Find the path of the TV show in the JSON data based on the title."""
+    for directory in json_data:
+        if directory['type'] == 'directory' and directory['path'] == 'tv':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
+                    return sub_directory
+    return None
+
+def get_film_id(title):
+    """Generate a film ID based on the title."""
+    return title.replace(" ", "_").lower()
+
+def prefetch_metadata():
+    """Prefetch metadata for all items in the file structure."""
+    for item in file_structure:
+        if 'contents' in item:
+            for sub_item in item['contents']:
+                original_title = sub_item['path'].split('/')[-1]
+                media_type = 'series' if item['path'].startswith('tv') else 'movie'
+                title = original_title
+                year = None
+                
+                # Extract year from the title if available
+                match = re.search(r'\((\d{4})\)', original_title)
+                if match:
+                    year_str = match.group(1)
+                    if year_str.isdigit() and len(year_str) == 4:
+                        title = original_title[:match.start()].strip()
+                        year = int(year_str)
+                else:
+                    parts = original_title.rsplit(' ', 1)
+                    if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
+                        title = parts[0].strip()
+                        year = int(parts[-1])
+                
+                fetch_and_cache_json(original_title, title, media_type, year)
+
+def bytes_to_human_readable(num, suffix="B"):
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(num) < 1024.0:
+            return f"{num:3.1f} {unit}{suffix}"
+        num /= 1024.0
+    return f"{num:.1f} Y{suffix}"
+
+def encode_episodeid(title,season,episode):
+    return f"{title}_{season}_{episode}"
+
+def get_all_tv_shows(indexed_cache):
+    """Get all TV shows from the indexed cache structure JSON file."""
+    tv_shows = {}
+    for directory in indexed_cache:
+        if directory['type'] == 'directory' and directory['path'] == 'tv':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory':
+                    show_title = sub_directory['path'].split('/')[-1]
+                    tv_shows[show_title] = []
+                    for season_directory in sub_directory['contents']:
+                        if season_directory['type'] == 'directory':
+                            season = season_directory['path'].split('/')[-1]
+                            for episode in season_directory['contents']:
+                                if episode['type'] == 'file':
+                                    tv_shows[show_title].append({
+                                        "season": season,
+                                        "episode": episode['path'].split('/')[-1],
+                                        "path": episode['path']
+                                    })
+    return tv_shows
+
+def get_all_films(indexed_cache):
+    """Get all films from the indexed cache structure JSON file."""
+    films = []
+    for directory in indexed_cache:
+        if directory['type'] == 'directory' and directory['path'] == 'films':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory':
+                    films.append(sub_directory['path'])
+    return films
+
+def start_prefetching():
+    """Start the metadata prefetching in a separate thread."""
+    prefetch_metadata()
+
+# Start prefetching metadata
+thread = threading.Thread(target=start_prefetching)
+thread.daemon = True
+thread.start()
+
+# API Endpoints
+
+@app.route('/api/film', methods=['GET'])
+def get_movie_api():
+    """Endpoint to get the movie by title."""
+    title = request.args.get('title')
+    if not title:
+        return jsonify({"error": "Title parameter is required"}), 400
+    
+    # Load the film store JSON
+    with open(FILM_STORE_JSON_PATH, 'r') as json_file:
+        film_store_data = json.load(json_file)
+
+    # Check if the film is already cached
+    if title in film_store_data:
+        cache_path = film_store_data[title]
+        if os.path.exists(cache_path):
+            return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
+    
+    movie_path = find_movie_path(file_structure, title)
+    
+    if not movie_path:
+        return jsonify({"error": "Movie not found"}), 404
+    
+    cache_path = os.path.join(CACHE_DIR, movie_path)
+    file_url = f"https://huggingface.co/{REPO}/resolve/main/{movie_path}"
+    proxies = get_system_proxies()
+    film_id = get_film_id(title)
+    
+    # Start the download in a separate thread if not already downloading
+    if film_id not in download_threads or not download_threads[film_id].is_alive():
+        thread = threading.Thread(target=download_film, args=(file_url, TOKEN, cache_path, proxies, film_id, title))
+        download_threads[film_id] = thread
+        thread.start()
+    
+    return jsonify({"status": "Download started", "film_id": film_id})
+
+@app.route('/api/tv', methods=['GET'])
+def get_tv_show_api():
+    """Endpoint to get the TV show by title, season, and episode."""
+    title = request.args.get('title')
+    season = request.args.get('season')
+    episode = request.args.get('episode')
+    
+    if not title or not season or not episode:
+        return jsonify({"error": "Title, season, and episode parameters are required"}), 400
+
+    # Load the TV store JSON
+    with open(TV_STORE_JSON_PATH, 'r') as json_file:
+        tv_store_data = json.load(json_file)
+
+    # Check if the episode is already cached
+    if title in tv_store_data and season in tv_store_data[title]:
+        for ep in tv_store_data[title][season]:
+            if episode in ep:
+                cache_path = tv_store_data[title][season][ep]
+                if os.path.exists(cache_path):
+                    return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
+
+    tv_path = find_tv_path(file_structure, title)
+    
+    if not tv_path:
+        return jsonify({"error": "TV show not found"}), 404
+
+    episode_path = None
+    for directory in file_structure:
+        if directory['type'] == 'directory' and directory['path'] == 'tv':
+            for sub_directory in directory['contents']:
+                if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
+                    for season_dir in sub_directory['contents']:
+                        if season_dir['type'] == 'directory' and season in season_dir['path']:
+                            for episode_file in season_dir['contents']:
+                                if episode_file['type'] == 'file' and episode in episode_file['path']:
+                                    episode_path = episode_file['path']
+                                    break
+
+    if not episode_path:
+        return jsonify({"error": "Episode not found"}), 404
+    
+    cache_path = os.path.join(CACHE_DIR, episode_path)
+    file_url = f"https://huggingface.co/{REPO}/resolve/main/{episode_path}"
+    proxies = get_system_proxies()
+    episode_id = encode_episodeid(title,season,episode)
+    
+    # Start the download in a separate thread if not already downloading
+    if episode_id not in download_threads or not download_threads[episode_id].is_alive():
+        thread = threading.Thread(target=download_episode, args=(file_url, TOKEN, cache_path, proxies, episode_id, title))
+        download_threads[episode_id] = thread
+        thread.start()
+    
+    return jsonify({"status": "Download started", "episode_id": episode_id})
+
+
+@app.route('/api/progress/<id>', methods=['GET'])
+def get_progress_api(id):
+    """Endpoint to get the download progress of a movie or TV show episode."""
+    progress = get_download_progress(id)
+    return jsonify({"id": id, "progress": progress})
+
+@app.route('/api/cache/size', methods=['GET'])
+def get_cache_size_api():
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(CACHE_DIR):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            total_size += os.path.getsize(fp)
+    readable_size = bytes_to_human_readable(total_size)
+    return jsonify({"cache_size": readable_size})
+
+@app.route('/api/cache/clear', methods=['POST'])
+def clear_cache_api():
+    for dirpath, dirnames, filenames in os.walk(CACHE_DIR):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            os.remove(fp)
+    return jsonify({"status": "Cache cleared"})
+
+@app.route('/api/tv/store', methods=['GET'])
+def get_tv_store_api():
+    """Endpoint to get the TV store JSON."""
+    if os.path.exists(TV_STORE_JSON_PATH):
+        with open(TV_STORE_JSON_PATH, 'r') as json_file:
+            tv_store_data = json.load(json_file)
+        return jsonify(tv_store_data)
+    return jsonify({}), 404
+
+@app.route('/api/film/store', methods=['GET'])
+def get_film_store_api():
+    """Endpoint to get the TV store JSON."""
+    if os.path.exists(FILM_STORE_JSON_PATH):
+        with open(FILM_STORE_JSON_PATH, 'r') as json_file:
+            tv_store_data = json.load(json_file)
+        return jsonify(tv_store_data)
+    return jsonify({}), 404
+
+#################################################
+# No change needed
+
+@app.route('/api/filmid', methods=['GET'])
+def get_film_id_by_title_api():
+    """Endpoint to get the film ID by providing the movie title."""
+    title = request.args.get('title')
+    if not title:
+        return jsonify({"error": "Title parameter is required"}), 400
+    film_id = get_film_id(title)
+    return jsonify({"film_id": film_id})
+
+@app.route('/api/episodeid', methods=['GET'])
+def get_episode_id_api():
+    """Endpoint to get the episode ID by providing the TV show title, season, and episode."""
+    title = request.args.get('title')
+    season = request.args.get('season')
+    episode = request.args.get('episode')
+    if not title or not season or not episode:
+        return jsonify({"error": "Title, season, and episode parameters are required"}), 400
+    episode_id = encode_episodeid(title,season,episode)
+    return jsonify({"episode_id": episode_id})
+
+@app.route('/api/film/metadata', methods=['GET'])
+def get_film_metadata_api():
+    """Endpoint to get the film metadata by title."""
+    title = request.args.get('title')
+    if not title:
+        return jsonify({'error': 'No title provided'}), 400
+    
+    json_cache_path = os.path.join(CACHE_DIR, f"{urllib.parse.quote(title)}.json")
+    
+    if os.path.exists(json_cache_path):
+        with open(json_cache_path, 'r') as f:
+            data = json.load(f)
+        return jsonify(data)
+    
+    return jsonify({'error': 'Metadata not found'}), 404
+
+@app.route('/api/tv/metadata', methods=['GET'])
+def get_tv_metadata_api():
+    """Endpoint to get the TV show metadata by title."""
+    title = request.args.get('title')
+    if not title:
+        return jsonify({'error': 'No title provided'}), 400
+    
+    json_cache_path = os.path.join(CACHE_DIR, f"{urllib.parse.quote(title)}.json")
+    
+    if os.path.exists(json_cache_path):
+        with open(json_cache_path, 'r') as f:
+            data = json.load(f)
+        
+        # Add the file structure to the metadata
+        tv_structure_data = get_tv_structure(file_structure, title)
+        if tv_structure_data:
+            data['file_structure'] = tv_structure_data
+        
+        return jsonify(data)
+    
+    return jsonify({'error': 'Metadata not found'}), 404
+
+
+@app.route("/api/film/all")
+def get_all_films_api():
+    return get_all_films(file_structure)
+
+@app.route("/api/tv/all")
+def get_all_tvshows_api():
+    return get_all_tv_shows(file_structure)
+
+#############################################################
+# unique api's
+@app.route('/api/register/<instanceid>',methodes=['POST'])
+def register_instance(instanceid):
+    # need to add instance registration logic
+    return jsonify({f'{instanceid} registered'})
+
+# Routes
+@app.route('/')
+def index():
+    return "Load Balancer is Running ..."
+
+# Main entry point
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0", port=7860)

hf_scrapper.py

@@ -0,0 +1,249 @@
+import os
+import requests
+import json
+import urllib.request
+import time
+from requests.exceptions import RequestException
+from tqdm import tqdm
+
+CACHE_DIR = os.getenv("CACHE_DIR")
+CACHE_JSON_PATH = os.path.join(CACHE_DIR, "cached_films.json")
+
+download_progress = {}
+
+def get_system_proxies():
+    """
+    Retrieves the system's HTTP and HTTPS proxies.
+
+    Returns:
+        dict: A dictionary containing the proxies.
+    """
+    try:
+        proxies = urllib.request.getproxies()
+        print("System proxies:", proxies)
+        return {
+            "http": proxies.get("http"),
+            "https": proxies.get("http")
+        }
+    except Exception as e:
+        print(f"Error getting system proxies: {e}")
+        return {}
+
+def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
+    """
+    Downloads a file from the specified URL and saves it to the cache path.
+    Tracks the download progress.
+
+    Args:
+        file_url (str): The URL of the file to download.
+        token (str): The authorization token for the request.
+        cache_path (str): The path to save the downloaded file.
+        proxies (dict): Proxies for the request.
+        film_id (str): Unique identifier for the film download.
+        title (str): The title of the film.
+        chunk_size (int): Size of each chunk to download.
+    """
+    print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+    headers = {'Authorization': f'Bearer {token}'}
+    try:
+        response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+        response.raise_for_status()
+        
+        total_size = int(response.headers.get('content-length', 0))
+        download_progress[film_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
+
+        os.makedirs(os.path.dirname(cache_path), exist_ok=True)
+        with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
+            for data in response.iter_content(chunk_size=chunk_size):
+                file.write(data)
+                pbar.update(len(data))
+                download_progress[film_id]["downloaded"] += len(data)
+        
+        print(f'File cached to {cache_path} successfully.')
+        update_film_store_json(title, cache_path)
+        download_progress[film_id]["status"] = "Completed"
+    except RequestException as e:
+        print(f"Error downloading file: {e}")
+        download_progress[film_id]["status"] = "Failed"
+    except IOError as e:
+        print(f"Error writing file {cache_path}: {e}")
+        download_progress[film_id]["status"] = "Failed"
+    finally:
+        if download_progress[film_id]["status"] != "Downloading":
+            download_progress[film_id]["end_time"] = time.time()
+
+def get_download_progress(id):
+    """
+    Gets the download progress for a specific film.
+
+    Args:
+        film_id (str): The unique identifier for the film download.
+
+    Returns:
+        dict: A dictionary containing the total size, downloaded size, progress percentage, status, and ETA.
+    """
+    if id in download_progress:
+        total = download_progress[id]["total"]
+        downloaded = download_progress[id]["downloaded"]
+        status = download_progress[id].get("status", "In Progress")
+        progress = (downloaded / total) * 100 if total > 0 else 0
+
+        eta = None
+        if status == "Downloading" and downloaded > 0:
+            elapsed_time = time.time() - download_progress[id]["start_time"]
+            estimated_total_time = elapsed_time * (total / downloaded)
+            eta = estimated_total_time - elapsed_time
+        elif status == "Completed":
+            eta = 0
+
+        return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
+    return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
+
+def update_film_store_json(title, cache_path):
+    """
+    Updates the film store JSON with the new file.
+
+    Args:
+        title (str): The title of the film.
+        cache_path (str): The local path where the file is saved.
+    """
+    FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
+
+    film_store_data = {}
+    if os.path.exists(FILM_STORE_JSON_PATH):
+        with open(FILM_STORE_JSON_PATH, 'r') as json_file:
+            film_store_data = json.load(json_file)
+
+    film_store_data[title] = cache_path
+
+    with open(FILM_STORE_JSON_PATH, 'w') as json_file:
+        json.dump(film_store_data, json_file, indent=2)
+    print(f'Film store updated with {title}.')
+
+
+###############################################################################
+def download_episode(file_url, token, cache_path, proxies, episode_id, title, chunk_size=100 * 1024 * 1024):
+    """
+    Downloads a file from the specified URL and saves it to the cache path.
+    Tracks the download progress.
+
+    Args:
+        file_url (str): The URL of the file to download.
+        token (str): The authorization token for the request.
+        cache_path (str): The path to save the downloaded file.
+        proxies (dict): Proxies for the request.
+        film_id (str): Unique identifier for the film download.
+        title (str): The title of the film.
+        chunk_size (int): Size of each chunk to download.
+    """
+    print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+    headers = {'Authorization': f'Bearer {token}'}
+    try:
+        response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+        response.raise_for_status()
+        
+        total_size = int(response.headers.get('content-length', 0))
+        download_progress[episode_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
+
+        os.makedirs(os.path.dirname(cache_path), exist_ok=True)
+        with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
+            for data in response.iter_content(chunk_size=chunk_size):
+                file.write(data)
+                pbar.update(len(data))
+                download_progress[episode_id]["downloaded"] += len(data)
+        
+        print(f'File cached to {cache_path} successfully.')
+        update_tv_store_json(title, cache_path)
+        download_progress[episode_id]["status"] = "Completed"
+    except RequestException as e:
+        print(f"Error downloading file: {e}")
+        download_progress[episode_id]["status"] = "Failed"
+    except IOError as e:
+        print(f"Error writing file {cache_path}: {e}")
+        download_progress[episode_id]["status"] = "Failed"
+    finally:
+        if download_progress[episode_id]["status"] != "Downloading":
+            download_progress[episode_id]["end_time"] = time.time()
+
+
+def update_tv_store_json(title, cache_path):
+    """
+    Updates the TV store JSON with the new file, organizing by title, season, and episode.
+
+    Args:
+        title (str): The title of the TV show.
+        cache_path (str): The local path where the file is saved.
+    """
+    TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
+
+    tv_store_data = {}
+    if os.path.exists(TV_STORE_JSON_PATH):
+        with open(TV_STORE_JSON_PATH, 'r') as json_file:
+            tv_store_data = json.load(json_file)
+
+    # Extract season and episode information from the cache_path
+    season_part = os.path.basename(os.path.dirname(cache_path))  # Extracts 'Season 1'
+    episode_part = os.path.basename(cache_path)  # Extracts 'Grand Blue Dreaming - S01E01 - Deep Blue HDTV-720p.mp4'
+
+    # Create the structure if not already present
+    if title not in tv_store_data:
+        tv_store_data[title] = {}
+    
+    if season_part not in tv_store_data[title]:
+        tv_store_data[title][season_part] = {}
+
+    # Assuming episode_part is unique for each episode within a season
+    tv_store_data[title][season_part][episode_part] = cache_path
+
+    with open(TV_STORE_JSON_PATH, 'w') as json_file:
+        json.dump(tv_store_data, json_file, indent=2)
+
+    print(f'TV store updated with {title}, {season_part}, {episode_part}.')
+
+###############################################################################
+def get_file_structure(repo, token, path="", proxies=None):
+    """
+    Fetches the file structure of a specified Hugging Face repository.
+
+    Args:
+        repo (str): The name of the repository.
+        token (str): The authorization token for the request.
+        path (str, optional): The specific path in the repository. Defaults to "".
+        proxies (dict, optional): The proxies to use for the request. Defaults to None.
+
+    Returns:
+        list: A list of file structure information.
+    """
+    api_url = f"https://huggingface.co/api/models/{repo}/tree/main/{path}"
+    headers = {'Authorization': f'Bearer {token}'}
+    print(f"Fetching file structure from URL: {api_url} with proxies: {proxies}")
+    try:
+        response = requests.get(api_url, headers=headers, proxies=proxies)
+        response.raise_for_status()
+        return response.json()
+    except RequestException as e:
+        print(f"Error fetching file structure: {e}")
+        return []
+
+def write_file_structure_to_json(file_structure, file_path):
+    """
+    Writes the file structure to a JSON file.
+
+    Args:
+        file_structure (list): The file structure data.
+        file_path (str): The path where the JSON file will be saved.
+    """
+    try:
+        with open(file_path, 'w') as json_file:
+            json.dump(file_structure, json_file, indent=2)
+        print(f'File structure written to {file_path}')
+    except IOError as e:
+        print(f"Error writing file structure to JSON: {e}")
+
+if __name__ == "__main__":
+    file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
+    token = os.getenv("TOKEN")
+    cache_path = os.path.join(CACHE_DIR, "films/Funky Monkey 2004/Funky Monkey (2004) Web-dl 1080p.mp4")
+    proxies = get_system_proxies()
+    film_id = "funky_monkey_2004"  # Unique identifier for the film download
+    download_film(file_url, token, cache_path, proxies=proxies, film_id=film_id)

indexer.py

@@ -0,0 +1,32 @@
+import json
+from hf_scrapper import get_system_proxies, get_file_structure, write_file_structure_to_json
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+def index_repository(token, repo, current_path="", proxies=None):
+    file_structure = get_file_structure(repo, token, current_path, proxies)
+    full_structure = []
+    for item in file_structure:
+        if item['type'] == 'directory':
+            sub_directory_structure = index_repository(token, repo, item['path'], proxies)
+            full_structure.append({
+                "type": "directory",
+                "path": item['path'],
+                "contents": sub_directory_structure
+            })
+        else:
+            full_structure.append(item)
+    return full_structure
+
+def indexer():
+    token = os.getenv("TOKEN")
+    repo = os.getenv("REPO")
+    output_path = os.getenv("INDEX_FILE")
+    
+    proxies = get_system_proxies()
+    full_structure = index_repository(token, repo, "", proxies)
+    write_file_structure_to_json(full_structure, output_path)
+    print(f"Full file structure for repository '{repo}' has been indexed and saved to {output_path}")
+    

requirements.txt

@@ -0,0 +1,5 @@
+flask
+Flask-Cors
+requests
+python-dotenv
+tqdm

tvdb.py

@@ -0,0 +1,70 @@
+# tvdb.py
+import os
+import requests
+import urllib.parse
+from datetime import datetime, timedelta
+from dotenv import load_dotenv
+import json
+from hf_scrapper import get_system_proxies
+
+load_dotenv()
+THETVDB_API_KEY = os.getenv("THETVDB_API_KEY")
+THETVDB_API_URL = os.getenv("THETVDB_API_URL")
+CACHE_DIR = os.getenv("CACHE_DIR")
+TOKEN_EXPIRY = None
+THETVDB_TOKEN = None
+
+
+proxies = get_system_proxies()
+
+def authenticate_thetvdb():
+    global THETVDB_TOKEN, TOKEN_EXPIRY
+    auth_url = f"{THETVDB_API_URL}/login"
+    auth_data = {
+        "apikey": THETVDB_API_KEY
+    }
+    try:
+        response = requests.post(auth_url, json=auth_data, proxies=proxies)
+        response.raise_for_status()
+        response_data = response.json()
+        THETVDB_TOKEN = response_data['data']['token']
+        TOKEN_EXPIRY = datetime.now() + timedelta(days=30)
+    except requests.RequestException as e:
+        print(f"Authentication failed: {e}")
+        THETVDB_TOKEN = None
+        TOKEN_EXPIRY = None
+
+def get_thetvdb_token():
+    global THETVDB_TOKEN, TOKEN_EXPIRY
+    if not THETVDB_TOKEN or datetime.now() >= TOKEN_EXPIRY:
+        authenticate_thetvdb()
+    return THETVDB_TOKEN
+
+def fetch_and_cache_json(original_title, title, media_type, year=None):
+    if year:
+        search_url = f"{THETVDB_API_URL}/search?query={urllib.parse.quote(title)}&type={media_type}&year={year}"
+    else:
+        search_url = f"{THETVDB_API_URL}/search?query={urllib.parse.quote(title)}&type={media_type}"
+    
+    token = get_thetvdb_token()
+    if not token:
+        print("Authentication failed")
+        return
+    
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "accept": "application/json",
+    }
+    
+    try:
+        response = requests.get(search_url, headers=headers, proxies=proxies)
+        response.raise_for_status()
+        data = response.json()
+        
+        if 'data' in data and data['data']:
+            json_cache_path = os.path.join(CACHE_DIR, f"{urllib.parse.quote(original_title)}.json")
+            with open(json_cache_path, 'w') as f:
+                json.dump(data, f)
+                
+    except requests.RequestException as e:
+        print(f"Error fetching data: {e}")