Spaces:
Build error
Build error
Commit
·
c603936
1
Parent(s):
159a4b1
update
Browse files- app.py +36 -13
- hf_scrapper.py +17 -14
app.py
CHANGED
|
@@ -13,12 +13,17 @@ app = Flask(__name__)
|
|
| 13 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
| 14 |
INDEX_FILE = os.getenv("INDEX_FILE")
|
| 15 |
TOKEN = os.getenv("TOKEN")
|
|
|
|
| 16 |
download_threads = {}
|
| 17 |
|
| 18 |
# Ensure CACHE_DIR exists
|
| 19 |
if not os.path.exists(CACHE_DIR):
|
| 20 |
os.makedirs(CACHE_DIR)
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
# Index the file structure
|
| 23 |
indexer()
|
| 24 |
|
|
@@ -94,6 +99,17 @@ def get_movie():
|
|
| 94 |
if not title:
|
| 95 |
return jsonify({"error": "Title parameter is required"}), 400
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
json_data = load_json(INDEX_FILE)
|
| 98 |
movie_path = find_movie_path(json_data, title)
|
| 99 |
|
|
@@ -101,20 +117,17 @@ def get_movie():
|
|
| 101 |
return jsonify({"error": "Movie not found"}), 404
|
| 102 |
|
| 103 |
cache_path = os.path.join(CACHE_DIR, movie_path)
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
film_id = get_film_id(title)
|
| 108 |
-
|
| 109 |
-
# Start the download in a separate thread if not already downloading
|
| 110 |
-
if film_id not in download_threads or not download_threads[film_id].is_alive():
|
| 111 |
-
thread = threading.Thread(target=download_file, args=(file_url, TOKEN, cache_path, proxies, film_id))
|
| 112 |
-
download_threads[film_id] = thread
|
| 113 |
-
thread.start()
|
| 114 |
-
|
| 115 |
-
return jsonify({"status": "Download started", "film_id": film_id})
|
| 116 |
|
| 117 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
|
| 119 |
@app.route('/api/progress/<film_id>', methods=['GET'])
|
| 120 |
def get_progress(film_id):
|
|
@@ -131,6 +144,16 @@ def get_film_id_by_title():
|
|
| 131 |
film_id = get_film_id(title)
|
| 132 |
return jsonify({"film_id": film_id})
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
# Main entry point
|
| 135 |
if __name__ == "__main__":
|
| 136 |
app.run(debug=True, host="0.0.0.0", port=7860)
|
|
|
|
| 13 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
| 14 |
INDEX_FILE = os.getenv("INDEX_FILE")
|
| 15 |
TOKEN = os.getenv("TOKEN")
|
| 16 |
+
FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
|
| 17 |
download_threads = {}
|
| 18 |
|
| 19 |
# Ensure CACHE_DIR exists
|
| 20 |
if not os.path.exists(CACHE_DIR):
|
| 21 |
os.makedirs(CACHE_DIR)
|
| 22 |
|
| 23 |
+
if not os.path.exists(FILM_STORE_JSON_PATH):
|
| 24 |
+
with open(FILM_STORE_JSON_PATH, 'w') as json_file:
|
| 25 |
+
json.dump({}, json_file)
|
| 26 |
+
|
| 27 |
# Index the file structure
|
| 28 |
indexer()
|
| 29 |
|
|
|
|
| 99 |
if not title:
|
| 100 |
return jsonify({"error": "Title parameter is required"}), 400
|
| 101 |
|
| 102 |
+
# Load the film store JSON
|
| 103 |
+
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
| 104 |
+
film_store_data = json.load(json_file)
|
| 105 |
+
|
| 106 |
+
# Check if the film is already cached
|
| 107 |
+
if title in film_store_data:
|
| 108 |
+
cache_path = film_store_data[title]
|
| 109 |
+
if os.path.exists(cache_path):
|
| 110 |
+
return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
|
| 111 |
+
|
| 112 |
+
# If not cached, find the movie path in the index file
|
| 113 |
json_data = load_json(INDEX_FILE)
|
| 114 |
movie_path = find_movie_path(json_data, title)
|
| 115 |
|
|
|
|
| 117 |
return jsonify({"error": "Movie not found"}), 404
|
| 118 |
|
| 119 |
cache_path = os.path.join(CACHE_DIR, movie_path)
|
| 120 |
+
file_url = f"https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/{movie_path}"
|
| 121 |
+
proxies = get_system_proxies()
|
| 122 |
+
film_id = get_film_id(title)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
|
| 124 |
+
# Start the download in a separate thread if not already downloading
|
| 125 |
+
if film_id not in download_threads or not download_threads[film_id].is_alive():
|
| 126 |
+
thread = threading.Thread(target=download_file, args=(file_url, TOKEN, cache_path, proxies, film_id, title))
|
| 127 |
+
download_threads[film_id] = thread
|
| 128 |
+
thread.start()
|
| 129 |
+
|
| 130 |
+
return jsonify({"status": "Download started", "film_id": film_id})
|
| 131 |
|
| 132 |
@app.route('/api/progress/<film_id>', methods=['GET'])
|
| 133 |
def get_progress(film_id):
|
|
|
|
| 144 |
film_id = get_film_id(title)
|
| 145 |
return jsonify({"film_id": film_id})
|
| 146 |
|
| 147 |
+
@app.route('/api/film_store', methods=['GET'])
|
| 148 |
+
def get_film_store():
|
| 149 |
+
"""Endpoint to get the film store JSON."""
|
| 150 |
+
if os.path.exists(FILM_STORE_JSON_PATH):
|
| 151 |
+
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
| 152 |
+
film_store_data = json.load(json_file)
|
| 153 |
+
return jsonify(film_store_data)
|
| 154 |
+
return jsonify({}), 404
|
| 155 |
+
|
| 156 |
+
|
| 157 |
# Main entry point
|
| 158 |
if __name__ == "__main__":
|
| 159 |
app.run(debug=True, host="0.0.0.0", port=7860)
|
hf_scrapper.py
CHANGED
|
@@ -28,7 +28,7 @@ def get_system_proxies():
|
|
| 28 |
print(f"Error getting system proxies: {e}")
|
| 29 |
return {}
|
| 30 |
|
| 31 |
-
def download_file(file_url, token, cache_path, proxies, film_id, chunk_size=100 * 1024 * 1024):
|
| 32 |
"""
|
| 33 |
Downloads a file from the specified URL and saves it to the cache path.
|
| 34 |
Tracks the download progress.
|
|
@@ -39,6 +39,7 @@ def download_file(file_url, token, cache_path, proxies, film_id, chunk_size=100
|
|
| 39 |
cache_path (str): The path to save the downloaded file.
|
| 40 |
proxies (dict): Proxies for the request.
|
| 41 |
film_id (str): Unique identifier for the film download.
|
|
|
|
| 42 |
chunk_size (int): Size of each chunk to download.
|
| 43 |
"""
|
| 44 |
print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
|
|
@@ -58,7 +59,7 @@ def download_file(file_url, token, cache_path, proxies, film_id, chunk_size=100
|
|
| 58 |
download_progress[film_id]["downloaded"] += len(data)
|
| 59 |
|
| 60 |
print(f'File cached to {cache_path} successfully.')
|
| 61 |
-
|
| 62 |
except RequestException as e:
|
| 63 |
print(f"Error downloading file: {e}")
|
| 64 |
except IOError as e:
|
|
@@ -84,25 +85,27 @@ def get_download_progress(film_id):
|
|
| 84 |
return {"total": total, "downloaded": downloaded, "progress": progress}
|
| 85 |
return {"total": 0, "downloaded": 0, "progress": 0}
|
| 86 |
|
| 87 |
-
def
|
| 88 |
"""
|
| 89 |
-
Updates the
|
| 90 |
|
| 91 |
Args:
|
| 92 |
-
|
| 93 |
cache_path (str): The local path where the file is saved.
|
| 94 |
"""
|
| 95 |
-
|
| 96 |
-
if os.path.exists(CACHE_JSON_PATH):
|
| 97 |
-
with open(CACHE_JSON_PATH, 'r') as json_file:
|
| 98 |
-
cache_data = json.load(json_file)
|
| 99 |
|
| 100 |
-
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
-
with open(CACHE_JSON_PATH, 'w') as json_file:
|
| 104 |
-
json.dump(cache_data, json_file, indent=2)
|
| 105 |
-
print(f'Cache updated with {film_title}.')
|
| 106 |
|
| 107 |
def get_file_structure(repo, token, path="", proxies=None):
|
| 108 |
"""
|
|
|
|
| 28 |
print(f"Error getting system proxies: {e}")
|
| 29 |
return {}
|
| 30 |
|
| 31 |
+
def download_file(file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
|
| 32 |
"""
|
| 33 |
Downloads a file from the specified URL and saves it to the cache path.
|
| 34 |
Tracks the download progress.
|
|
|
|
| 39 |
cache_path (str): The path to save the downloaded file.
|
| 40 |
proxies (dict): Proxies for the request.
|
| 41 |
film_id (str): Unique identifier for the film download.
|
| 42 |
+
title (str): The title of the film.
|
| 43 |
chunk_size (int): Size of each chunk to download.
|
| 44 |
"""
|
| 45 |
print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
|
|
|
|
| 59 |
download_progress[film_id]["downloaded"] += len(data)
|
| 60 |
|
| 61 |
print(f'File cached to {cache_path} successfully.')
|
| 62 |
+
update_film_store_json(title, cache_path)
|
| 63 |
except RequestException as e:
|
| 64 |
print(f"Error downloading file: {e}")
|
| 65 |
except IOError as e:
|
|
|
|
| 85 |
return {"total": total, "downloaded": downloaded, "progress": progress}
|
| 86 |
return {"total": 0, "downloaded": 0, "progress": 0}
|
| 87 |
|
| 88 |
+
def update_film_store_json(title, cache_path):
|
| 89 |
"""
|
| 90 |
+
Updates the film store JSON with the new file.
|
| 91 |
|
| 92 |
Args:
|
| 93 |
+
title (str): The title of the film.
|
| 94 |
cache_path (str): The local path where the file is saved.
|
| 95 |
"""
|
| 96 |
+
FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
+
film_store_data = {}
|
| 99 |
+
if os.path.exists(FILM_STORE_JSON_PATH):
|
| 100 |
+
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
| 101 |
+
film_store_data = json.load(json_file)
|
| 102 |
+
|
| 103 |
+
film_store_data[title] = cache_path
|
| 104 |
+
|
| 105 |
+
with open(FILM_STORE_JSON_PATH, 'w') as json_file:
|
| 106 |
+
json.dump(film_store_data, json_file, indent=2)
|
| 107 |
+
print(f'Film store updated with {title}.')
|
| 108 |
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
def get_file_structure(repo, token, path="", proxies=None):
|
| 111 |
"""
|