Spaces:
Paused
Paused
File size: 7,464 Bytes
0942c27 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
# model_loader.py
import json
import os
import glob
from huggingface_hub import hf_hub_download, list_repo_files
# --- Globale Konfiguration und Variablen ---
# Diese Variablen werden von app.py importiert.
MODEL_CONFIG_FILE = "models.json"
DEFAULT_LOCAL_DIR = "./models"
MODEL_DROPDOWN_CHOICES = []
MODEL_FILE_MAPPING = {}
os.makedirs(DEFAULT_LOCAL_DIR, exist_ok=True)
# ----------------------------------------------------------------------
def download_models():
"""Liest models.json, lädt Modelldateien mit HF_TOKEN herunter und füllt die globale Map."""
global MODEL_DROPDOWN_CHOICES
global MODEL_FILE_MAPPING
# Sicherstellen, dass die Listen/Maps leer sind, falls die Funktion mehrfach aufgerufen wird
MODEL_DROPDOWN_CHOICES.clear()
MODEL_FILE_MAPPING.clear()
hf_token = os.environ.get('HF_TOKEN')
if not hf_token:
print("⚠️ HF_TOKEN not found")
else:
print(f"🔑 HF_TOKEN found")
try:
with open(MODEL_CONFIG_FILE, 'r') as f:
config = json.load(f)
except FileNotFoundError:
print(f"❌ ERROR: '{MODEL_CONFIG_FILE}' not found.")
MODEL_DROPDOWN_CHOICES.append("ERROR: models.json missing")
return
except json.JSONDecodeError as e:
print(f"❌ ERROR: {MODEL_CONFIG_FILE} is not a valid JSON. Error: {e}")
MODEL_DROPDOWN_CHOICES.append("ERROR: models.json invalid")
return
local_dir = config.get('local_dir', DEFAULT_LOCAL_DIR)
if not os.path.exists(local_dir):
os.makedirs(local_dir, exist_ok=True)
print(f"Local directory {local_dir} created.")
models_list = config.get('models', [])
print(f"✨ Starting download of {len(models_list)} configured models...")
for model_entry in models_list:
name = model_entry.get('name')
repo_id = model_entry.get('repo_id')
file_name = model_entry.get('file_name')
folder_name = model_entry.get('folder_name')
meta_repo = model_entry.get('meta_repo')
meta_file = model_entry.get('meta_file')
if not name or not repo_id:
print(f"⚠️ WARNING: Entry without 'name' or 'repo_id' found: {model_entry}")
continue
MODEL_DROPDOWN_CHOICES.append(name)
# Single File(s) (file_name)
if file_name:
print(f" -> Downloading single file for '{name}': {file_name}")
try:
# Check if file_name contains wildcards
if '*' in file_name or '?' in file_name or '[' in file_name:
all_files = list_repo_files(repo_id=repo_id, token=hf_token)
matching_files = []
for f in all_files:
if glob.fnmatch.fnmatch(f, file_name):
matching_files.append(f)
if not matching_files:
print(f" ⚎ No files matched pattern: {file_name}")
continue
# Download all matching files
for matching_file in matching_files:
print(f" - Downloading {matching_file}")
hf_hub_download(
repo_id=repo_id,
filename=matching_file,
local_dir=local_dir,
token=hf_token
)
# Store the first matching file path for model initialization
MODEL_FILE_MAPPING[name] = os.path.join(local_dir, matching_files[0])
print(f" -> Downloaded {len(matching_files)} files for {name}")
else:
# Regular file download
hf_hub_download(
repo_id=repo_id,
filename=file_name,
local_dir=local_dir,
token=hf_token
)
# Store the full path (relative to installation)
MODEL_FILE_MAPPING[name] = os.path.join(local_dir, file_name)
print(f" -> Downloaded: {name}")
except Exception as e:
print(f"❌ ERROR during download of {name} ({file_name}): {e}")
# Folder Download (folder_name)
elif folder_name:
print(f" -> Downloading folder for '{name}': {folder_name}")
try:
all_files = list_repo_files(repo_id=repo_id, token=hf_token)
files_to_download = sorted([
filename
for filename in all_files
if filename.startswith(f"{folder_name}/")
])
except Exception as e:
print(f"❌ ERROR during listing files in repo {repo_id}: {e}")
continue
if not files_to_download:
print(f"⚠️ WARNING: No files found in folder '{folder_name}'.")
continue
first_part_filename = files_to_download[0]
for filename in files_to_download:
print(f" - Downloading {filename}")
try:
hf_hub_download(
repo_id=repo_id,
filename=filename,
local_dir=local_dir,
token=hf_token
)
except Exception as e:
print(f"❌ ERROR during download of {filename}: {e}")
# For Llama-CPP-Initialization store the path to the first part
# Path: <local_dir>/<folder_name>/<first_file_part>
MODEL_FILE_MAPPING[name] = os.path.join(local_dir, first_part_filename)
print(f" -> Downloaded folder: {name}. First part: {MODEL_FILE_MAPPING[name]}")
# Meta File Download (meta_file and meta_repo)
elif meta_repo and meta_file:
print(f" -> Downloading meta file for '{name}': {meta_file} from {meta_repo}")
try:
# Debug: Print the download parameters
print(f" Debug: Downloading meta file from repo {meta_repo} with filename {meta_file}")
# Download meta file from meta_repo
hf_hub_download(
repo_id=meta_repo,
filename=meta_file,
local_dir=local_dir,
token=hf_token
)
# Store the meta file path for model initialization
MODEL_FILE_MAPPING[name] = os.path.join(local_dir, meta_file)
print(f" -> Downloaded meta file: {name}")
except Exception as e:
print(f"❌ ERROR during download of meta file {name} ({meta_file}): {e}")
# Try to print more details about the error
import traceback
traceback.print_exc()
else:
print(f"⚠️ WARNING: For '{name}' neither 'file_name', 'folder_name' nor 'meta_file' was given.")
print("--- Download process completed. ---")
# --- Global Downloads once started ---
download_models()
# ---------------------------------------- |