Create model_loader.py

model_loader.py

@@ -0,0 +1,179 @@
+# model_loader.py
+
+import json
+import os
+import glob
+from huggingface_hub import hf_hub_download, list_repo_files
+
+# --- Globale Konfiguration und Variablen ---
+# Diese Variablen werden von app.py importiert.
+MODEL_CONFIG_FILE = "models.json"
+DEFAULT_LOCAL_DIR = "./models"
+MODEL_DROPDOWN_CHOICES = []
+MODEL_FILE_MAPPING = {} 
+
+os.makedirs(DEFAULT_LOCAL_DIR, exist_ok=True)
+# ----------------------------------------------------------------------
+
+def download_models():
+    """Liest models.json, lädt Modelldateien mit HF_TOKEN herunter und füllt die globale Map."""
+    global MODEL_DROPDOWN_CHOICES
+    global MODEL_FILE_MAPPING
+    
+    # Sicherstellen, dass die Listen/Maps leer sind, falls die Funktion mehrfach aufgerufen wird
+    MODEL_DROPDOWN_CHOICES.clear()
+    MODEL_FILE_MAPPING.clear()
+
+    hf_token = os.environ.get('HF_TOKEN')
+    if not hf_token:
+        print("⚠️ HF_TOKEN not found")
+    else:
+        print(f"🔑 HF_TOKEN found")
+
+    try:
+        with open(MODEL_CONFIG_FILE, 'r') as f:
+            config = json.load(f)
+    except FileNotFoundError:
+        print(f"❌ ERROR: '{MODEL_CONFIG_FILE}' not found.")
+        MODEL_DROPDOWN_CHOICES.append("ERROR: models.json missing")
+        return
+    except json.JSONDecodeError as e:
+        print(f"❌ ERROR: {MODEL_CONFIG_FILE} is not a valid JSON. Error: {e}")
+        MODEL_DROPDOWN_CHOICES.append("ERROR: models.json invalid")
+        return
+
+    local_dir = config.get('local_dir', DEFAULT_LOCAL_DIR)
+    if not os.path.exists(local_dir):
+        os.makedirs(local_dir, exist_ok=True)
+        print(f"Local directory {local_dir} created.")
+
+    models_list = config.get('models', [])
+    
+    print(f"✨ Starting download of {len(models_list)} configured models...")
+
+    for model_entry in models_list:
+        name = model_entry.get('name')
+        repo_id = model_entry.get('repo_id')
+        file_name = model_entry.get('file_name')
+        folder_name = model_entry.get('folder_name')
+        meta_repo = model_entry.get('meta_repo')
+        meta_file = model_entry.get('meta_file')
+        
+        if not name or not repo_id:
+            print(f"⚠️ WARNING: Entry without 'name' or 'repo_id' found: {model_entry}")
+            continue
+
+        MODEL_DROPDOWN_CHOICES.append(name)
+        
+        # Single File(s) (file_name)
+        if file_name:
+            print(f"  -> Downloading single file for '{name}': {file_name}")
+            try:
+                # Check if file_name contains wildcards
+                if '*' in file_name or '?' in file_name or '[' in file_name:
+                    all_files = list_repo_files(repo_id=repo_id, token=hf_token)
+                    matching_files = []
+                    for f in all_files:
+                        if glob.fnmatch.fnmatch(f, file_name):
+                            matching_files.append(f)
+                                        
+                    if not matching_files:
+                        print(f"  ⚎ No files matched pattern: {file_name}")
+                        continue
+                                        
+                    # Download all matching files
+                    for matching_file in matching_files:
+                        print(f"    - Downloading {matching_file}")
+                        hf_hub_download(
+                            repo_id=repo_id,
+                            filename=matching_file,
+                            local_dir=local_dir,
+                            token=hf_token
+                        )
+                                        
+                    # Store the first matching file path for model initialization
+                    MODEL_FILE_MAPPING[name] = os.path.join(local_dir, matching_files[0])
+                    print(f"  -> Downloaded {len(matching_files)} files for {name}")
+                else:
+                    # Regular file download
+                    hf_hub_download(
+                        repo_id=repo_id,
+                        filename=file_name,
+                        local_dir=local_dir,
+                        token=hf_token
+                    )
+                    # Store the full path (relative to installation)
+                    MODEL_FILE_MAPPING[name] = os.path.join(local_dir, file_name)
+                    print(f"  -> Downloaded: {name}")
+            except Exception as e:
+                print(f"❌ ERROR during download of {name} ({file_name}): {e}")
+
+        # Folder Download (folder_name)
+        elif folder_name:
+            print(f"  -> Downloading folder for '{name}': {folder_name}")
+            try:
+                all_files = list_repo_files(repo_id=repo_id, token=hf_token)
+                files_to_download = sorted([
+                    filename
+                    for filename in all_files
+                    if filename.startswith(f"{folder_name}/")
+                ])
+            except Exception as e:
+                print(f"❌ ERROR during listing files in repo {repo_id}: {e}")
+                continue
+                        
+            if not files_to_download:
+                print(f"⚠️ WARNING: No files found in folder '{folder_name}'.")
+                continue
+            
+            first_part_filename = files_to_download[0]
+            
+            for filename in files_to_download:
+                print(f"    - Downloading {filename}")
+                try:
+                    hf_hub_download(
+                        repo_id=repo_id,
+                        filename=filename,
+                        local_dir=local_dir,
+                        token=hf_token
+                    )
+                except Exception as e:
+                    print(f"❌ ERROR during download of {filename}: {e}")
+            
+            # For Llama-CPP-Initialization store the path to the first part
+            # Path: <local_dir>/<folder_name>/<first_file_part>
+            MODEL_FILE_MAPPING[name] = os.path.join(local_dir, first_part_filename)
+            print(f"  -> Downloaded folder: {name}. First part: {MODEL_FILE_MAPPING[name]}")
+
+        # Meta File Download (meta_file and meta_repo)
+        elif meta_repo and meta_file:
+            print(f"  -> Downloading meta file for '{name}': {meta_file} from {meta_repo}")
+            try:
+                # Debug: Print the download parameters
+                print(f"    Debug: Downloading meta file from repo {meta_repo} with filename {meta_file}")
+                                
+                # Download meta file from meta_repo
+                hf_hub_download(
+                    repo_id=meta_repo,
+                    filename=meta_file,
+                    local_dir=local_dir,
+                    token=hf_token
+                )
+                # Store the meta file path for model initialization
+                MODEL_FILE_MAPPING[name] = os.path.join(local_dir, meta_file)
+                print(f"  -> Downloaded meta file: {name}")
+            except Exception as e:
+                print(f"❌ ERROR during download of meta file {name} ({meta_file}): {e}")
+                # Try to print more details about the error
+                import traceback
+                traceback.print_exc()
+
+        else:
+            print(f"⚠️ WARNING: For '{name}' neither 'file_name', 'folder_name' nor 'meta_file' was given.")
+            
+    print("--- Download process completed. ---")
+
+
+# --- Global Downloads once started ---
+download_models()
+# ----------------------------------------