Deploy Gradio app with multiple files

app.py

@@ -0,0 +1,398 @@
+import gradio as gr
+import os
+from pathlib import Path
+from typing import List, Dict, Optional, Generator
+import json
+import time
+
+from models import ModelManager
+from utils import get_available_models, format_chat_history, parse_model_info
+
+class ChatbotApp:
+    def __init__(self):
+        self.model_manager = ModelManager()
+        self.current_model = None
+        self.chat_history = []
+        self.system_prompt = "You are a helpful assistant."
+        
+    def load_model(self, model_path: str, context_size: int = 2048, gpu_layers: int = 0) -> str:
+        """Load a GGUF model"""
+        try:
+            if not model_path or not os.path.exists(model_path):
+                return "❌ Please select a valid model file"
+            
+            success = self.model_manager.load_model(
+                model_path=model_path,
+                context_size=context_size,
+                gpu_layers=gpu_layers
+            )
+            
+            if success:
+                self.current_model = Path(model_path).name
+                return f"✅ Successfully loaded: {self.current_model}"
+            else:
+                return "❌ Failed to load model"
+                
+        except Exception as e:
+            return f"❌ Error loading model: {str(e)}"
+    
+    def unload_model(self) -> str:
+        """Unload the current model"""
+        self.model_manager.unload_model()
+        self.current_model = None
+        return "✅ Model unloaded"
+    
+    def chat_response(
+        self, 
+        message: str, 
+        history: List[List[str]], 
+        temperature: float,
+        max_tokens: int,
+        top_p: float,
+        repeat_penalty: float
+    ) -> Generator[str, None, None]:
+        """Generate response from the model"""
+        if not self.model_manager.is_loaded():
+            yield "❌ No model loaded. Please load a model first."
+            return
+        
+        try:
+            # Format chat history
+            formatted_history = format_chat_history(history, self.system_prompt)
+            
+            # Generate response
+            response_text = ""
+            for chunk in self.model_manager.generate(
+                prompt=formatted_history + message,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                top_p=top_p,
+                repeat_penalty=repeat_penalty
+            ):
+                response_text += chunk
+                yield response_text
+                
+        except Exception as e:
+            yield f"❌ Error generating response: {str(e)}"
+    
+    def clear_chat(self) -> List[List[str]]:
+        """Clear chat history"""
+        self.chat_history = []
+        return []
+    
+    def get_model_info(self) -> str:
+        """Get information about the loaded model"""
+        if not self.current_model:
+            return "No model loaded"
+        
+        try:
+            model_info = self.model_manager.get_model_info()
+            if model_info:
+                return json.dumps(model_info, indent=2)
+            return "Model info not available"
+        except Exception as e:
+            return f"Error getting model info: {str(e)}"
+
+def create_interface():
+    """Create the Gradio interface"""
+    app = ChatbotApp()
+    
+    with gr.Blocks(theme=gr.themes.Soft(), title="Local GGUF Chatbot") as demo:
+        gr.Markdown("""
+        # 🤖 Local GGUF Chatbot
+        Built with [anycoder](https://huggingface.co/spaces/akhaliq/anycoder)
+        
+        Chat with local GGUF models using llama.cpp. Load your models and start chatting!
+        """)
+        
+        with gr.Tabs():
+            # Chat Tab
+            with gr.Tab("💬 Chat"):
+                with gr.Row():
+                    with gr.Column(scale=3):
+                        chatbot = gr.Chatbot(
+                            label="Chat",
+                            height=500,
+                            show_copy_button=True,
+                            type="messages"
+                        )
+                        
+                        with gr.Row():
+                            msg = gr.Textbox(
+                                label="Message",
+                                placeholder="Type your message here...",
+                                scale=4
+                            )
+                            send_btn = gr.Button("Send", scale=1)
+                            clear_btn = gr.Button("Clear", scale=1)
+                    
+                    with gr.Column(scale=1):
+                        gr.Markdown("### ⚙️ Generation Parameters")
+                        
+                        temperature = gr.Slider(
+                            minimum=0.1,
+                            maximum=2.0,
+                            value=0.7,
+                            step=0.1,
+                            label="Temperature"
+                        )
+                        
+                        max_tokens = gr.Slider(
+                            minimum=1,
+                            maximum=4096,
+                            value=512,
+                            step=1,
+                            label="Max Tokens"
+                        )
+                        
+                        top_p = gr.Slider(
+                            minimum=0.1,
+                            maximum=1.0,
+                            value=0.9,
+                            step=0.05,
+                            label="Top P"
+                        )
+                        
+                        repeat_penalty = gr.Slider(
+                            minimum=1.0,
+                            maximum=2.0,
+                            value=1.1,
+                            step=0.05,
+                            label="Repeat Penalty"
+                        )
+                        
+                        system_prompt = gr.Textbox(
+                            label="System Prompt",
+                            value="You are a helpful assistant.",
+                            lines=3
+                        )
+                        
+                        model_status = gr.Textbox(
+                            label="Model Status",
+                            value="No model loaded",
+                            interactive=False
+                        )
+            
+            # Model Management Tab
+            with gr.Tab("📁 Models"):
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("### Load Model")
+                        
+                        model_file = gr.File(
+                            label="Select GGUF Model",
+                            file_types=[".gguf"],
+                            file_count="single"
+                        )
+                        
+                        with gr.Row():
+                            context_size = gr.Slider(
+                                minimum=512,
+                                maximum=8192,
+                                value=2048,
+                                step=512,
+                                label="Context Size"
+                            )
+                            
+                            gpu_layers = gr.Slider(
+                                minimum=0,
+                                maximum=99,
+                                value=0,
+                                step=1,
+                                label="GPU Layers"
+                            )
+                        
+                        load_btn = gr.Button("Load Model", variant="primary")
+                        unload_btn = gr.Button("Unload Model")
+                        
+                        load_status = gr.Textbox(
+                            label="Load Status",
+                            interactive=False
+                        )
+                    
+                    with gr.Column():
+                        gr.Markdown("### Available Models")
+                        
+                        available_models = gr.JSON(
+                            label="Models Directory",
+                            value=get_available_models()
+                        )
+                        
+                        model_info = gr.JSON(
+                            label="Model Information",
+                            visible=False
+                        )
+                        
+                        refresh_btn = gr.Button("Refresh Models")
+            
+            # Settings Tab
+            with gr.Tab("⚙️ Settings"):
+                gr.Markdown("### Application Settings")
+                
+                with gr.Row():
+                    with gr.Column():
+                        models_dir = gr.Textbox(
+                            label="Models Directory",
+                            value="./models",
+                            placeholder="Path to models directory"
+                        )
+                        
+                        save_chat = gr.Checkbox(
+                            label="Save Chat History",
+                            value=True
+                        )
+                        
+                        chat_format = gr.Dropdown(
+                            label="Chat Format",
+                            choices=["chatml", "llama2", "alpaca", "vicuna"],
+                            value="chatml"
+                        )
+                
+                with gr.Column():
+                    gr.Markdown("### Model Directory Info")
+                    dir_info = gr.JSON(label="Directory Info")
+                    
+                    update_dir_btn = gr.Button("Update Directory")
+        
+        # Event handlers
+        def update_system_prompt(prompt):
+            app.system_prompt = prompt
+            return prompt
+        
+        system_prompt.change(update_system_prompt, system_prompt)
+        
+        # Chat functionality
+        def user_message(user_input, history):
+            if not user_input.strip():
+                return "", history
+            
+            history.append({"role": "user", "content": user_input})
+            return "", history
+        
+        def bot_response(history, temp, max_tok, top_p_val, repeat_pen):
+            if not history:
+                return history
+            
+            last_message = history[-1]["content"] if history else ""
+            
+            # Add assistant message placeholder
+            history.append({"role": "assistant", "content": ""})
+            
+            # Generate response
+            response = ""
+            for chunk in app.chat_response(
+                last_message,
+                [{"role": h["role"], "content": h["content"]} for h in history[:-2]],
+                temp,
+                max_tok,
+                top_p_val,
+                repeat_pen
+            ):
+                history[-1]["content"] = chunk
+                yield history
+        
+        msg.submit(
+            user_message,
+            [msg, chatbot],
+            [msg, chatbot]
+        ).then(
+            bot_response,
+            [chatbot, temperature, max_tokens, top_p, repeat_penalty],
+            chatbot
+        )
+        
+        send_btn.click(
+            user_message,
+            [msg, chatbot],
+            [msg, chatbot]
+        ).then(
+            bot_response,
+            [chatbot, temperature, max_tokens, top_p, repeat_penalty],
+            chatbot
+        )
+        
+        clear_btn.click(app.clear_chat, outputs=chatbot)
+        
+        # Model loading
+        def handle_model_load(file_obj, ctx_size, gpu_layers):
+            if file_obj is None:
+                return "❌ Please select a model file"
+            
+            status = app.load_model(file_obj.name, ctx_size, gpu_layers)
+            
+            # Update model status
+            if app.current_model:
+                model_status_text = f"✅ Loaded: {app.current_model}"
+            else:
+                model_status_text = "No model loaded"
+            
+            return status, model_status_text
+        
+        load_btn.click(
+            handle_model_load,
+            [model_file, context_size, gpu_layers],
+            [load_status, model_status]
+        )
+        
+        unload_btn.click(
+            app.unload_model,
+            outputs=[load_status, model_status]
+        )
+        
+        # Refresh models
+        refresh_btn.click(
+            lambda: get_available_models(),
+            outputs=available_models
+        )
+        
+        # Update model info when model is loaded
+        def update_model_info():
+            if app.current_model:
+                try:
+                    info = app.model_manager.get_model_info()
+                    return info
+                except:
+                    return None
+            return None
+        
+        demo.load(
+            update_model_info,
+            outputs=[model_info]
+        )
+        
+        # Update directory info
+        def update_directory_info(directory):
+            try:
+                path = Path(directory)
+                if path.exists():
+                    return {
+                        "exists": True,
+                        "is_directory": path.is_dir(),
+                        "file_count": len(list(path.glob("*.gguf"))),
+                        "size_mb": sum(f.stat().st_size for f in path.glob("*.gguf")) / (1024 * 1024)
+                    }
+                else:
+                    return {"exists": False}
+            except Exception as e:
+                return {"error": str(e)}
+        
+        update_dir_btn.click(
+            update_directory_info,
+            models_dir,
+            dir_info
+        )
+    
+    return demo
+
+if __name__ == "__main__":
+    # Create models directory if it doesn't exist
+    os.makedirs("./models", exist_ok=True)
+    
+    # Create and launch the interface
+    demo = create_interface()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        show_api=True
+    )

models.py

@@ -0,0 +1,234 @@
+import os
+from pathlib import Path
+from typing import Optional, Dict, Generator, List
+import json
+import logging
+
+# Try to import llama-cpp-python
+try:
+    from llama_cpp import Llama
+    LLAMA_AVAILABLE = True
+except ImportError:
+    LLAMA_AVAILABLE = False
+    Llama = None
+    logging.warning("llama-cpp-python not installed. Install with: pip install llama-cpp-python")
+
+class ModelManager:
+    """Manages loading and inference of GGUF models"""
+    
+    def __init__(self):
+        self.model: Optional[Llama] = None
+        self.model_path: Optional[str] = None
+        self.context_size: int = 2048
+        self.gpu_layers: int = 0
+        
+    def is_loaded(self) -> bool:
+        """Check if a model is loaded"""
+        return self.model is not None
+    
+    def load_model(
+        self, 
+        model_path: str, 
+        context_size: int = 2048, 
+        gpu_layers: int = 0,
+        n_ctx: Optional[int] = None,
+        n_gpu_layers: Optional[int] = None,
+        verbose: bool = True
+    ) -> bool:
+        """Load a GGUF model"""
+        if not LLAMA_AVAILABLE:
+            logging.error("llama-cpp-python is not installed")
+            return False
+        
+        try:
+            # Unload existing model if any
+            if self.model:
+                self.unload_model()
+            
+            # Set parameters
+            self.context_size = n_ctx or context_size
+            self.gpu_layers = n_gpu_layers or gpu_layers
+            self.model_path = model_path
+            
+            # Load the model
+            self.model = Llama(
+                model_path=model_path,
+                n_ctx=self.context_size,
+                n_gpu_layers=self.gpu_layers,
+                verbose=verbose,
+                embedding=False,
+                f16_kv=True,
+                use_mmap=True,
+                use_mlock=False,
+                logits_all=False,
+                vocab_only=False
+            )
+            
+            logging.info(f"Model loaded successfully: {model_path}")
+            return True
+            
+        except Exception as e:
+            logging.error(f"Failed to load model: {str(e)}")
+            self.model = None
+            self.model_path = None
+            return False
+    
+    def unload_model(self):
+        """Unload the current model"""
+        if self.model:
+            del self.model
+            self.model = None
+            self.model_path = None
+            logging.info("Model unloaded")
+    
+    def generate(
+        self,
+        prompt: str,
+        temperature: float = 0.7,
+        max_tokens: int = 512,
+        top_p: float = 0.9,
+        repeat_penalty: float = 1.1,
+        stop: Optional[List[str]] = None,
+        stream: bool = True
+    ) -> Generator[str, None, None]:
+        """Generate text from the model"""
+        if not self.model:
+            raise ValueError("No model loaded")
+        
+        try:
+            # Generate response
+            if stream:
+                for chunk in self.model(
+                    prompt,
+                    max_tokens=max_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                    repeat_penalty=repeat_penalty,
+                    stop=stop or [],
+                    stream=True
+                ):
+                    if chunk["choices"]:
+                        yield chunk["choices"][0]["text"]
+            else:
+                output = self.model(
+                    prompt,
+                    max_tokens=max_tokens,
+                    temperature=temperature,
+                    top_p=top_p,
+                    repeat_penalty=repeat_penalty,
+                    stop=stop or [],
+                    stream=False
+                )
+                yield output["choices"][0]["text"]
+                
+        except Exception as e:
+            logging.error(f"Generation error: {str(e)}")
+            raise
+    
+    def get_model_info(self) -> Optional[Dict]:
+        """Get information about the loaded model"""
+        if not self.model:
+            return None
+        
+        try:
+            # Extract model metadata
+            metadata = getattr(self.model, 'metadata', {})
+            
+            # Try to get tokenizer info
+            try:
+                vocab_size = len(self.model._model.tokenizer().vocab())
+            except:
+                vocab_size = None
+            
+            # Basic model info
+            info = {
+                "model_path": self.model_path,
+                "context_size": self.context_size,
+                "gpu_layers": self.gpu_layers,
+                "vocab_size": vocab_size,
+            }
+            
+            # Add metadata if available
+            if metadata:
+                # Extract common metadata fields
+                common_fields = [
+                    "general.architecture",
+                    "llama.vocab_size",
+                    "llama.context_length",
+                    "llama.embedding_length",
+                    "llama.block_count",
+                    "llama.feed_forward_length",
+                    "llama.attention.head_count",
+                    "llama.attention.head_count_kv",
+                    "llama.rope.dimension_count",
+                    "llama.attention.layer_norm_rms_epsilon",
+                    "tokenizer.ggml.model",
+                    "tokenizer.ggml.tokens",
+                ]
+                
+                for field in common_fields:
+                    if field in metadata:
+                        info[field] = metadata[field]
+                
+                # Add all metadata as raw for debugging
+                info["raw_metadata"] = {k: v for k, v in metadata.items() 
+                                      if not isinstance(v, (bytes, bytearray))}
+            
+            return info
+            
+        except Exception as e:
+            logging.error(f"Error getting model info: {str(e)}")
+            return {"error": str(e)}
+    
+    def tokenize(self, text: str) -> List[int]:
+        """Tokenize text"""
+        if not self.model:
+            raise ValueError("No model loaded")
+        
+        try:
+            return self.model.tokenize(text.encode("utf-8"))
+        except Exception as e:
+            logging.error(f"Tokenization error: {str(e)}")
+            return []
+    
+    def detokenize(self, tokens: List[int]) -> str:
+        """Detokenize tokens"""
+        if not self.model:
+            raise ValueError("No model loaded")
+        
+        try:
+            return self.model.detokenize(tokens).decode("utf-8")
+        except Exception as e:
+            logging.error(f"Detokenization error: {str(e)}")
+            return ""
+
+def check_model_compatibility(model_path: str) -> Dict:
+    """Check if a model file is compatible"""
+    result = {
+        "exists": False,
+        "readable": False,
+        "gguf": False,
+        "size_mb": 0,
+        "error": None
+    }
+    
+    try:
+        path = Path(model_path)
+        result["exists"] = path.exists()
+        
+        if result["exists"]:
+            result["size_mb"] = path.stat().st_size / (1024 * 1024)
+            result["gguf"] = path.suffix.lower() == ".gguf"
+            
+            # Try to read file header
+            try:
+                with open(path, "rb") as f:
+                    header = f.read(4)
+                    result["readable"] = len(header) == 4
+            except:
+                result["readable"] = False
+        
+    except Exception as e:
+        result["error"] = str(e)
+    
+    return result

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=4.0.0
+llama-cpp-python
+pathlib
+typing
+json
+logging
+datetime

utils.py

@@ -0,0 +1,224 @@
+import os
+import json
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+import logging
+from datetime import datetime
+
+def get_available_models(directory: str = "./models") -> Dict[str, Any]:
+    """Get available GGUF models in the directory"""
+    try:
+        path = Path(directory)
+        if not path.exists():
+            return {"error": f"Directory {directory} does not exist"}
+        
+        models = []
+        for file in path.glob("*.gguf"):
+            try:
+                stat = file.stat()
+                models.append({
+                    "name": file.name,
+                    "path": str(file),
+                    "size_mb": round(stat.st_size / (1024 * 1024), 2),
+                    "modified": datetime.fromtimestamp(stat.st_mtime).isoformat()
+                })
+            except Exception as e:
+                logging.warning(f"Error reading {file}: {e}")
+        
+        # Sort by name
+        models.sort(key=lambda x: x["name"])
+        
+        return {
+            "directory": directory,
+            "exists": True,
+            "model_count": len(models),
+            "models": models
+        }
+        
+    except Exception as e:
+        return {"error": str(e)}
+
+def format_chat_history(history: List[List[str]], system_prompt: str = "") -> str:
+    """Format chat history for the model"""
+    formatted = ""
+    
+    if system_prompt:
+        formatted += f"<|system|>\n{system_prompt}\n<|end|>\n\n"
+    
+    for message in history:
+        role, content = message
+        if role == "user":
+            formatted += f"<|user|>\n{content}\n<|end|>\n\n"
+        elif role == "assistant":
+            formatted += f"<|assistant|>\n{content}\n<|end|>\n\n"
+    
+    formatted += "<|assistant|>\n"
+    return formatted
+
+def format_chat_history_messages(history: List[Dict[str, str]], system_prompt: str = "") -> str:
+    """Format chat history (message format) for the model"""
+    formatted = ""
+    
+    if system_prompt:
+        formatted += f"<|system|>\n{system_prompt}\n<|end|>\n\n"
+    
+    for message in history:
+        role = message.get("role", "")
+        content = message.get("content", "")
+        
+        if role == "user":
+            formatted += f"<|user|>\n{content}\n<|end|>\n\n"
+        elif role == "assistant":
+            formatted += f"<|assistant|>\n{content}\n<|end|>\n\n"
+    
+    formatted += "<|assistant|>\n"
+    return formatted
+
+def parse_model_info(metadata: Dict[str, Any]) -> Dict[str, Any]:
+    """Parse model metadata into a readable format"""
+    parsed = {
+        "architecture": "Unknown",
+        "parameters": "Unknown",
+        "context_length": "Unknown",
+        "embedding_size": "Unknown",
+        "layers": "Unknown",
+        "heads": "Unknown"
+    }
+    
+    # Try to extract common fields
+    if "general.architecture" in metadata:
+        parsed["architecture"] = metadata["general.architecture"]
+    
+    if "llama.block_count" in metadata:
+        parsed["layers"] = metadata["llama.block_count"]
+    
+    if "llama.context_length" in metadata:
+        parsed["context_length"] = metadata["llama.context_length"]
+    
+    if "llama.embedding_length" in metadata:
+        parsed["embedding_size"] = metadata["llama.embedding_length"]
+    
+    if "llama.attention.head_count" in metadata:
+        parsed["heads"] = metadata["llama.attention.head_count"]
+    
+    # Estimate parameters based on architecture
+    if parsed["architecture"] == "llama":
+        try:
+            layers = int(parsed["layers"]) if parsed["layers"] != "Unknown" else 0
+            embed_size = int(parsed["embedding_size"]) if parsed["embedding_size"] != "Unknown" else 0
+            
+            if layers > 0 and embed_size > 0:
+                # Rough estimate for LLaMA parameters
+                params = layers * (12 * embed_size * embed_size + 13 * embed_size)
+                if params > 1e9:
+                    parsed["parameters"] = f"{params / 1e9:.1f}B"
+                elif params > 1e6:
+                    parsed["parameters"] = f"{params / 1e6:.1f}M"
+                else:
+                    parsed["parameters"] = str(params)
+        except:
+            pass
+    
+    return parsed
+
+def save_chat_history(history: List[Dict[str, str]], filename: str = None) -> str:
+    """Save chat history to a JSON file"""
+    if filename is None:
+        filename = f"chat_history_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+    
+    try:
+        with open(filename, 'w', encoding='utf-8') as f:
+            json.dump(history, f, indent=2, ensure_ascii=False)
+        return filename
+    except Exception as e:
+        logging.error(f"Failed to save chat history: {e}")
+        return ""
+
+def load_chat_history(filename: str) -> List[Dict[str, str]]:
+    """Load chat history from a JSON file"""
+    try:
+        with open(filename, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    except Exception as e:
+        logging.error(f"Failed to load chat history: {e}")
+        return []
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count (rough approximation)"""
+    # Simple approximation: ~4 characters per token
+    return len(text) // 4
+
+def validate_model_file(model_path: str) -> Dict[str, Any]:
+    """Validate a model file"""
+    result = {
+        "valid": False,
+        "exists": False,
+        "readable": False,
+        "size_mb": 0,
+        "file_type": None,
+        "error": None
+    }
+    
+    try:
+        path = Path(model_path)
+        result["exists"] = path.exists()
+        
+        if not result["exists"]:
+            result["error"] = "File does not exist"
+            return result
+        
+        result["size_mb"] = round(path.stat().st_size / (1024 * 1024), 2)
+        result["file_type"] = path.suffix.lower()
+        
+        if result["file_type"] != ".gguf":
+            result["error"] = "Not a GGUF file"
+            return result
+        
+        # Try to read first few bytes
+        try:
+            with open(path, "rb") as f:
+                header = f.read(4)
+                result["readable"] = len(header) == 4
+                result["valid"] = result["readable"]
+        except Exception as e:
+            result["error"] = f"Cannot read file: {str(e)}"
+        
+    except Exception as e:
+        result["error"] = str(e)
+    
+    return result
+
+def create_default_config() -> Dict[str, Any]:
+    """Create default configuration"""
+    return {
+        "models_directory": "./models",
+        "default_context_size": 2048,
+        "default_gpu_layers": 0,
+        "default_temperature": 0.7,
+        "default_max_tokens": 512,
+        "default_top_p": 0.9,
+        "default_repeat_penalty": 1.1,
+        "system_prompt": "You are a helpful assistant.",
+        "chat_format": "chatml",
+        "auto_save_chat": True,
+        "theme": "soft"
+    }
+
+def load_config(config_path: str = "config.json") -> Dict[str, Any]:
+    """Load configuration from file"""
+    try:
+        if Path(config_path).exists():
+            with open(config_path, 'r') as f:
+                return json.load(f)
+    except Exception as e:
+        logging.warning(f"Failed to load config: {e}")
+    
+    return create_default_config()
+
+def save_config(config: Dict[str, Any], config_path: str = "config.json"):
+    """Save configuration to file"""
+    try:
+        with open(config_path, 'w') as f:
+            json.dump(config, f, indent=2)
+    except Exception as e:
+        logging.error(f"Failed to save config: {e}")