model_service.py

import os
import gc
import time
import torch
import config
from huggingface_hub import hf_hub_download
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from model_registry import PLAIN_MODELS, CPU_GGUF_MODELS, validate_model_id

# Global state for loaded model & tokenizer
active_model = None
active_tokenizer = None
active_model_id = None
model_load_time_ms = 0.0
loaded_model_hash = ""
loaded_model_revision = "main"

def get_context_limit(model_id: str) -> int:
    if model_id == "plain-llama-3-8b":
        return config.MODEL_CONTEXT_LIMIT_LLAMA_3_8B
    elif model_id == "plain-llama-3.2-3b":
        return config.MODEL_CONTEXT_LIMIT_LLAMA_3_2_3B
    elif model_id == "plain-qwen-0.5b":
        return config.MODEL_CONTEXT_LIMIT_QWEN_0_5B
    return 2048

def unload_all_models():
    global active_model, active_tokenizer, active_model_id, loaded_model_hash, loaded_model_revision
    if active_model is not None:
        # If it is a GGUF model, we may want to call its destructor if needed (llama_cpp cleans up automatically on delete)
        del active_model
        active_model = None
    if active_tokenizer is not None:
        del active_tokenizer
        active_tokenizer = None
    active_model_id = None
    loaded_model_hash = ""
    loaded_model_revision = "main"
    
    # Run garbage collection
    gc.collect()
    # Clear PyTorch CUDA Cache if GPU is active
    if torch.cuda.is_available():
        torch.cuda.empty_cache()

def load_tokenizer_only(model_id: str):
    validate_model_id(model_id)
    repo_id = PLAIN_MODELS[model_id]
    
    # Always load Hugging Face tokenizer for chat template rendering and token counting
    tokenizer = AutoTokenizer.from_pretrained(
        repo_id,
        token=config.HF_TOKEN if config.HF_TOKEN else None,
        trust_remote_code=True
    )
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    return tokenizer

def load_model_and_tokenizer(model_id: str):
    global active_model, active_tokenizer, active_model_id, model_load_time_ms, loaded_model_hash, loaded_model_revision
    
    validate_model_id(model_id)
    
    if active_model_id == model_id:
        return active_model, active_tokenizer
        
    unload_all_models()
    
    start_time = time.time()
    
    # Load Hugging Face tokenizer first (used for both GPU & CPU profiles)
    tokenizer = load_tokenizer_only(model_id)
    
    ctx_limit = get_context_limit(model_id)
    
    if config.INFERENCE_PROFILE == "gpu-transformers":
        # Load Hugging Face Model with bitsandbytes 4-bit quantization
        repo_id = PLAIN_MODELS[model_id]
        
        bnb_config = BitsAndBytesConfig(
            load_in_4bit=True,
            bnb_4bit_quant_type="nf4",
            bnb_4bit_use_double_quant=True,
            bnb_4bit_compute_dtype=torch.float16
        )
        
        model = AutoModelForCausalLM.from_pretrained(
            repo_id,
            quantization_config=bnb_config,
            device_map="auto",
            token=config.HF_TOKEN if config.HF_TOKEN else None,
            trust_remote_code=True
        )
        loaded_model_revision = "main"
        loaded_model_hash = "gpu-transformers-hash-4bit"
        
    elif config.INFERENCE_PROFILE == "cpu-gguf":
        # Load GGUF model via llama-cpp-python
        import llama_cpp
        
        gguf_cfg = CPU_GGUF_MODELS[model_id]
        repo_id = gguf_cfg["repo_id"]
        filename = gguf_cfg["filename"]
        
        # Download file using huggingface_hub
        model_path = hf_hub_download(
            repo_id=repo_id,
            filename=filename,
            token=config.HF_TOKEN if config.HF_TOKEN else None
        )
        
        # Initialize llama.cpp model
        model = llama_cpp.Llama(
            model_path=model_path,
            n_ctx=ctx_limit,
            n_threads=config.MODEL_THREADS,
            n_batch=config.MODEL_BATCH_SIZE,
            verbose=False
        )
        loaded_model_revision = "main"
        # Extract file size/hash metadata
        import hashlib
        h = hashlib.sha256()
        with open(model_path, "rb") as f:
            # Read first 1MB for hashing to avoid blocking
            h.update(f.read(1024 * 1024))
        loaded_model_hash = h.hexdigest()
        
    else:
        raise ValueError(f"Unknown INFERENCE_PROFILE: {config.INFERENCE_PROFILE}")
        
    active_model = model
    active_tokenizer = tokenizer
    active_model_id = model_id
    model_load_time_ms = (time.time() - start_time) * 1000.0
    
    return active_model, active_tokenizer