config.py

import os

# Hugging Face Settings
HF_TOKEN = os.getenv("HF_TOKEN", "")

# Inference Settings
# "gpu-transformers" or "cpu-gguf"
INFERENCE_PROFILE = os.getenv("INFERENCE_PROFILE", "cpu-gguf")
DEFAULT_MODEL_ID = os.getenv("DEFAULT_MODEL_ID", "plain-qwen-0.5b")

# Context window limits (parsed from env, or defaulting to official specs)
# Llama 3 8B Instruct has an 8k context window officially
MODEL_CONTEXT_LIMIT_LLAMA_3_8B = int(os.getenv("MODEL_CONTEXT_LIMIT_LLAMA_3_8B", "8192"))
# Llama 3.2 3B Instruct has a 128k context window officially
MODEL_CONTEXT_LIMIT_LLAMA_3_2_3B = int(os.getenv("MODEL_CONTEXT_LIMIT_LLAMA_3_2_3B", "131072"))
# Qwen 1.5 0.5B Chat has a 32k context window officially
MODEL_CONTEXT_LIMIT_QWEN_0_5B = int(os.getenv("MODEL_CONTEXT_LIMIT_QWEN_0_5B", "32768"))

# CPU execution settings
MODEL_THREADS = int(os.getenv("MODEL_THREADS", "2"))
MODEL_BATCH_SIZE = int(os.getenv("MODEL_BATCH_SIZE", "256"))
MAX_OUTPUT_TOKENS = int(os.getenv("MAX_OUTPUT_TOKENS", "1024"))

# Security and API Configuration
BENCHMARK_API_KEY = os.getenv("BENCHMARK_API_KEY", "")
ALLOWED_ORIGINS = os.getenv("ALLOWED_ORIGINS", "*")

# Enforce clean writeable directories for Hugging Face caching inside container sandbox
HOME = os.getenv("HOME", "/tmp")
HF_HOME = os.getenv("HF_HOME", "/tmp/huggingface_cache")
XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", "/tmp/cache")
TORCH_EXTENSIONS_DIR = os.getenv("TORCH_EXTENSIONS_DIR", "/tmp/torch_extensions")
TRITON_CACHE_DIR = os.getenv("TRITON_CACHE_DIR", "/tmp/triton_cache")

# Setup environment variables dynamically
os.environ["HF_HOME"] = HF_HOME
os.environ["XDG_CACHE_HOME"] = XDG_CACHE_HOME
os.environ["TORCH_EXTENSIONS_DIR"] = TORCH_EXTENSIONS_DIR
os.environ["TRITON_CACHE_DIR"] = TRITON_CACHE_DIR

# Ensure cache directories exist
for path in [HF_HOME, XDG_CACHE_HOME, TORCH_EXTENSIONS_DIR, TRITON_CACHE_DIR]:
    os.makedirs(path, exist_ok=True)