import os # Hugging Face Settings HF_TOKEN = os.getenv("HF_TOKEN", "") # Inference Settings # "gpu-transformers" or "cpu-gguf" INFERENCE_PROFILE = os.getenv("INFERENCE_PROFILE", "cpu-gguf") DEFAULT_MODEL_ID = os.getenv("DEFAULT_MODEL_ID", "plain-qwen-0.5b") # Context window limits (parsed from env, or defaulting to official specs) # Llama 3 8B Instruct has an 8k context window officially MODEL_CONTEXT_LIMIT_LLAMA_3_8B = int(os.getenv("MODEL_CONTEXT_LIMIT_LLAMA_3_8B", "8192")) # Llama 3.2 3B Instruct has a 128k context window officially MODEL_CONTEXT_LIMIT_LLAMA_3_2_3B = int(os.getenv("MODEL_CONTEXT_LIMIT_LLAMA_3_2_3B", "131072")) # Qwen 1.5 0.5B Chat has a 32k context window officially MODEL_CONTEXT_LIMIT_QWEN_0_5B = int(os.getenv("MODEL_CONTEXT_LIMIT_QWEN_0_5B", "32768")) # CPU execution settings MODEL_THREADS = int(os.getenv("MODEL_THREADS", "2")) MODEL_BATCH_SIZE = int(os.getenv("MODEL_BATCH_SIZE", "256")) MAX_OUTPUT_TOKENS = int(os.getenv("MAX_OUTPUT_TOKENS", "1024")) # Security and API Configuration BENCHMARK_API_KEY = os.getenv("BENCHMARK_API_KEY", "") ALLOWED_ORIGINS = os.getenv("ALLOWED_ORIGINS", "*") # Enforce clean writeable directories for Hugging Face caching inside container sandbox HOME = os.getenv("HOME", "/tmp") HF_HOME = os.getenv("HF_HOME", "/tmp/huggingface_cache") XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", "/tmp/cache") TORCH_EXTENSIONS_DIR = os.getenv("TORCH_EXTENSIONS_DIR", "/tmp/torch_extensions") TRITON_CACHE_DIR = os.getenv("TRITON_CACHE_DIR", "/tmp/triton_cache") # Setup environment variables dynamically os.environ["HF_HOME"] = HF_HOME os.environ["XDG_CACHE_HOME"] = XDG_CACHE_HOME os.environ["TORCH_EXTENSIONS_DIR"] = TORCH_EXTENSIONS_DIR os.environ["TRITON_CACHE_DIR"] = TRITON_CACHE_DIR # Ensure cache directories exist for path in [HF_HOME, XDG_CACHE_HOME, TORCH_EXTENSIONS_DIR, TRITON_CACHE_DIR]: os.makedirs(path, exist_ok=True)