File size: 7,370 Bytes
b5d7fb6 e6f5bd6 12ed0e8 e6f5bd6 b5d7fb6 12ed0e8 b5d7fb6 12ed0e8 6409244 12ed0e8 797021e f80a9a0 8371a0a 77a0152 fbff1f2 93cec89 65cc3d9 7c804b1 b2b2074 ea4cd5d b2b2074 eb2c5d5 b5d7fb6 fe3feaa b5d7fb6 57f3b8a b5d7fb6 57f3b8a 81c8014 e6f5bd6 68d8ff7 6409244 8371a0a 797021e 12ed0e8 797021e 8371a0a f80a9a0 8371a0a 77a0152 fbff1f2 93cec89 65cc3d9 7c804b1 b2b2074 ea4cd5d 29cc542 b2b2074 eb2c5d5 65cc3d9 b5d7fb6 68d8ff7 b5d7fb6 bf15c31 68d8ff7 3d26a51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | FROM ghcr.io/ggml-org/llama.cpp:server
RUN apt update && apt install wget -y && rm -rf /var/lib/apt/lists/*
# Create a dedicated directory to organize the models
RUN mkdir -p /models
# --- MODEL 1: Gemma 4 Multimodal + Speculative Setup ---
RUN wget "https://huggingface.co/unsloth/gemma-4-E2B-it-qat-GGUF/resolve/main/gemma-4-E2B-it-qat-UD-Q4_K_XL.gguf" -O /models/gemma-4-E2B-it-qat-UD-Q4_K_XL.gguf
RUN wget "https://huggingface.co/unsloth/gemma-4-E2B-it-qat-GGUF/resolve/main/mtp-gemma-4-E2B-it.gguf" -O /models/mtp-gemma-4-E2B-it.gguf
RUN wget "https://huggingface.co/unsloth/gemma-4-E2B-it-qat-GGUF/resolve/main/mmproj-F16.gguf" -O /models/gemma-4-E2B-it-mmproj.gguf
# --- MODEL 2: Gemma 3 270M IT Q8_0 ---
RUN wget "https://huggingface.co/unsloth/gemma-3-270m-it-GGUF/resolve/main/gemma-3-270m-it-Q8_0.gguf" -O /models/gemma-3-270m-it-Q8_0.gguf
# --- MODEL 3: Gemma 3 1B Model ---
RUN wget "https://huggingface.co/MaziyarPanahi/gemma-3-1b-it-GGUF/resolve/main/gemma-3-1b-it.Q3_K_M.gguf" -O /models/gemma-3-1b-it.Q3_K_M.gguf
# --- MODEL 4: Ornith 1.0 9B Model ---
RUN wget "https://huggingface.co/deepreinforce-ai/Ornith-1.0-9B-GGUF/resolve/main/ornith-1.0-9b-Q4_K_M.gguf" -O /models/ornith-1.0-9b-Q4_K_M.gguf
# --- MODEL 5: Qwen 3.5 0.8B Multimodal (Vision) ---
RUN wget "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q4_0.gguf" -O /models/Qwen3.5-0.8B-Q4_0.gguf
RUN wget "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/mmproj-BF16.gguf" -O /models/mmproj-BF16.gguf
# --- MODEL 6: GPT OSS 20B ---
RUN wget "https://huggingface.co/unsloth/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-Q4_0.gguf" -O /models/gpt-oss-20b-Q4_0.gguf
# --- MODEL 7: Gemma 4 12B IT ---
RUN wget "https://huggingface.co/unsloth/gemma-4-12b-it-GGUF/resolve/main/gemma-4-12b-it-Q4_K_M.gguf" -O /models/gemma-4-12b-it-Q4_K_M.gguf
# --- MODEL 8: Gemma 4 E4B IT ---
RUN wget "https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/gemma-4-E4B-it-Q4_K_M.gguf" -O /models/gemma-4-E4B-it-Q4_K_M.gguf
# --- MODEL 9: MiniCPM5 1B ---
RUN wget "https://huggingface.co/openbmb/MiniCPM5-1B-GGUF/resolve/main/MiniCPM5-1B-Q4_K_M.gguf" -O /models/MiniCPM5-1B-Q4_K_M.gguf
# --- MODEL 10: Nanbeige4.1 3B ---
RUN wget "https://huggingface.co/Mungert/Nanbeige4.1-3B-GGUF/resolve/main/Nanbeige4.1-3B-q4_k_m.gguf?download=true" -O /models/Nanbeige4.1-3B-q4_k_m.gguf
# --- MODEL 11: TwIL-LM3 Q5_K_S ---
RUN wget "https://huggingface.co/holooo/TwIL-LM3-Q5_K_S-GGUF/resolve/main/twil-lm3-q5_k_s.gguf?download=true" -O /models/twil-lm3-q5_k_s.gguf
# --- MODEL 12: LiquidAI LFM2.5 Audio 1.5B ---
RUN wget "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct-GGUF/resolve/main/LFM2.5-1.2B-Instruct-Q4_0.gguf?download=true" -O /models/LFM2.5-1.2B-Instruct-Q4_0.gguf
# --- MODEL 13: LiquidAI LFM2.5 8B A1B ---
RUN wget "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B-GGUF/resolve/main/LFM2.5-8B-A1B-Q4_0.gguf?download=true" -O /models/LFM2.5-8B-A1B-Q4_0.gguf
# --- MODEL 14: LiquidAI LFM2.5 VL 450M ---
RUN wget "https://huggingface.co/LiquidAI/LFM2.5-VL-450M-GGUF/resolve/main/LFM2.5-VL-450M-Q4_0.gguf?download=true" -O /models/LFM2.5-VL-450M-Q4_0.gguf
# --- CREATE ROUTER CONFIGURATION ---
RUN echo "[gemma-4-E2B-it-vision]" > /models.ini && \
echo "model = /models/gemma-4-E2B-it-qat-UD-Q4_K_XL.gguf" >> /models.ini && \
echo "spec-draft-model = /models/mtp-gemma-4-E2B-it.gguf" >> /models.ini && \
echo "mmproj = /models/gemma-4-E2B-it-mmproj.gguf" >> /models.ini && \
echo "ctx-size = 131072" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "ubatch-size = 128" >> /models.ini && \
echo "batch-size = 512" >> /models.ini && \
echo "spec-type = draft-mtp" >> /models.ini && \
echo "spec-draft-n-max = 3" >> /models.ini && \
echo "" >> /models.ini && \
echo "[gemma-3-1b]" >> /models.ini && \
echo "model = /models/gemma-3-1b-it.Q3_K_M.gguf" >> /models.ini && \
echo "ctx-size = 131072" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[gemma-3-270m]" >> /models.ini && \
echo "model = /models/gemma-3-270m-it-Q8_0.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[ornith-9b]" >> /models.ini && \
echo "model = /models/ornith-1.0-9b-Q4_K_M.gguf" >> /models.ini && \
echo "ctx-size = 8192" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[qwen-3.5-0.8b-vision]" >> /models.ini && \
echo "model = /models/Qwen3.5-0.8B-Q4_0.gguf" >> /models.ini && \
echo "mmproj = /models/mmproj-BF16.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[gpt-oss-20b]" >> /models.ini && \
echo "model = /models/gpt-oss-20b-Q4_0.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[gemma-4-12b]" >> /models.ini && \
echo "model = /models/gemma-4-12b-it-Q4_K_M.gguf" >> /models.ini && \
echo "ctx-size = 131072" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[gemma-4-E4B-it]" >> /models.ini && \
echo "model = /models/gemma-4-E4B-it-Q4_K_M.gguf" >> /models.ini && \
echo "ctx-size = 131072" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[minicpm5-1b]" >> /models.ini && \
echo "model = /models/MiniCPM5-1B-Q4_K_M.gguf" >> /models.ini && \
echo "ctx-size = 131072" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[nanbeige4.1-3b]" >> /models.ini && \
echo "model = /models/Nanbeige4.1-3B-q4_k_m.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[twil-lm3]" >> /models.ini && \
echo "model = /models/twil-lm3-q5_k_s.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[lfm2.5-1.2b instruct]" >> /models.ini && \
echo "model = /models/LFM2.5-1.2B-Instruct-Q4_0.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[lfm2.5-8b-a1b]" >> /models.ini && \
echo "model = /models/LFM2.5-8B-A1B-Q4_0.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini && \
echo "" >> /models.ini && \
echo "[lfm2.5-vl-450m]" >> /models.ini && \
echo "model = /models/LFM2.5-VL-450M-Q4_0.gguf" >> /models.ini && \
echo "ctx-size = 32768" >> /models.ini && \
echo "flash-attn = on" >> /models.ini
# --- START SERVER IN ROUTER MODE WITH CORS, TOOLS & MCP ENABLED ---
CMD [ \
"--models-preset", "/models.ini", \
"--port", "7860", \
"--host", "0.0.0.0", \
"-t", "2", \
"-tb", "2", \
"--tools", "all", \
"--jinja", \
"--webui-mcp-proxy", \
"--cors-origins", "https://we2app-v4.hf.space" \
] |