Update Dockerfile
Browse files- Dockerfile +38 -3
Dockerfile
CHANGED
|
@@ -9,9 +9,24 @@ FROM ubuntu:22.04
|
|
| 9 |
ENV DEBIAN_FRONTEND=noninteractive
|
| 10 |
ENV PYTHONUNBUFFERED=1
|
| 11 |
ENV HF_REPO_ID=NOT-OMEGA/NanoMind
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
ENV OMP_NUM_THREADS=1
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
RUN apt-get update && apt-get install -y \
|
| 16 |
python3.11 \
|
| 17 |
python3-pip \
|
|
@@ -34,7 +49,17 @@ COPY index.html .
|
|
| 34 |
COPY model.bin .
|
| 35 |
COPY tokenizer.bin .
|
| 36 |
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
-o inference inference.cpp -lm && \
|
| 39 |
chmod +x inference && \
|
| 40 |
echo "β
inference binary compiled"
|
|
@@ -47,4 +72,14 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
|
|
| 47 |
CMD curl -f http://localhost:7860/health || exit 1
|
| 48 |
|
| 49 |
EXPOSE 7860
|
| 50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
ENV DEBIAN_FRONTEND=noninteractive
|
| 10 |
ENV PYTHONUNBUFFERED=1
|
| 11 |
ENV HF_REPO_ID=NOT-OMEGA/NanoMind
|
| 12 |
+
|
| 13 |
+
# ββ Concurrency tuning ββββββββββββββββββββββββββββββββββββββββ
|
| 14 |
+
# HF free tier = 2 vCPU. Rule: N_ENGINES Γ OMP_NUM_THREADS β€ vCPU count.
|
| 15 |
+
# 4 engines Γ 1 thread = 4 logical workers on 2 cores (light context switching,
|
| 16 |
+
# better queue saturation than 2 engines Γ 1 thread).
|
| 17 |
+
# Do NOT set OMP_NUM_THREADS > 1 here β engines will fight for cores
|
| 18 |
+
# and tok/s will DROP. Keep it at 1.
|
| 19 |
+
ENV N_ENGINES=4
|
| 20 |
ENV OMP_NUM_THREADS=1
|
| 21 |
|
| 22 |
+
# ββ gomp thread pinning β prevents OS from migrating OMP threads ββ
|
| 23 |
+
# Each engine stays on its assigned core β fewer cache misses β ~5-10% faster matmul
|
| 24 |
+
ENV GOMP_CPU_AFFINITY="0-3"
|
| 25 |
+
ENV OMP_PROC_BIND=close
|
| 26 |
+
|
| 27 |
+
# ββ Python GIL / asyncio tuning βββββββββββββββββββββββββββββββ
|
| 28 |
+
ENV PYTHONOPTIMIZE=1
|
| 29 |
+
|
| 30 |
RUN apt-get update && apt-get install -y \
|
| 31 |
python3.11 \
|
| 32 |
python3-pip \
|
|
|
|
| 49 |
COPY model.bin .
|
| 50 |
COPY tokenizer.bin .
|
| 51 |
|
| 52 |
+
# ββ Compiler flags explanation βββββββββββββββββββββββββββββββββ
|
| 53 |
+
# -O3 : max optimization
|
| 54 |
+
# -march=native : AVX2 + FMA auto-detected at build time
|
| 55 |
+
# -fopenmp : OpenMP parallelism across attention heads
|
| 56 |
+
# -ffast-math : allows fused multiply-add reordering (safe for inference)
|
| 57 |
+
# -funroll-loops : unroll small inner loops (matmul hot path)
|
| 58 |
+
# -fno-stack-protector : remove stack canary overhead in tight loops
|
| 59 |
+
# -std=c++17 : required for structured bindings + std::optional
|
| 60 |
+
RUN g++ -O3 -march=native -fopenmp -ffast-math \
|
| 61 |
+
-funroll-loops -fno-stack-protector \
|
| 62 |
+
-std=c++17 \
|
| 63 |
-o inference inference.cpp -lm && \
|
| 64 |
chmod +x inference && \
|
| 65 |
echo "β
inference binary compiled"
|
|
|
|
| 72 |
CMD curl -f http://localhost:7860/health || exit 1
|
| 73 |
|
| 74 |
EXPOSE 7860
|
| 75 |
+
|
| 76 |
+
# ββ Use uvicorn directly β skips the main.py import overhead ββ
|
| 77 |
+
# workers=1 is correct: parallelism comes from N_ENGINES async engines,
|
| 78 |
+
# NOT from multiple uvicorn workers (those would each spawn their own
|
| 79 |
+
# engine pool and OOM the container).
|
| 80 |
+
CMD ["python", "-m", "uvicorn", "main:app", \
|
| 81 |
+
"--host", "0.0.0.0", \
|
| 82 |
+
"--port", "7860", \
|
| 83 |
+
"--workers", "1", \
|
| 84 |
+
"--loop", "uvloop", \
|
| 85 |
+
"--http", "httptools"]
|