sterepando commited on
Commit
48c286a
·
verified ·
1 Parent(s): acb2be3

Update Dockerfile

Browse files
Files changed (1) hide show
  1. Dockerfile +67 -113
Dockerfile CHANGED
@@ -1,7 +1,5 @@
1
  FROM python:3.11-bookworm
2
 
3
- # 1. Установка системных зависимостей
4
- # Добавляем ninja-build и другие инструменты
5
  RUN apt-get update && apt-get install -y \
6
  wget unzip git build-essential pkg-config \
7
  libffi-dev libssl-dev zlib1g-dev ncurses-dev \
@@ -10,40 +8,32 @@ RUN apt-get update && apt-get install -y \
10
  cmake llvm clang \
11
  perl \
12
  patchelf zip \
13
- libsndfile1-dev \
14
- ninja-build \
15
  && rm -rf /var/lib/apt/lists/*
16
 
17
- # Создаем пользователя
18
- RUN useradd -m -u 1000 user
19
- ENV HOME=/home/user
20
- ENV WORK_DIR=/home/user/build
21
- ENV OUTPUT_DIR=/home/user/app/wheels
22
-
23
- RUN mkdir -p ${WORK_DIR} ${OUTPUT_DIR} && chown -R user:user /home/user
24
-
25
- USER user
26
- WORKDIR ${WORK_DIR}
27
-
28
- # 2. Переменные окружения
29
  ENV NDK_VERSION=r26b
30
  ENV ANDROID_API=24
31
  ENV ANDROID_ARCH=aarch64
32
- ENV ANDROID_HOME=${HOME}/android-sdk
33
- ENV NDK_HOME=${HOME}/android-ndk
34
- # Дублируем для совместимости разных систем сборки
35
- ENV ANDROID_NDK_ROOT=${HOME}/android-ndk
36
  ENV PYTHON_VERSION=3.11.9
37
  ENV OPENSSL_VERSION=3.3.0
38
- ENV RUSTUP_HOME=${HOME}/.rustup
39
- ENV CARGO_HOME=${HOME}/.cargo
40
- ENV PATH=$PATH:${HOME}/.cargo/bin:${HOME}/.local/bin
41
-
42
- # 3. Установка Rust (для tokenizers)
43
  RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
44
  RUN rustup target add aarch64-linux-android
45
 
46
- # 4. Скачивание и настройка NDK
 
 
 
 
 
 
 
47
  RUN wget -q https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux.zip && \
48
  unzip -q android-ndk-${NDK_VERSION}-linux.zip && \
49
  mv android-ndk-${NDK_VERSION} ${NDK_HOME} && \
@@ -65,32 +55,31 @@ ENV CFLAGS="-target ${TARGET} -fPIC -D__ANDROID__"
65
  ENV CXXFLAGS="-target ${TARGET} -fPIC -D__ANDROID__"
66
  ENV LDFLAGS="-target ${TARGET} -fPIC"
67
 
68
- # 5. Сборка OpenSSL (Static)
69
  WORKDIR ${WORK_DIR}
70
  RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz && \
71
  tar -xf openssl-${OPENSSL_VERSION}.tar.gz
72
 
73
  WORKDIR ${WORK_DIR}/openssl-${OPENSSL_VERSION}
74
- # Добавляем путь к NDK в начало PATH для Configure
75
- ENV PATH=${TOOLCHAIN}/bin:$PATH
76
  RUN ./Configure android-arm64 \
77
  -D__ANDROID_API__=${ANDROID_API} \
78
  --prefix=${WORK_DIR}/openssl-install \
79
  --openssldir=${WORK_DIR}/openssl-install \
80
- no-shared no-tests && \
81
- make -j$(nproc) && make install_sw
 
82
 
83
  ENV OPENSSL_DIR=${WORK_DIR}/openssl-install
84
  ENV OPENSSL_LIB_DIR=${OPENSSL_DIR}/lib64
85
  ENV OPENSSL_INCLUDE_DIR=${OPENSSL_DIR}/include
86
  ENV OPENSSL_STATIC=1
87
 
88
- # 6. Сборка Python (Host & Target stub)
89
  WORKDIR ${WORK_DIR}
90
  RUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
91
  tar -xf Python-${PYTHON_VERSION}.tgz
92
 
93
  WORKDIR ${WORK_DIR}/Python-${PYTHON_VERSION}
 
94
  RUN echo "ac_cv_file__dev_ptmx=yes" > config.site && \
95
  echo "ac_cv_file__dev_ptc=no" >> config.site && \
96
  echo "ac_cv_func_wcsftime=no" >> config.site && \
@@ -108,109 +97,75 @@ RUN CONFIG_SITE=config.site ./configure \
108
  --disable-ipv6 \
109
  --enable-shared \
110
  --without-ensurepip \
111
- --with-build-python=python3 \
112
  --disable-test-modules \
113
  --without-doc-strings \
114
  --with-system-ffi=no \
115
  --without-decimal \
116
- --without-ctypes && \
117
- make -j$(nproc) libpython3.11.so && \
118
- make install -i
 
119
 
120
- # Подготавливаем файл sysconfig
121
  RUN mkdir -p ${WORK_DIR}/python-install/lib/python3.11
 
122
  RUN find ${WORK_DIR}/Python-${PYTHON_VERSION} -name "_sysconfigdata*.py" -not -name "*x86_64*" \
123
  -exec cp {} ${WORK_DIR}/python-install/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py \;
124
 
125
- # !!! ИСПРАВЛЕНИЕ !!!
126
- # Копируем конфиг в правильное место (/usr/local/lib, а не /usr/lib)
127
- USER root
128
  RUN cp ${WORK_DIR}/python-install/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py \
129
  /usr/local/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py
130
- USER user
131
 
132
- # 7. УСТАНОВКА ИНСТРУМЕНТОВ СБОРКИ
133
- RUN pip install --upgrade pip && \
134
- pip install --user --no-cache-dir \
135
- Cython wheel setuptools \
136
- "meson-python>=0.13.1" "ninja>=1.10" "patchelf" \
137
- "maturin>=1.0,<2.0" \
138
- "scikit-build-core" "cmake" \
139
- "flit-core" "hatchling" \
140
- "cffi" "pyyaml"
141
-
142
- # 8. Настройка переменных окружения для кросс-компиляции
143
- ENV PKG_CONFIG_PATH=""
144
- ENV PYTHON_ANDROID_HOME=${WORK_DIR}/python-install
145
- ENV _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata__linux_aarch64-linux-android
146
- ENV _PYTHON_HOST_PLATFORM=linux-aarch64
147
- # Добавляем путь к sysconfig в PYTHONPATH для гарантии
148
- ENV PYTHONPATH=/usr/local/lib/python3.11
149
-
150
- # Флаги для Rust/Maturin
151
- ENV PYO3_CROSS_LIB_DIR=${PYTHON_ANDROID_HOME}/lib
152
- ENV PYO3_CROSS_INCLUDE_DIR=${PYTHON_ANDROID_HOME}/include/python3.11
153
- ENV PYO3_CROSS_PYTHON_VERSION=3.11
154
- ENV CARGO_BUILD_TARGET=aarch64-linux-android
155
- ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="${CC}"
156
- ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_RUSTFLAGS="-C link-arg=-L${PYTHON_ANDROID_HOME}/lib -C link-arg=-lpython3.11 -L${OPENSSL_LIB_DIR}"
157
 
158
- # Флаги для C/C++
159
- ENV CFLAGS="${CFLAGS} -I${PYTHON_ANDROID_HOME}/include/python3.11 -I${OPENSSL_INCLUDE_DIR}"
160
- ENV LDFLAGS="${LDFLAGS} -L${PYTHON_ANDROID_HOME}/lib -L${OPENSSL_LIB_DIR}"
161
- ENV LDSHARED="${CC} -shared"
162
 
163
- WORKDIR ${OUTPUT_DIR}
 
164
 
165
- # 9. Скрипт сборки
166
  RUN echo "#!/bin/bash" > build.sh && \
167
  echo "set -e" >> build.sh && \
168
- # Специфичные настройки для Llama.cpp
169
- echo "export CMAKE_ARGS=\"-DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-${ANDROID_API} -DCMAKE_TOOLCHAIN_FILE=${NDK_HOME}/build/cmake/android.toolchain.cmake -DGGML_OPENCL=OFF -DGGML_VULKAN=OFF\"" >> build.sh && \
170
- echo "" >> build.sh && \
171
- echo "build_pkg() {" >> build.sh && \
172
- echo " PKG=\$1" >> build.sh && \
173
- echo " echo \"--------------------------------------------------\"" >> build.sh && \
174
- echo " echo \"BUILDING: \$PKG\"" >> build.sh && \
175
- echo " echo \"--------------------------------------------------\"" >> build.sh && \
176
- # Используем --no-deps, чтобы pip не пытался установить зависимости в хост-систему
177
- echo " python3 -m pip wheel \"\$PKG\" --wheel-dir=${OUTPUT_DIR} --no-binary=:all: --no-build-isolation --no-deps -v || echo \"FAILED: \$PKG\"" >> build.sh && \
178
- echo "}" >> build.sh && \
 
 
 
 
179
  echo "" >> build.sh && \
180
- # Порядок сборки
181
- echo "build_pkg 'numpy<2.0'" >> build.sh && \
182
- echo "build_pkg 'flask'" >> build.sh && \
183
- echo "build_pkg 'flask_compress'" >> build.sh && \
184
- echo "build_pkg 'tokenizers'" >> build.sh && \
185
- echo "build_pkg 'transformers'" >> build.sh && \
186
- echo "build_pkg 'llama-cpp-python'" >> build.sh && \
187
- # soundfile часто проблемный из-за libsndfile, пробуем
188
- echo "build_pkg 'soundfile'" >> build.sh && \
189
- echo "echo 'Build complete.'" >> build.sh && \
190
  chmod +x build.sh
191
 
192
- # Запуск сборки
193
  RUN ./build.sh
194
 
195
- # 10. Патчинг (исправление имени библиотеки Python)
196
- RUN echo "Patching wheels..." && \
197
  for whl in *.whl; do \
198
- if [ -f "$whl" ]; then \
199
- echo "Processing $whl..."; \
200
- mkdir -p tmp_wheel; \
201
- unzip -q "$whl" -d tmp_wheel; \
202
- find tmp_wheel -name "*.so" | while read so_file; do \
203
- # Если библиотека ссылается на libpython3.11.so.1.0, меняем на libpython3.11.so
204
- if patchelf --print-needed "$so_file" | grep -q "libpython3.11.so.1.0"; then \
205
- echo "Patching $so_file"; \
206
- patchelf --replace-needed "libpython3.11.so.1.0" "libpython3.11.so" "$so_file"; \
207
- fi; \
208
- done; \
209
- cd tmp_wheel; zip -r -q "../$whl" .; cd ..; rm -rf tmp_wheel; \
210
- fi; \
 
 
 
211
  done
212
 
213
- # 11. Переименование
214
  RUN for f in *.whl; do \
215
  if [ -f "$f" ]; then \
216
  new_name=$(echo "$f" | sed 's/linux_aarch64/android_24_arm64_v8a/g'); \
@@ -220,9 +175,8 @@ RUN for f in *.whl; do \
220
  fi \
221
  done
222
 
223
- # 12. Индекс
224
- RUN echo "<html><head><title>MandreLib Wheels</title></head><body>" > index.html && \
225
- echo "<h1>Android Wheels (API 24, ARM64)</h1><ul>" >> index.html && \
226
  find . -name "*.whl" -printf "<li><a href=\"%P\">%P</a></li>\n" | sort >> index.html && \
227
  echo "</ul></body></html>" >> index.html
228
 
 
1
  FROM python:3.11-bookworm
2
 
 
 
3
  RUN apt-get update && apt-get install -y \
4
  wget unzip git build-essential pkg-config \
5
  libffi-dev libssl-dev zlib1g-dev ncurses-dev \
 
8
  cmake llvm clang \
9
  perl \
10
  patchelf zip \
 
 
11
  && rm -rf /var/lib/apt/lists/*
12
 
13
+ ENV WORK_DIR=/build
14
+ ENV OUTPUT_DIR=/app/wheels
 
 
 
 
 
 
 
 
 
 
15
  ENV NDK_VERSION=r26b
16
  ENV ANDROID_API=24
17
  ENV ANDROID_ARCH=aarch64
18
+ ENV ANDROID_HOME=/opt/android-sdk
19
+ ENV NDK_HOME=/opt/android-ndk
20
+ ENV ANDROID_NDK_ROOT=/opt/android-ndk
 
21
  ENV PYTHON_VERSION=3.11.9
22
  ENV OPENSSL_VERSION=3.3.0
23
+ ENV RUSTUP_HOME=/opt/rust
24
+ ENV CARGO_HOME=/opt/rust
25
+ ENV PATH=$PATH:/opt/rust/bin
 
 
26
  RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
27
  RUN rustup target add aarch64-linux-android
28
 
29
+ RUN pip install --upgrade pip && \
30
+ pip install --no-cache-dir \
31
+ Cython wheel setuptools poetry-core hatchling \
32
+ hatch-fancy-pypi-readme hatch-vcs flit-core \
33
+ expandvars maturin typing-extensions
34
+
35
+ WORKDIR ${WORK_DIR}
36
+
37
  RUN wget -q https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux.zip && \
38
  unzip -q android-ndk-${NDK_VERSION}-linux.zip && \
39
  mv android-ndk-${NDK_VERSION} ${NDK_HOME} && \
 
55
  ENV CXXFLAGS="-target ${TARGET} -fPIC -D__ANDROID__"
56
  ENV LDFLAGS="-target ${TARGET} -fPIC"
57
 
 
58
  WORKDIR ${WORK_DIR}
59
  RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz && \
60
  tar -xf openssl-${OPENSSL_VERSION}.tar.gz
61
 
62
  WORKDIR ${WORK_DIR}/openssl-${OPENSSL_VERSION}
63
+
 
64
  RUN ./Configure android-arm64 \
65
  -D__ANDROID_API__=${ANDROID_API} \
66
  --prefix=${WORK_DIR}/openssl-install \
67
  --openssldir=${WORK_DIR}/openssl-install \
68
+ no-shared no-tests
69
+
70
+ RUN make -j$(nproc) && make install_sw
71
 
72
  ENV OPENSSL_DIR=${WORK_DIR}/openssl-install
73
  ENV OPENSSL_LIB_DIR=${OPENSSL_DIR}/lib64
74
  ENV OPENSSL_INCLUDE_DIR=${OPENSSL_DIR}/include
75
  ENV OPENSSL_STATIC=1
76
 
 
77
  WORKDIR ${WORK_DIR}
78
  RUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
79
  tar -xf Python-${PYTHON_VERSION}.tgz
80
 
81
  WORKDIR ${WORK_DIR}/Python-${PYTHON_VERSION}
82
+
83
  RUN echo "ac_cv_file__dev_ptmx=yes" > config.site && \
84
  echo "ac_cv_file__dev_ptc=no" >> config.site && \
85
  echo "ac_cv_func_wcsftime=no" >> config.site && \
 
97
  --disable-ipv6 \
98
  --enable-shared \
99
  --without-ensurepip \
100
+ --with-build-python=/usr/local/bin/python3 \
101
  --disable-test-modules \
102
  --without-doc-strings \
103
  --with-system-ffi=no \
104
  --without-decimal \
105
+ --without-ctypes
106
+
107
+ RUN make -j$(nproc) libpython3.11.so || true
108
+ RUN make install -i
109
 
 
110
  RUN mkdir -p ${WORK_DIR}/python-install/lib/python3.11
111
+
112
  RUN find ${WORK_DIR}/Python-${PYTHON_VERSION} -name "_sysconfigdata*.py" -not -name "*x86_64*" \
113
  -exec cp {} ${WORK_DIR}/python-install/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py \;
114
 
 
 
 
115
  RUN cp ${WORK_DIR}/python-install/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py \
116
  /usr/local/lib/python3.11/_sysconfigdata__linux_aarch64-linux-android.py
 
117
 
118
+ WORKDIR ${OUTPUT_DIR}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
 
120
+ ENV TARGET_PACKAGE="heroku-tl-new==1.7.2"
 
 
 
121
 
122
+ ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="${CC}"
123
+ ENV CARGO_TARGET_AARCH64_LINUX_ANDROID_RUSTFLAGS="-C link-arg=-L${WORK_DIR}/python-install/lib -C link-arg=-lpython3.11 -L ${OPENSSL_LIB_DIR}"
124
 
 
125
  RUN echo "#!/bin/bash" > build.sh && \
126
  echo "set -e" >> build.sh && \
127
+ echo "export PYTHON_ANDROID_HOME=${WORK_DIR}/python-install" >> build.sh && \
128
+ echo "export CFLAGS=\"${CFLAGS} -I\${PYTHON_ANDROID_HOME}/include/python3.11 -I${OPENSSL_INCLUDE_DIR}\"" >> build.sh && \
129
+ echo "export LDFLAGS=\"${LDFLAGS} -L\${PYTHON_ANDROID_HOME}/lib -L${OPENSSL_LIB_DIR}\"" >> build.sh && \
130
+ echo "export LDSHARED=\"${CC} -shared\"" >> build.sh && \
131
+ echo "export _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata__linux_aarch64-linux-android" >> build.sh && \
132
+ echo "export _PYTHON_HOST_PLATFORM=linux-aarch64" >> build.sh && \
133
+ echo "export PYO3_CROSS_LIB_DIR=${WORK_DIR}/python-install/lib" >> build.sh && \
134
+ echo "export PYO3_CROSS_INCLUDE_DIR=${WORK_DIR}/python-install/include/python3.11" >> build.sh && \
135
+ echo "export PYO3_CROSS_PYTHON_VERSION=3.11" >> build.sh && \
136
+ echo "export CARGO_BUILD_TARGET=aarch64-linux-android" >> build.sh && \
137
+ echo "export PYO3_NO_PYTHON_VERSION_CHECK=1" >> build.sh && \
138
+ echo "export AARCH64_LINUX_ANDROID_OPENSSL_DIR=${OPENSSL_DIR}" >> build.sh && \
139
+ echo "export AARCH64_LINUX_ANDROID_OPENSSL_LIB_DIR=${OPENSSL_LIB_DIR}" >> build.sh && \
140
+ echo "export AARCH64_LINUX_ANDROID_OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}" >> build.sh && \
141
+ echo "export OPENSSL_STATIC=1" >> build.sh && \
142
  echo "" >> build.sh && \
143
+ echo "echo 'Starting Build for ${TARGET_PACKAGE}...'" >> build.sh && \
144
+ echo "pip wheel ${TARGET_PACKAGE} --wheel-dir=${OUTPUT_DIR} --no-binary=:all: --no-build-isolation -v" >> build.sh && \
 
 
 
 
 
 
 
 
145
  chmod +x build.sh
146
 
 
147
  RUN ./build.sh
148
 
149
+ RUN echo "Patching wheels to fix libpython dependency name..." && \
 
150
  for whl in *.whl; do \
151
+ echo "Processing $whl..."; \
152
+ mkdir -p tmp_wheel; \
153
+ unzip -q "$whl" -d tmp_wheel; \
154
+ # Find all .so files
155
+ find tmp_wheel -name "*.so" | while read so_file; do \
156
+ # If it links against the wrong name, swap it
157
+ if patchelf --print-needed "$so_file" | grep -q "libpython3.11.so.1.0"; then \
158
+ echo " -> Patching $so_file (replacing needed lib)"; \
159
+ patchelf --replace-needed "libpython3.11.so.1.0" "libpython3.11.so" "$so_file"; \
160
+ fi; \
161
+ done; \
162
+ # Repack
163
+ cd tmp_wheel; \
164
+ zip -r -q "../$whl" .; \
165
+ cd ..; \
166
+ rm -rf tmp_wheel; \
167
  done
168
 
 
169
  RUN for f in *.whl; do \
170
  if [ -f "$f" ]; then \
171
  new_name=$(echo "$f" | sed 's/linux_aarch64/android_24_arm64_v8a/g'); \
 
175
  fi \
176
  done
177
 
178
+ RUN echo "<html><head><title>Heroku-TL Wheels</title></head><body>" > index.html && \
179
+ echo "<h1>Wheels for heroku-tl-new (API 24, ARM64) [PATCHED NAME]</h1><ul>" >> index.html && \
 
180
  find . -name "*.whl" -printf "<li><a href=\"%P\">%P</a></li>\n" | sort >> index.html && \
181
  echo "</ul></body></html>" >> index.html
182