The Merge - builds

base image separation

docker/base/Dockerfile

@@ -18,12 +18,15 @@ ENV TZ=UTC
 # Copy contents of the project to /
 COPY ./fs/ /
 
-# install python
-RUN bash /ins/install_python.sh
-
 # install packages software
 RUN bash /ins/install_base_packages.sh
 
+# install python after packages to ensure version overriding
+RUN bash /ins/install_python.sh
+
+# install searxng
+RUN bash /ins/install_searxng.sh
+
 # configure ssh
 RUN bash /ins/configure_ssh.sh
 

docker/base/build.txt

@@ -13,3 +13,6 @@ docker buildx build -t frdel/agent-zero-base:latest --platform linux/amd64,linux
 
 # without cache
 docker buildx build -t frdel/agent-zero-base:latest --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) --no-cache .
+
+# plain output
+--progress=plain

docker/base/fs/etc/searxng/limiter.toml

@@ -0,0 +1,33 @@
+[real_ip]
+# Number of values to trust for X-Forwarded-For.
+x_for = 1
+
+# The prefix defines the number of leading bits in an address that are compared
+# to determine whether or not an address is part of a (client) network.
+ipv4_prefix = 32
+ipv6_prefix = 48
+
+[botdetection.ip_limit]
+# To get unlimited access in a local network, by default link-local addresses
+# (networks) are not monitored by the ip_limit
+filter_link_local = false
+
+# Activate link_token method in the ip_limit method
+link_token = false
+
+[botdetection.ip_lists]
+# In the limiter, the ip_lists method has priority over all other methods.
+# If an IP is in the pass_ip list, it has unrestricted access and is not
+# checked if, for example, the "user agent" suggests a bot (e.g., curl).
+block_ip = [
+    # '93.184.216.34',  # Example IPv4 address
+    # '257.1.1.1',      # Invalid IP --> will be ignored, logged in ERROR class
+]
+pass_ip = [
+    # '192.168.0.0/16',  # IPv4 private network
+    # 'fe80::/10',       # IPv6 link-local; overrides botdetection.ip_limit.filter_link_local
+]
+
+# Activate passlist of (hardcoded) IPs from the SearXNG organization,
+# e.g., `check.searx.space`.
+pass_searxng_org = true

docker/base/fs/etc/searxng/settings.yml

@@ -0,0 +1,78 @@
+# SearXNG settings
+
+use_default_settings: true
+
+general:
+  debug: false
+  instance_name: "SearXNG"
+
+search:
+  safe_search: 0
+  # autocomplete: 'duckduckgo'
+  formats:
+    - json
+    # - html
+
+server:
+  # Is overwritten by ${SEARXNG_SECRET}
+  secret_key: "dummy"
+  port: 55510
+  limiter: false
+  image_proxy: false
+  # public URL of the instance, to ensure correct inbound links. Is overwritten
+  # by ${SEARXNG_URL}.
+  # base_url: http://example.com/location
+
+# redis:
+#   # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
+#   url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
+
+ui:
+  static_use_hash: true
+
+# preferences:
+#   lock:
+#     - autocomplete
+#     - method
+
+enabled_plugins:
+  - 'Hash plugin'
+  - 'Self Informations'
+  - 'Tracker URL remover'
+  - 'Ahmia blacklist'
+  # - 'Hostnames plugin'  # see 'hostnames' configuration below
+  # - 'Open Access DOI rewrite'
+
+# plugins:
+#   - only_show_green_results
+
+# hostnames:
+#   replace:
+#     '(.*\.)?youtube\.com$': 'invidious.example.com'
+#     '(.*\.)?youtu\.be$': 'invidious.example.com'
+#   remove:
+#     - '(.*\.)?facebook.com$'
+#   low_priority:
+#     - '(.*\.)?google\.com$'
+#   high_priority:
+#     - '(.*\.)?wikipedia.org$'
+
+engines:
+
+#   - name: fdroid
+#     disabled: false
+#
+#   - name: apk mirror
+#     disabled: false
+#
+#   - name: mediathekviewweb
+#     categories: TV
+#     disabled: false
+#
+#   - name: invidious
+#     disabled: false
+#     base_url:
+#       - https://invidious.snopyta.org
+#       - https://invidious.tiekoetter.com
+#       - https://invidio.xamh.de
+#       - https://inv.riverside.rocks

docker/base/fs/ins/configure_venv.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-if [ ! -d /opt/venv ]; then
-    # Create and activate Python virtual environment
-    python3 -m venv /opt/venv
-    source /opt/venv/bin/activate
-else
-    source /opt/venv/bin/activate
-fi

docker/base/fs/ins/install_base_packages.sh

@@ -1,19 +1,9 @@
 #!/bin/bash
 set -e
 
-echo "=====BASE PACKAGES====="
+echo "====================BASE PACKAGES START===================="
 
-# Hold python3 packages to prevent overrides
-apt-mark hold python3
-
-# Install with --no-install-recommends to minimize unwanted dependencies
 apt-get install -y --no-install-recommends \
-    nodejs openssh-server sudo curl wget git ffmpeg supervisor cron
-
-
-echo "=====AFTER UPDATE====="
-
+    nodejs npm openssh-server sudo curl wget git ffmpeg supervisor cron
 
-echo "=====PYTHON VERSION: $(python3 --version) ====="
-echo "=====PYTHON OTHERS: $(ls /usr/bin/python*) ====="
-sleep 10
+echo "====================BASE PACKAGES END===================="

docker/base/fs/ins/install_python.sh

@@ -1,44 +1,52 @@
 #!/bin/bash
 set -e
 
-# echo "=====PYTHON 3.12 & SID REPO====="
+echo "====================PYTHON START===================="
 
-# apt clean
+echo "====================PYTHON 3.12 & SID REPO===================="
 
-# # ★ 1. Add sid repo & pin it for python 3.12
-# echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/debian-sid.list
-# cat >/etc/apt/preferences.d/python312 <<'EOF'
-# Package: *
-# Pin: release a=sid
-# Pin-Priority: 100
+apt clean
 
-# Package: python3.12*
-# Pin: release a=sid
-# Pin-Priority: 990
+# ★ 1. Add sid repo & pin it for python 3.12
+echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/debian-sid.list
+cat >/etc/apt/preferences.d/python312 <<'EOF'
+Package: *
+Pin: release a=sid
+Pin-Priority: 100
 
-# # Prevent Python 3.13 from being installed
-# Package: python3.13*
-# Pin: release *
-# Pin-Priority: -1
-# EOF
+Package: python3.12*
+Pin: release a=sid
+Pin-Priority: 990
 
-# apt-get update && apt-get -y upgrade
+# Prevent Python 3.13 from being installed
+Package: python3.13*
+Pin: release *
+Pin-Priority: -1
+EOF
 
-# apt-get install -y --no-install-recommends \
-#     python3.12 python3.12-venv python3.12-dev
+apt-get update && apt-get -y upgrade
+
+apt-get install -y --no-install-recommends \
+    python3.12 python3.12-venv python3.12-dev
 
-# # ★ 3. Switch the interpreter
-# # update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 0
+# ★ 3. Switch the interpreter
+# update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 0
 # update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
 # update-alternatives --set python3 /usr/bin/python3.12
 
-# echo "=====PYTHON VERSION: $(python3 --version) ====="
-# echo "=====PYTHON OTHERS: $(ls /usr/bin/python*) ====="
-# sleep 10
+echo "====================PYTHON VERSION: $(python3 --version) ===================="
+echo "====================PYTHON OTHERS: $(ls /usr/bin/python*) "
 
-apt-get update && apt-get -y upgrade
+echo "====================PYTHON VENV===================="
+
+# create and activate default venv
+python3.12 -m venv /opt/venv
+source /opt/venv/bin/activate
+
+# upgrade pip and install static packages
+pip install --upgrade pip ipython requests
+# Install some packages in specific variants
+pip install torch --index-url https://download.pytorch.org/whl/cpu
 
-apt-get install -y --no-install-recommends \
-    python3 python3-venv python3-dev
 
-. /ins/configure_venv.sh
+echo "====================PYTHON END===================="

docker/{run → base}/fs/ins/install_searxng.sh

@@ -1,10 +1,13 @@
 #!/bin/bash
 
+echo "====================SEARXNG1 START===================="
+
 # Install necessary packages
 apt-get install -y \
-    python3-dev python3-babel python3-venv \
-    uwsgi uwsgi-plugin-python3 \
+    python3.12-dev python3.12-venv \
     git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
+#    python3.12-babel uwsgi uwsgi-plugin-python3
+
 
 # Add the searxng system user
 useradd --shell /bin/bash --system \
@@ -19,5 +22,8 @@ usermod -aG sudo searxng
 mkdir "/usr/local/searxng"
 chown -R "searxng:searxng" "/usr/local/searxng"
 
+echo "====================SEARXNG1 END===================="
+
 # Start a new shell as the searxng user and run the installation script
-su - searxng -c "bash /ins/install_searxng2.sh"
+su - searxng -c "bash /ins/install_searxng2.sh"
+

docker/{run → base}/fs/ins/install_searxng2.sh

@@ -1,11 +1,16 @@
 #!/bin/bash
 
+echo "====================SEARXNG2 START===================="
+
+
 # clone SearXNG repo
 git clone "https://github.com/searxng/searxng" \
                    "/usr/local/searxng/searxng-src"
 
+echo "====================SEARXNG2 VENV===================="
+
 # create virtualenv:
-python3 -m venv "/usr/local/searxng/searx-pyenv"
+python3.12 -m venv "/usr/local/searxng/searx-pyenv"
 
 # make it default
 echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
@@ -14,6 +19,8 @@ echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
 # activate venv
 source "/usr/local/searxng/searx-pyenv/bin/activate"
 
+echo "====================SEARXNG2 INST===================="
+
 # update pip's boilerplate
 pip install -U pip
 pip install -U setuptools
@@ -25,4 +32,6 @@ cd "/usr/local/searxng/searxng-src"
 pip install --use-pep517 --no-build-isolation -e .
 
 # cleanup cache
-pip cache purge
+pip cache purge
+
+echo "====================SEARXNG2 END===================="

docker/run/Dockerfile

@@ -1,6 +1,6 @@
-# Use the latest slim version of Debian
-FROM agent-zero-base:local
-# FROM frdel/agent-zero-base:latest
+# Use the pre-built base image for A0
+# FROM agent-zero-base:local
+FROM frdel/agent-zero-base:latest
 
 # Check if the argument is provided, else throw an error
 ARG BRANCH
@@ -27,7 +27,7 @@ RUN echo "cache buster $CACHE_DATE" && bash /ins/install_A02.sh $BRANCH
 RUN bash /ins/post_install.sh $BRANCH
 
 # Expose ports
-EXPOSE 22 80 3000-9999
+EXPOSE 22 80 9000-9009
 
 RUN chmod +x /exe/initialize.sh /exe/run_A0.sh /exe/run_searxng.sh /exe/run_tunnel_api.sh
 

docker/run/build.txt

@@ -19,3 +19,7 @@ docker buildx build --build-arg BRANCH=testing -t frdel/agent-zero-run:testing -
 
 # main
 docker buildx build --build-arg BRANCH=main -t frdel/agent-zero-run:vx.x.x  -t frdel/agent-zero-run:latest --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .
+
+
+# plain output
+--progress=plain

docker/run/fs/exe/initialize.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "Running initialization script..."
+
 # branch from parameter
 if [ -z "$1" ]; then
     echo "Error: Branch parameter is empty. Please provide a valid branch name."
@@ -17,4 +19,5 @@ chmod 444 /root/.profile
 # update package list to save time later
 apt-get update > /dev/null 2>&1 &
 
-exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+# Redirect only stdout to /dev/null to suppress normal messages but keep errors visible
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf > /dev/null

docker/run/fs/ins/install_A0.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 # branch from parameter
 if [ -z "$1" ]; then
@@ -13,11 +14,11 @@ git clone -b "$BRANCH" "https://github.com/frdel/agent-zero" "/git/agent-zero"
 # setup python environment
 . "/ins/setup_venv.sh" "$@"
 
-# Ensure the virtual environment and pip setup
-pip install --upgrade pip ipython requests
-
-# Install some packages in specific variants
-pip install torch --index-url https://download.pytorch.org/whl/cpu
+# moved to base image
+# # Ensure the virtual environment and pip setup
+# pip install --upgrade pip ipython requests
+# # Install some packages in specific variants
+# pip install torch --index-url https://download.pytorch.org/whl/cpu
 
 # Install remaining A0 python packages
 pip install -r /git/agent-zero/requirements.txt

docker/run/fs/ins/install_A02.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 # cachebuster script, this helps speed up docker builds
+
+# remove repo
 rm -rf /git/agent-zero
 
 # run the original install script again

docker/run/fs/ins/install_additional.sh

@@ -3,5 +3,5 @@
 # install playwright - moved to install A0
 # bash /ins/install_playwright.sh "$@"
 
-# searxng
-bash /ins/install_searxng.sh "$@"
+# searxng - moved to base image
+# bash /ins/install_searxng.sh "$@"

docker/run/fs/ins/install_playwright.sh

@@ -3,13 +3,13 @@
 # activate venv
 . "/ins/setup_venv.sh" "$@"
 
-# install playwright if not installed
+# install playwright if not installed (should be from requirements.txt)
 pip install playwright
 
 # install chromium with dependencies
 # for kali-based
 # if [ "$@" = "hacking" ]; then
-    apt-get install -y fonts-unifont libnss3 libnspr4
+    apt-get install -y fonts-unifont libnss3 libnspr4 libatk1.0-0 libatspi2.0-0 libxcomposite1 libxdamage1      
     playwright install chromium-headless-shell
 # else
 #     # for debian based

docker/run/fs/ins/setup_venv.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
-if [ ! -d /opt/venv ]; then
-    # Create and activate Python virtual environment
-    python3 -m venv /opt/venv
+# this has to be ready from base image
+# if [ ! -d /opt/venv ]; then
+#     # Create and activate Python virtual environment
+#     python3.12 -m venv /opt/venv
+#     source /opt/venv/bin/activate
+# else
     source /opt/venv/bin/activate
-else
-    source /opt/venv/bin/activate
-fi
+# fi