fix: harden SQLBot images and streamline builds

.github/workflows/build_and_push.yml

@@ -14,6 +14,14 @@ on:
         default: false
         required: true
         type: boolean
+      runtimeBaseImage:
+        description: 'SQLBot runtime base image (optional tag or digest override)'
+        required: false
+        type: string
+      builderBaseImage:
+        description: 'SQLBot build base image paired with the runtime base (optional tag or digest)'
+        required: false
+        type: string
       architecture:
         description: 'Architecture'
         required: true
@@ -80,8 +88,11 @@ jobs:
           username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }}
           password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }}
       - name: Docker Buildx (build-and-push)
+        env:
+          SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || 'registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest' }}
+          SQLBOT_BUILD_IMAGE: ${{ inputs.builderBaseImage || 'registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-build-base:latest' }}
         run: |
-          docker buildx build --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --build-arg SQLBOT_BUILD_IMAGE --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
 
   build-and-push-to-dockerhub:
     if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
@@ -128,5 +139,8 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Docker Buildx (build-and-push)
+        env:
+          SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || 'docker.io/dataease/sqlbot-python-pg:latest' }}
+          SQLBOT_BUILD_IMAGE: ${{ inputs.builderBaseImage || 'docker.io/dataease/sqlbot-build-base:latest' }}
         run: |
-          docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --build-arg SQLBOT_BUILD_IMAGE --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}

.github/workflows/build_base_and_push.yml

@@ -44,19 +44,22 @@ jobs:
           ref: ${{ github.ref_name }}
       - name: Prepare
         id: prepare
+        env:
+          DOCKER_IMAGE: ${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg
+          TAG_NAME: ${{ inputs.dockerImageTag }}
+          TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }}
         run: |
-          DOCKER_IMAGE=${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg
-          DOCKER_PLATFORMS=${{ github.event.inputs.architecture }}
-          TAG_NAME=${{ github.event.inputs.dockerImageTag }}
-          TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
-          if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then            
-            DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest"
-          else
-            DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
-          fi
-          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
-            --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
-            ${DOCKER_IMAGE_TAGS} .
+          image_tags() {
+            local image="$1"
+            local tags="$image:$TAG_NAME"
+            if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then
+              tags="$tags,$image:${TAG_NAME%%.*},$image:latest"
+            fi
+            printf '%s' "$tags"
+          }
+          printf '%s=%s\n' \
+            runtime_tags "$(image_tags "$DOCKER_IMAGE")" \
+            build_tags "$(image_tags "${DOCKER_IMAGE%/*}/sqlbot-build-base")" >> "$GITHUB_OUTPUT"
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -68,8 +71,14 @@ jobs:
           username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }}
           password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }}
       - name: Docker Buildx (build-and-push)
+        env:
+          SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.runtime_tags }}
+          SQLBOT_BUILD_TAGS: ${{ steps.prepare.outputs.build_tags }}
+          SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }}
         run: |
-          docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \
+            --set 'base.output=type=image,push=true,oci-mediatypes=false' \
+            --set 'build-base.output=type=image,push=true,oci-mediatypes=false' base build-base
 
   build-and-push-to-dockerhub:
     if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
@@ -81,19 +90,22 @@ jobs:
           ref: ${{ github.ref_name }}
       - name: Prepare
         id: prepare
+        env:
+          DOCKER_IMAGE: dataease/sqlbot-python-pg
+          TAG_NAME: ${{ inputs.dockerImageTag }}
+          TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }}
         run: |
-          DOCKER_IMAGE=dataease/sqlbot-python-pg
-          DOCKER_PLATFORMS=${{ github.event.inputs.architecture }}
-          TAG_NAME=${{ github.event.inputs.dockerImageTag }}          
-          TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
-          if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then            
-            DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest"
-          else
-            DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
-          fi
-          echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
-            --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
-            ${DOCKER_IMAGE_TAGS} .
+          image_tags() {
+            local image="$1"
+            local tags="$image:$TAG_NAME"
+            if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then
+              tags="$tags,$image:${TAG_NAME%%.*},$image:latest"
+            fi
+            printf '%s' "$tags"
+          }
+          printf '%s=%s\n' \
+            runtime_tags "$(image_tags "$DOCKER_IMAGE")" \
+            build_tags "$(image_tags "${DOCKER_IMAGE%/*}/sqlbot-build-base")" >> "$GITHUB_OUTPUT"
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -104,5 +116,11 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Docker Buildx (build-and-push)
+        env:
+          SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.runtime_tags }}
+          SQLBOT_BUILD_TAGS: ${{ steps.prepare.outputs.build_tags }}
+          SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }}
         run: |
-          docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }}
+          docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \
+            --set 'base.output=type=image,push=true,oci-mediatypes=false' \
+            --set 'build-base.output=type=image,push=true,oci-mediatypes=false' base build-base

Dockerfile

@@ -1,72 +1,82 @@
-# Build sqlbot
-FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model
-FROM --platform=${BUILDPLATFORM} registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-ui-builder
-ENV SQLBOT_HOME=/opt/sqlbot
-ENV APP_HOME=${SQLBOT_HOME}/app
-ENV UI_HOME=${SQLBOT_HOME}/frontend
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN mkdir -p ${APP_HOME} ${UI_HOME}
-
-COPY frontend /tmp/frontend
-RUN cd /tmp/frontend && npm install && npm run build && mv dist ${UI_HOME}/dist
-
-
-FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-builder
-# Set build environment variables
-ENV PYTHONUNBUFFERED=1
-ENV SQLBOT_HOME=/opt/sqlbot
-ENV APP_HOME=${SQLBOT_HOME}/app
-ENV UI_HOME=${SQLBOT_HOME}/frontend
-ENV PYTHONPATH=${SQLBOT_HOME}/app
-ENV PATH="${APP_HOME}/.venv/bin:$PATH"
-ENV UV_COMPILE_BYTECODE=1
-ENV UV_LINK_MODE=copy
-ENV DEBIAN_FRONTEND=noninteractive
+# syntax=docker/dockerfile:1
+ARG SQLBOT_RUNTIME_IMAGE=registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest
+ARG SQLBOT_BUILD_IMAGE=registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-build-base:latest
+ARG NODE_IMAGE=node:22-bookworm-slim
 
-# Create necessary directories
-RUN mkdir -p ${APP_HOME} ${UI_HOME}
-
-WORKDIR ${APP_HOME}
-
-COPY  --from=sqlbot-ui-builder ${UI_HOME} ${UI_HOME}
-# Install dependencies
-RUN test -f "./uv.lock" && \
-    --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=backend/uv.lock,target=uv.lock \
-    --mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \
-    uv sync --frozen --no-install-project || echo "uv.lock file not found, skipping intermediate-layers"
+FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model
 
-COPY ./backend ${APP_HOME}
+FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS sqlbot-ui-builder
+WORKDIR /tmp/frontend
+COPY frontend/ ./
+RUN npm install && npm run build
+
+FROM ${SQLBOT_RUNTIME_IMAGE} AS runtime-base
+# Rebuild Dockerfile-base first, then point this build at the rebuilt image.
+RUN if ! node -e 'if (process.versions.node.split(".")[0] !== "22") process.exit(1)' \
+        || [ -e /opt/dmdbms/bin/libssl.so ] \
+        || ! dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libharfbuzz0b)" ge '6.0.0+dfsg-3+sqlbot1' \
+        || dpkg-query -W libtiff6 >/dev/null 2>&1; then \
+        echo 'Rebuild Dockerfile-base and set SQLBOT_RUNTIME_IMAGE to the new image.' >&2; \
+        exit 1; \
+    fi
+
+FROM ${SQLBOT_BUILD_IMAGE} AS build-base
+
+FROM build-base AS sqlbot-builder
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/opt/sqlbot/app \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=never
+WORKDIR /opt/sqlbot/app
+COPY backend/ ./
+
+# Resolve the security constraints together with the application dependencies.
+# Elasticsearch 7.17.13 permits urllib3 2.x on Python 3.11. The source-built
+# PostgreSQL and SQL Server drivers use the same patched OS libraries as runtime.
+RUN python - <<'PY'
+from pathlib import Path
+
+project = Path('pyproject.toml')
+contents = project.read_text()
+for old, new in (
+    ('psycopg[binary]<4.0.0,>=3.1.13', 'psycopg[c]<4.0.0,>=3.1.13'),
+    ('psycopg2-binary (>=2.9.12,<3.0.0)', 'psycopg2 (>=2.9.12,<3.0.0)'),
+    ('elasticsearch[requests] (>=7.10,<8.0)', 'elasticsearch[requests] (>=7.17.13,<8.0)'),
+    ('dmpython==2.5.22', 'dmpython==2.6.1'),
+):
+    assert contents.count(old) == 1, old
+    contents = contents.replace(old, new)
+contents = contents.replace(
+    '[tool.uv]\n',
+    '[tool.uv]\nconstraint-dependencies = ["idna>=3.7", "setuptools>=78.1.1", "urllib3>=2.2.2,<3", "pymssql==2.4.6"]\n',
+    1,
+)
+project.write_text(contents)
+PY
 
-# Final sync to ensure all dependencies are installed
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --extra cpu --default-index https://pypi.org/simple
-
-# Build g2-ssr
-FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS ssr-builder
+    uv sync --no-dev --extra cpu --python /usr/local/bin/python3.11 \
+        --default-index https://pypi.org/simple \
+        --no-binary-package psycopg2 --no-binary-package pymssql \
+    && uv pip check --python .venv/bin/python \
+    && .venv/bin/python -c 'import psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"'
 
+FROM build-base AS ssr-builder
 WORKDIR /app
-
-# Install build dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential python3 pkg-config \
-    libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
-    libpixman-1-dev libfreetype6-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# configure npm
-RUN npm config set fund false \
-    && npm config set audit false \
-    && npm config set progress false
-
-COPY g2-ssr/app.js g2-ssr/package.json /app/
-COPY g2-ssr/charts/* /app/charts/
-RUN npm install
-
-# Runtime stage
-FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest
-
+COPY g2-ssr/app.js g2-ssr/package.json ./
+COPY g2-ssr/charts/ ./charts/
+# g2-ssr uses canvas. Drop the unused node-canvas fork, which targets old V8.
+# Build the Node 22-compatible canvas release against distro libraries.
+RUN npm pkg delete dependencies.node-canvas dependencies.nan \
+    && npm pkg set 'overrides.canvas=^3.2.3' \
+    && npm_config_build_from_source=true npm install --no-fund --no-audit --no-progress
+
+FROM runtime-base
+COPY --from=build-base /usr/local/lib/libsybdb.so.5.* /usr/local/lib/
+COPY --from=build-base /etc/freetds/ /etc/freetds/
+COPY --from=build-base /usr/local/share/doc/freetds/COPYING_LIB.txt /usr/local/share/doc/freetds/COPYING_LIB.txt
+RUN ldconfig
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
     echo "Asia/Shanghai" > /etc/timezone
 
@@ -75,6 +85,8 @@ ENV PYTHONUNBUFFERED=1
 ENV SQLBOT_HOME=/opt/sqlbot
 ENV PYTHONPATH=${SQLBOT_HOME}/app
 ENV PATH="${SQLBOT_HOME}/app/.venv/bin:$PATH"
+# The DM SSL symlinks need the wheel's hashed crypto library at load time.
+ENV LD_LIBRARY_PATH="${SQLBOT_HOME}/app/.venv/lib/python3.11/site-packages/dmpython.libs:$LD_LIBRARY_PATH"
 
 ENV POSTGRES_DB=sqlbot
 ENV POSTGRES_USER=root
@@ -83,14 +95,26 @@ ENV POSTGRES_PASSWORD=Password123@pg
 # Copy necessary files from builder
 COPY start.sh /opt/sqlbot/app/start.sh
 COPY g2-ssr/*.ttf /usr/share/fonts/truetype/liberation/
-COPY --from=sqlbot-builder ${SQLBOT_HOME} ${SQLBOT_HOME}
-COPY --from=ssr-builder /app /opt/sqlbot/g2-ssr
+COPY --from=sqlbot-builder \
+    --exclude=.venv/lib/python3.11/site-packages/torch/test \
+    /opt/sqlbot/app/ /opt/sqlbot/app/
+COPY --from=sqlbot-ui-builder /tmp/frontend/dist/ /opt/sqlbot/frontend/dist/
+COPY --from=ssr-builder /app/ /opt/sqlbot/g2-ssr/
 COPY g2-ssr/supervisord.conf /etc/supervisor/conf.d/g2-ssr.conf
-COPY --from=vector-model /opt/maxkb/app/model /opt/sqlbot/models
+# SQLBot runs this embedding model with PyTorch; omit its duplicate ONNX export.
+COPY --from=vector-model --exclude=embedding/shibing624_text2vec-base-chinese/onnx /opt/maxkb/app/model /opt/sqlbot/models
 
 WORKDIR ${SQLBOT_HOME}/app
 
-RUN mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs
+RUN set -eu; \
+    mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs; \
+    dmssl="$SQLBOT_HOME/app/.venv/lib/python3.11/site-packages/dmssl"; \
+    for library in libssl.so libcrypto.so; do \
+        test -f "$dmssl/$library"; \
+        ln -s "$dmssl/$library" "$DM_HOME/bin/$library"; \
+    done; \
+    python -c 'import ctypes, os, ssl; import dmPython, psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"; ctypes.CDLL(os.environ["DM_HOME"] + "/bin/libssl.so")'; \
+    node -e "require('/opt/sqlbot/g2-ssr/node_modules/@antv/g2-ssr')"
 
 EXPOSE 3000 8000 8001 5432