Skip to content

Commit 550fa31

Browse files
xuwei-fit2cloudxuwei-fit2cloud
committed
fix: harden container images and offline installer
1 parent 0d1a4d3 commit 550fa31

11 files changed

Lines changed: 380 additions & 161 deletions

File tree

‎.github/workflows/build_and_push.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ on:
1414
default: false
1515
required: true
1616
type: boolean
17+
runtimeBaseImage:
18+
description: 'SQLBot runtime base image (optional tag or digest override)'
19+
required: false
20+
type: string
1721
architecture:
1822
description: 'Architecture'
1923
required: true
@@ -80,8 +84,10 @@ jobs:
8084
username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }}
8185
password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }}
8286
- name: Docker Buildx (build-and-push)
87+
env:
88+
SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || 'registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest' }}
8389
run: |
84-
docker buildx build --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
90+
docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --provenance=false --sbom=false --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
8591
8692
build-and-push-to-dockerhub:
8793
if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
@@ -128,5 +134,7 @@ jobs:
128134
username: ${{ secrets.DOCKERHUB_USERNAME }}
129135
password: ${{ secrets.DOCKERHUB_TOKEN }}
130136
- name: Docker Buildx (build-and-push)
137+
env:
138+
SQLBOT_RUNTIME_IMAGE: ${{ inputs.runtimeBaseImage || 'registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest' }}
131139
run: |
132-
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}
140+
docker buildx build --build-arg SQLBOT_RUNTIME_IMAGE --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }}

‎.github/workflows/build_base_and_push.yml‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,16 @@ jobs:
4444
ref: ${{ github.ref_name }}
4545
- name: Prepare
4646
id: prepare
47+
env:
48+
DOCKER_IMAGE: ${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg
49+
TAG_NAME: ${{ inputs.dockerImageTag }}
50+
TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }}
4751
run: |
48-
DOCKER_IMAGE=${{ secrets.ALIYUN_REGISTRY_HOST }}/dataease/sqlbot-python-pg
49-
DOCKER_PLATFORMS=${{ github.event.inputs.architecture }}
50-
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
51-
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
52-
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
53-
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest"
54-
else
55-
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
52+
tags="$DOCKER_IMAGE:$TAG_NAME"
53+
if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then
54+
tags="$tags,$DOCKER_IMAGE:${TAG_NAME%%.*},$DOCKER_IMAGE:latest"
5655
fi
57-
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
58-
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
59-
${DOCKER_IMAGE_TAGS} .
56+
echo "tags=$tags" >> "$GITHUB_OUTPUT"
6057
- name: Set up QEMU
6158
uses: docker/setup-qemu-action@v3
6259
- name: Set up Docker Buildx
@@ -68,8 +65,12 @@ jobs:
6865
username: ${{ secrets.ALIYUN_REGISTRY_USERNAME }}
6966
password: ${{ secrets.ALIYUN_REGISTRY_PASSWORD }}
7067
- name: Docker Buildx (build-and-push)
68+
env:
69+
SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.tags }}
70+
SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }}
7171
run: |
72-
docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }}
72+
docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \
73+
--set 'base.output=type=image,push=true,oci-mediatypes=false' base
7374
7475
build-and-push-to-dockerhub:
7576
if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
@@ -81,19 +82,16 @@ jobs:
8182
ref: ${{ github.ref_name }}
8283
- name: Prepare
8384
id: prepare
85+
env:
86+
DOCKER_IMAGE: dataease/sqlbot-python-pg
87+
TAG_NAME: ${{ inputs.dockerImageTag }}
88+
TAG_NAME_WITH_LATEST: ${{ inputs.dockerImageTagWithLatest }}
8489
run: |
85-
DOCKER_IMAGE=dataease/sqlbot-python-pg
86-
DOCKER_PLATFORMS=${{ github.event.inputs.architecture }}
87-
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
88-
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
89-
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
90-
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*} --tag ${DOCKER_IMAGE}:latest"
91-
else
92-
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
90+
tags="$DOCKER_IMAGE:$TAG_NAME"
91+
if [[ "$TAG_NAME_WITH_LATEST" == 'true' ]]; then
92+
tags="$tags,$DOCKER_IMAGE:${TAG_NAME%%.*},$DOCKER_IMAGE:latest"
9393
fi
94-
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
95-
--build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
96-
${DOCKER_IMAGE_TAGS} .
94+
echo "tags=$tags" >> "$GITHUB_OUTPUT"
9795
- name: Set up QEMU
9896
uses: docker/setup-qemu-action@v3
9997
- name: Set up Docker Buildx
@@ -104,5 +102,9 @@ jobs:
104102
username: ${{ secrets.DOCKERHUB_USERNAME }}
105103
password: ${{ secrets.DOCKERHUB_TOKEN }}
106104
- name: Docker Buildx (build-and-push)
105+
env:
106+
SQLBOT_BASE_TAGS: ${{ steps.prepare.outputs.tags }}
107+
SQLBOT_BASE_PLATFORMS: ${{ inputs.architecture }}
107108
run: |
108-
docker buildx build -f Dockerfile-base --output "type=image,push=true,oci-mediatypes=false" --provenance=false --sbom=false ${{ steps.prepare.outputs.buildx_args }}
109+
docker buildx bake -f docker-bake.hcl --pull --no-cache --provenance=false --sbom=false \
110+
--set 'base.output=type=image,push=true,oci-mediatypes=false' base

‎.github/workflows/package_and_push.yml‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ jobs:
4141
- name: Prepare package files
4242
env:
4343
TAG_NAME: ${{ github.event.inputs.dockerImageTag }}
44-
DOCKER_DOWNLOAD_SERVER: ${{ secrets.DOCKER_DOWNLOAD_SERVER }}
4544
ALIYUN_REGISTRY_HOST: ${{ secrets.ALIYUN_REGISTRY_HOST }}
4645
ALIYUN_OSS_BUCKET: ${{ secrets.ALIYUN_OSS_BUCKET }}
4746
ALIYUN_OSS_BUCKET_ENDPOINT: ${{ secrets.ALIYUN_OSS_BUCKET_ENDPOINT }}
@@ -85,17 +84,8 @@ jobs:
8584
docker save $image_path | gzip > images/$image_name.tar.gz
8685
done
8786
88-
DOCKER_VERSION="docker-27.2.0"
89-
DOCKER_COMPOSE_VERSION="v2.29.2"
90-
91-
wget ${DOCKER_DOWNLOAD_SERVER}/download/${ARCH}/${DOCKER_VERSION}.tgz
92-
wget ${DOCKER_DOWNLOAD_SERVER}/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-${ARCH} && mv docker-compose-linux-${ARCH} docker-compose && chmod +x docker-compose
93-
tar -zxvf ${DOCKER_VERSION}.tgz
94-
rm -rf ${DOCKER_VERSION}.tgz
95-
mv docker bin && mkdir docker && mv bin docker/
96-
mv docker-compose docker/bin
97-
wget https://resource-fit2cloud-com.oss-cn-hangzhou.aliyuncs.com/docker/docker.service
98-
mkdir docker/service && mv docker.service docker/service/
87+
source ./docker-release.sh
88+
sqlbot_prepare_docker_bundle "$ARCH"
9989
10090
if [[ $ARCH == 'x86_64' ]]; then
10191
#打包离线包

‎Dockerfile‎

Lines changed: 114 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,110 @@
1-
# Build sqlbot
2-
FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model
3-
FROM --platform=${BUILDPLATFORM} registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-ui-builder
4-
ENV SQLBOT_HOME=/opt/sqlbot
5-
ENV APP_HOME=${SQLBOT_HOME}/app
6-
ENV UI_HOME=${SQLBOT_HOME}/frontend
7-
ENV DEBIAN_FRONTEND=noninteractive
8-
9-
RUN mkdir -p ${APP_HOME} ${UI_HOME}
10-
11-
COPY frontend /tmp/frontend
12-
RUN cd /tmp/frontend && npm install && npm run build && mv dist ${UI_HOME}/dist
13-
14-
15-
FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-builder
16-
# Set build environment variables
17-
ENV PYTHONUNBUFFERED=1
18-
ENV SQLBOT_HOME=/opt/sqlbot
19-
ENV APP_HOME=${SQLBOT_HOME}/app
20-
ENV UI_HOME=${SQLBOT_HOME}/frontend
21-
ENV PYTHONPATH=${SQLBOT_HOME}/app
22-
ENV PATH="${APP_HOME}/.venv/bin:$PATH"
23-
ENV UV_COMPILE_BYTECODE=1
24-
ENV UV_LINK_MODE=copy
25-
ENV DEBIAN_FRONTEND=noninteractive
26-
27-
# Create necessary directories
28-
RUN mkdir -p ${APP_HOME} ${UI_HOME}
1+
# syntax=docker/dockerfile:1
2+
ARG SQLBOT_RUNTIME_IMAGE=registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest
3+
ARG NODE_IMAGE=node:22-bookworm-slim
294

30-
WORKDIR ${APP_HOME}
31-
32-
COPY --from=sqlbot-ui-builder ${UI_HOME} ${UI_HOME}
33-
# Install dependencies
34-
RUN test -f "./uv.lock" && \
35-
--mount=type=cache,target=/root/.cache/uv \
36-
--mount=type=bind,source=backend/uv.lock,target=uv.lock \
37-
--mount=type=bind,source=backend/pyproject.toml,target=pyproject.toml \
38-
uv sync --frozen --no-install-project || echo "uv.lock file not found, skipping intermediate-layers"
5+
FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model
396

40-
COPY ./backend ${APP_HOME}
7+
FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS sqlbot-ui-builder
8+
WORKDIR /tmp/frontend
9+
COPY frontend/ ./
10+
RUN npm install && npm run build
11+
12+
FROM ${SQLBOT_RUNTIME_IMAGE} AS runtime-base
13+
# Rebuild Dockerfile-base first, then point this build at the rebuilt image.
14+
RUN if ! node -e 'if (process.versions.node.split(".")[0] !== "22") process.exit(1)' \
15+
|| [ -e /opt/dmdbms/bin/libssl.so ] \
16+
|| ! dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libharfbuzz0b)" ge '6.0.0+dfsg-3+sqlbot1' \
17+
|| dpkg-query -W libtiff6 >/dev/null 2>&1; then \
18+
echo 'Rebuild Dockerfile-base and set SQLBOT_RUNTIME_IMAGE to the new image.' >&2; \
19+
exit 1; \
20+
fi
21+
22+
FROM runtime-base AS driver-toolchain
23+
RUN apt-get -o APT::Update::Error-Mode=any update \
24+
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
25+
build-essential libpq-dev libssl-dev libkrb5-dev pkg-config \
26+
&& rm -rf /var/lib/apt/lists/*
27+
# Match the FreeTDS release used by pymssql 2.4.6 wheels and link it to
28+
# the runtime's maintained OpenSSL and Kerberos libraries.
29+
RUN set -eu; \
30+
curl -fsSL --retry 3 https://www.freetds.org/files/stable/freetds-1.5.19.tar.gz -o /tmp/freetds.tar.gz; \
31+
echo 'ff70d0c8ae8dd23e7b1314274d6e9d30b479998415760b3fa656754b737c2bd6 /tmp/freetds.tar.gz' | sha256sum -c -; \
32+
mkdir /tmp/freetds; \
33+
tar -xzf /tmp/freetds.tar.gz --strip-components=1 -C /tmp/freetds; \
34+
cd /tmp/freetds; \
35+
./configure --prefix=/usr/local --sysconfdir=/etc/freetds --enable-msdblib \
36+
--with-tdsver=auto --with-openssl=yes --with-gnutls=no --enable-krb5 \
37+
--disable-static --disable-odbc --disable-apps --disable-server --disable-pool; \
38+
make -j "$(nproc)"; \
39+
make install; \
40+
install -D -m 0644 COPYING_LIB.txt /usr/local/share/doc/freetds/COPYING_LIB.txt; \
41+
ldconfig; \
42+
cd /; \
43+
rm -rf /tmp/freetds /tmp/freetds.tar.gz
44+
ENV LINK_FREETDS_STATICALLY=NO \
45+
PYMSSQL_FREETDS=/usr/local
46+
47+
FROM driver-toolchain AS sqlbot-builder
48+
ENV PYTHONUNBUFFERED=1 \
49+
PYTHONPATH=/opt/sqlbot/app \
50+
UV_COMPILE_BYTECODE=1 \
51+
UV_LINK_MODE=copy \
52+
UV_PYTHON_DOWNLOADS=never
53+
WORKDIR /opt/sqlbot/app
54+
COPY backend/ ./
55+
56+
# Resolve the security constraints together with the application dependencies.
57+
# Elasticsearch 7.17.13 permits urllib3 2.x on Python 3.11. The source-built
58+
# PostgreSQL and SQL Server drivers use the same patched OS libraries as runtime.
59+
RUN python - <<'PY'
60+
from pathlib import Path
61+
62+
project = Path('pyproject.toml')
63+
contents = project.read_text()
64+
for old, new in (
65+
('psycopg[binary]<4.0.0,>=3.1.13', 'psycopg[c]<4.0.0,>=3.1.13'),
66+
('psycopg2-binary (>=2.9.12,<3.0.0)', 'psycopg2 (>=2.9.12,<3.0.0)'),
67+
('elasticsearch[requests] (>=7.10,<8.0)', 'elasticsearch[requests] (>=7.17.13,<8.0)'),
68+
('dmpython==2.5.22', 'dmpython==2.6.1'),
69+
):
70+
assert contents.count(old) == 1, old
71+
contents = contents.replace(old, new)
72+
contents = contents.replace(
73+
'[tool.uv]\n',
74+
'[tool.uv]\nconstraint-dependencies = ["idna>=3.7", "setuptools>=78.1.1", "urllib3>=2.2.2,<3", "pymssql==2.4.6"]\n',
75+
1,
76+
)
77+
project.write_text(contents)
78+
PY
4179

42-
# Final sync to ensure all dependencies are installed
4380
RUN --mount=type=cache,target=/root/.cache/uv \
44-
uv sync --extra cpu --default-index https://pypi.org/simple
45-
46-
# Build g2-ssr
47-
FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS ssr-builder
81+
uv sync --no-dev --extra cpu --python /usr/local/bin/python3.11 \
82+
--default-index https://pypi.org/simple \
83+
--no-binary-package psycopg2 --no-binary-package pymssql \
84+
&& uv pip check --python .venv/bin/python \
85+
&& .venv/bin/python -c 'import psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"'
4886

87+
FROM ${NODE_IMAGE} AS ssr-builder
4988
WORKDIR /app
50-
51-
# Install build dependencies
52-
RUN apt-get update && apt-get install -y --no-install-recommends \
53-
build-essential python3 pkg-config \
54-
libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
55-
libpixman-1-dev libfreetype6-dev \
89+
RUN apt-get -o APT::Update::Error-Mode=any update \
90+
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
91+
build-essential python3 pkg-config \
92+
libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
93+
libpixman-1-dev libfreetype6-dev \
5694
&& rm -rf /var/lib/apt/lists/*
57-
58-
# configure npm
59-
RUN npm config set fund false \
60-
&& npm config set audit false \
61-
&& npm config set progress false
62-
63-
COPY g2-ssr/app.js g2-ssr/package.json /app/
64-
COPY g2-ssr/charts/* /app/charts/
65-
RUN npm install
66-
67-
# Runtime stage
68-
FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-python-pg:latest
69-
95+
COPY g2-ssr/app.js g2-ssr/package.json ./
96+
COPY g2-ssr/charts/ ./charts/
97+
# g2-ssr uses canvas. Drop the unused node-canvas fork, which targets old V8.
98+
# Build the Node 22-compatible canvas release against distro libraries.
99+
RUN npm pkg delete dependencies.node-canvas dependencies.nan \
100+
&& npm pkg set 'overrides.canvas=^3.2.3' \
101+
&& npm_config_build_from_source=true npm install --no-fund --no-audit --no-progress
102+
103+
FROM runtime-base
104+
COPY --from=driver-toolchain /usr/local/lib/libsybdb.so.5.* /usr/local/lib/
105+
COPY --from=driver-toolchain /etc/freetds/ /etc/freetds/
106+
COPY --from=driver-toolchain /usr/local/share/doc/freetds/COPYING_LIB.txt /usr/local/share/doc/freetds/COPYING_LIB.txt
107+
RUN ldconfig
70108
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
71109
echo "Asia/Shanghai" > /etc/timezone
72110

@@ -75,6 +113,8 @@ ENV PYTHONUNBUFFERED=1
75113
ENV SQLBOT_HOME=/opt/sqlbot
76114
ENV PYTHONPATH=${SQLBOT_HOME}/app
77115
ENV PATH="${SQLBOT_HOME}/app/.venv/bin:$PATH"
116+
# The DM SSL symlinks need the wheel's hashed crypto library at load time.
117+
ENV LD_LIBRARY_PATH="${SQLBOT_HOME}/app/.venv/lib/python3.11/site-packages/dmpython.libs:$LD_LIBRARY_PATH"
78118

79119
ENV POSTGRES_DB=sqlbot
80120
ENV POSTGRES_USER=root
@@ -83,14 +123,23 @@ ENV POSTGRES_PASSWORD=Password123@pg
83123
# Copy necessary files from builder
84124
COPY start.sh /opt/sqlbot/app/start.sh
85125
COPY g2-ssr/*.ttf /usr/share/fonts/truetype/liberation/
86-
COPY --from=sqlbot-builder ${SQLBOT_HOME} ${SQLBOT_HOME}
87-
COPY --from=ssr-builder /app /opt/sqlbot/g2-ssr
126+
COPY --from=sqlbot-builder /opt/sqlbot/app/ /opt/sqlbot/app/
127+
COPY --from=sqlbot-ui-builder /tmp/frontend/dist/ /opt/sqlbot/frontend/dist/
128+
COPY --from=ssr-builder /app/ /opt/sqlbot/g2-ssr/
88129
COPY g2-ssr/supervisord.conf /etc/supervisor/conf.d/g2-ssr.conf
89130
COPY --from=vector-model /opt/maxkb/app/model /opt/sqlbot/models
90131

91132
WORKDIR ${SQLBOT_HOME}/app
92133

93-
RUN mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs
134+
RUN set -eu; \
135+
mkdir -p /opt/sqlbot/images /opt/sqlbot/g2-ssr/logs; \
136+
dmssl="$SQLBOT_HOME/app/.venv/lib/python3.11/site-packages/dmssl"; \
137+
for library in libssl.so libcrypto.so; do \
138+
test -f "$dmssl/$library"; \
139+
ln -s "$dmssl/$library" "$DM_HOME/bin/$library"; \
140+
done; \
141+
python -c 'import ctypes, os, ssl; import dmPython, psycopg, psycopg2, pymssql; assert psycopg.pq.__impl__ == "c"; ctypes.CDLL(os.environ["DM_HOME"] + "/bin/libssl.so")'; \
142+
node -e "require('/opt/sqlbot/g2-ssr/node_modules/@antv/g2-ssr')"
94143

95144
EXPOSE 3000 8000 8001 5432
96145

0 commit comments

Comments
 (0)