diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..66cf13a --- /dev/null +++ b/.env.example @@ -0,0 +1,43 @@ +# Docker Compose overrides. +# +# Copy to .env in the repo root (see docker-compose.example.yml for full setup): +# +# cp docker-compose.example.yml docker-compose.yml +# cp .env.example .env +# cp server/.env.example server/.env +# cp client/.env.example client/.env +# +# Base secrets and app config are loaded from server/.env and client/.env. +# Values here override those files for container networking. + +# PostgreSQL on the host machine. Copy DATABASE_URL from server/.env and replace +# localhost with host.docker.internal so containers can reach the host. +DATABASE_URL=postgresql://postgres:pass@host.docker.internal:5432/postgres + +# Host ports published by docker-compose (see SERVER_PORT / FRONTEND_PORT below). +FRONTEND_URL=http://localhost:8081 +VITE_API_URL=http://localhost:8080 + +# Map the container's pptruser to your host user so bind-mounted ./server and +# ./common stay writable in both directions. When the container runs pnpm install +# or writes build artifacts, those files appear on your host with your UID/GID. +# docker-compose.yml defaults to 1000; override if id -u / id -g differ (common on macOS). +DEV_UID=1000 +DEV_GID=1000 + +SERVER_PORT=8080 +FRONTEND_PORT=8081 + +# Node.js debugger (Chrome DevTools / VS Code attach). Unset DEBUG to disable. +# DEBUG=1 — inspect on server and worker +# DEBUG=server — inspect on API server only +# DEBUG=worker — inspect on worker only +# DEBUG=server,worker — inspect on both (same as 1) +# SERVER_DBG_PORT / WORKER_DBG_PORT — ports Node listens on inside each container (defaults 9229 / 9230). +# To attach the debugger from localhost, uncomment the matching inspect port lines in docker-compose.yml. +# Optional host-side ports (defaults match SERVER_DBG_PORT / WORKER_DBG_PORT): +# SERVER_DBG_PUBLISH_PORT=9229 +# WORKER_DBG_PUBLISH_PORT=9230 + +# Redis is internal-only (xtra/worker use redis://redis:6379). +# To expose on the host, add to docker-compose.yml: ports: ["6380:6379"] diff --git a/.gitignore b/.gitignore index 88b1012..a8f7598 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ lerna-debug.log* !.env.example !**/.env.example +# Local Docker Compose (copy from docker-compose.example.yml) +docker-compose.yml + # Build/test output dist/ dist-ssr/ @@ -61,3 +64,4 @@ override.tf.json *_override.tf.json .terraformrc terraform.rc +.pnpm-store diff --git a/client/Dockerfile.dev b/client/Dockerfile.dev new file mode 100644 index 0000000..994b6db --- /dev/null +++ b/client/Dockerfile.dev @@ -0,0 +1,12 @@ +FROM node:20-alpine + +RUN npm install -g pnpm + +WORKDIR /app + +COPY client/package.json client/pnpm-lock.yaml client/pnpm-workspace.yaml ./ +RUN pnpm install + +EXPOSE 80 + +CMD ["pnpm", "exec", "vite", "--host", "0.0.0.0", "--port", "80"] diff --git a/docker-compose.example.yml b/docker-compose.example.yml new file mode 100644 index 0000000..338563a --- /dev/null +++ b/docker-compose.example.yml @@ -0,0 +1,148 @@ +# Local Docker Compose template. +# +# Copy and customize (both files are gitignored once copied): +# +# cp docker-compose.example.yml docker-compose.yml +# cp .env.example .env +# cp server/.env.example server/.env +# cp client/.env.example client/.env +# +# Edit .env for host ports, DATABASE_URL, and DEV_UID/DEV_GID. Then: +# +# docker compose up --build +# +# API: http://localhost:8080 Frontend: http://localhost:8081 (defaults; see .env) + +name: xtra + +services: + redis: + image: redis:7-alpine + container_name: xtra_redis + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + xtra: + container_name: xtra + platform: linux/amd64 + build: + context: . + dockerfile: server/Dockerfile.dev + platforms: + - linux/amd64 + args: + DEV_UID: ${DEV_UID:-1000} + DEV_GID: ${DEV_GID:-1000} + cap_add: + - NET_BIND_SERVICE + security_opt: + - seccomp:unconfined + env_file: + - server/.env + - .env + environment: + DEV_UID: ${DEV_UID:-1000} + DEV_GID: ${DEV_GID:-1000} + DEBUG: ${DEBUG:-} + DEBUG_SERVICE: server + SERVER_DBG_PORT: ${SERVER_DBG_PORT:-9229} + PORT: "80" + REDIS_URL: redis://redis:6379 + EXTRACTION_FILES_PATH: /data/extractions + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./server:/app + - ./common:/common + - server-node-modules-amd64:/app/node_modules + - extraction-data:/data/extractions + - puppeteer-cache:/home/pptruser/.cache/puppeteer + ports: + - "${SERVER_PORT:-8080}:80" + # Optional: publish Node inspect to localhost + # - "${SERVER_DBG_PUBLISH_PORT:-${SERVER_DBG_PORT:-9229}}:${SERVER_DBG_PORT:-9229}" + # Advanced: use expose instead of ports if you reach services by container IP/DNS + # on the Docker network (not localhost). Omit ports and uncomment: + # expose: + # - "80" + # - "${SERVER_DBG_PORT:-9229}" + depends_on: + redis: + condition: service_healthy + command: ["pnpm", "run", "dev"] + + worker: + container_name: xtra_worker + platform: linux/amd64 + build: + context: . + dockerfile: worker.Dockerfile + platforms: + - linux/amd64 + entrypoint: ["sh", "/app/docker-dev-entrypoint.sh"] + security_opt: + - seccomp:unconfined + env_file: + - server/.env + - .env + environment: + DEV_UID: ${DEV_UID:-1000} + DEV_GID: ${DEV_GID:-1000} + DEBUG: ${DEBUG:-} + DEBUG_SERVICE: worker + WORKER_DBG_PORT: ${WORKER_DBG_PORT:-9230} + REDIS_URL: redis://redis:6379 + EXTRACTION_FILES_PATH: /data/extractions + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./server:/app + - ./common:/common + - server-node-modules-amd64:/app/node_modules + - extraction-data:/data/extractions + - puppeteer-cache:/home/pptruser/.cache/puppeteer + # Optional: publish Node inspect to localhost + # ports: + # - "${WORKER_DBG_PUBLISH_PORT:-${WORKER_DBG_PORT:-9230}}:${WORKER_DBG_PORT:-9230}" + # Advanced: expose inspect port on the container network only + # expose: + # - "${WORKER_DBG_PORT:-9230}" + depends_on: + redis: + condition: service_healthy + command: ["pnpm", "exec", "tsx", "src/worker.ts"] + + xtra_fe: + container_name: xtra_fe + build: + context: . + dockerfile: client/Dockerfile.dev + env_file: + - client/.env + - .env + environment: + VITE_API_URL: ${VITE_API_URL:-http://localhost:8080} + volumes: + - ./client:/app + - ./common:/common + - client-node-modules:/app/node_modules + ports: + - "${FRONTEND_PORT:-8081}:80" + # Advanced: use expose instead of ports if you reach the frontend by container IP/DNS + # expose: + # - "80" + depends_on: + - xtra + command: ["pnpm", "exec", "vite", "--host", "0.0.0.0", "--port", "80"] + +volumes: + redis-data: + extraction-data: + server-node-modules-amd64: + client-node-modules: + puppeteer-cache: diff --git a/server/Dockerfile.dev b/server/Dockerfile.dev new file mode 100644 index 0000000..e251f9c --- /dev/null +++ b/server/Dockerfile.dev @@ -0,0 +1,42 @@ +# Dev API server image — same Puppeteer/Chrome setup as worker.Dockerfile; +# source is bind-mounted and built at container start via docker-dev-entrypoint.sh. +FROM --platform=linux/amd64 node:20 + +ARG DEV_UID=1000 +ARG DEV_GID=1000 + +ENV LANG=en_US.UTF-8 + +RUN apt-get update \ + && apt-get install -y wget gnupg \ + && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \ + && sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] https://dl-ssl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf libxss1 dbus dbus-x11 \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -r pptruser && useradd -rm -g pptruser -G audio,video pptruser + +RUN usermod -u "${DEV_UID}" pptruser \ + && usermod -g "${DEV_GID}" pptruser \ + && chown -R pptruser:pptruser /home/pptruser + +RUN npm install pm2 -g +RUN npm install pnpm -g + +COPY server/package.json server/pnpm-lock.yaml server/pnpm-workspace.yaml /build/app/server/ +RUN cd /build/app/server && pnpm install + +USER pptruser +RUN /build/app/server/node_modules/.bin/rebrowser-puppeteer browsers install chrome +RUN /build/app/server/node_modules/.bin/rebrowser-puppeteer browsers install chrome-headless-shell + +USER root +RUN mkdir -p /app \ + && cp -R /build/app/server/node_modules /app/node_modules \ + && cp /build/app/server/package.json /build/app/server/pnpm-lock.yaml /build/app/server/pnpm-workspace.yaml /app/ + +WORKDIR /app + +ENTRYPOINT ["sh", "/app/docker-dev-entrypoint.sh"] +CMD ["pnpm", "run", "dev"] diff --git a/server/docker-dev-entrypoint.sh b/server/docker-dev-entrypoint.sh new file mode 100755 index 0000000..133f3cf --- /dev/null +++ b/server/docker-dev-entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -e + +DEV_UID="${DEV_UID:-1000}" +DEV_GID="${DEV_GID:-1000}" + +# Enable Node.js inspector when DEBUG matches this service (DEBUG_SERVICE=server|worker). +# DEBUG=1 (or true/yes/all) attaches all services; DEBUG=server,worker attaches listed ones only. +debug_inspect_enabled() { + [ -n "${DEBUG:-}" ] || return 1 + [ -n "${DEBUG_SERVICE:-}" ] || return 1 + + case "$DEBUG" in + 1|true|yes|all) return 0 ;; + esac + + _old_ifs=$IFS + IFS=',' + for _svc in $DEBUG; do + _svc=$(echo "$_svc" | tr -d ' ' | tr '[:upper:]' '[:lower:]') + case "$_svc" in + server|xtra) + [ "$DEBUG_SERVICE" = "server" ] && { IFS=$_old_ifs; return 0; } + ;; + worker) + [ "$DEBUG_SERVICE" = "worker" ] && { IFS=$_old_ifs; return 0; } + ;; + esac + done + IFS=$_old_ifs + return 1 +} + +setup_node_inspect() { + debug_inspect_enabled || return 0 + + case "$DEBUG_SERVICE" in + server) INSPECT_PORT="${SERVER_DBG_PORT:-9229}" ;; + worker) INSPECT_PORT="${WORKER_DBG_PORT:-9230}" ;; + *) return 0 ;; + esac + + export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--inspect=0.0.0.0:${INSPECT_PORT}" + echo "Node inspect enabled for ${DEBUG_SERVICE} on 0.0.0.0:${INSPECT_PORT}" +} + +# worker.Dockerfile uses the default pptruser uid; align with xtra/DEV_UID for shared volumes. +usermod -o -u "$DEV_UID" pptruser +usermod -g "$DEV_GID" pptruser +chown -R pptruser: /home/pptruser + +chown -R pptruser:pptruser /app +chown -R pptruser:pptruser /app/db 2>/dev/null || true + +mkdir -p /data/extractions /app/dist /home/pptruser/.cache/puppeteer +chown -R pptruser: /data/extractions +chown -R pptruser: /home/pptruser/.cache + +# xtra and worker share node_modules; serialize install/build to avoid races. +( + flock -w 600 9 || exit 1 + runuser -u pptruser -- env CI=true pnpm install + runuser -u pptruser -- pnpm run build + chown -R pptruser:pptruser /app/node_modules /app/dist +) 9>/tmp/pnpm-install.lock + +setup_node_inspect + +if [ -n "${NODE_OPTIONS:-}" ]; then + exec runuser -u pptruser -- env NODE_OPTIONS="$NODE_OPTIONS" "$@" +else + exec runuser -u pptruser -- "$@" +fi diff --git a/server/src/extraction/browser.ts b/server/src/extraction/browser.ts index 450f95f..b40fc27 100644 --- a/server/src/extraction/browser.ts +++ b/server/src/extraction/browser.ts @@ -99,6 +99,7 @@ export async function getCluster(proxyUrl?: string) { ignoreHTTPSErrors: true, dumpio: true, // Pipe Chrome process stdout/stderr to console when DEBUG=1 args: [ + "--disable-dev-shm-usage", "--font-render-hinting=none", "--force-gpu-mem-available-mb=4096", "--ignore-certificate-errors",