Add Docker based developer environment setup to better match the architecture

.env.example

@@ -0,0 +1,43 @@
+# Docker Compose overrides.
+#
+# Copy to .env in the repo root (see docker-compose.example.yml for full setup):
+#
+#   cp docker-compose.example.yml docker-compose.yml
+#   cp .env.example .env
+#   cp server/.env.example server/.env
+#   cp client/.env.example client/.env
+#
+# Base secrets and app config are loaded from server/.env and client/.env.
+# Values here override those files for container networking.
+
+# PostgreSQL on the host machine. Copy DATABASE_URL from server/.env and replace
+# localhost with host.docker.internal so containers can reach the host.
+DATABASE_URL=postgresql://postgres:pass@host.docker.internal:5432/postgres
+
+# Host ports published by docker-compose (see SERVER_PORT / FRONTEND_PORT below).
+FRONTEND_URL=http://localhost:8081
+VITE_API_URL=http://localhost:8080
+
+# Map the container's pptruser to your host user so bind-mounted ./server and
+# ./common stay writable in both directions. When the container runs pnpm install
+# or writes build artifacts, those files appear on your host with your UID/GID.
+# docker-compose.yml defaults to 1000; override if id -u / id -g differ (common on macOS).
+DEV_UID=1000
+DEV_GID=1000
+
+SERVER_PORT=8080
+FRONTEND_PORT=8081
+
+# Node.js debugger (Chrome DevTools / VS Code attach). Unset DEBUG to disable.
+# DEBUG=1            — inspect on server and worker
+# DEBUG=server       — inspect on API server only
+# DEBUG=worker       — inspect on worker only
+# DEBUG=server,worker — inspect on both (same as 1)
+# SERVER_DBG_PORT / WORKER_DBG_PORT — ports Node listens on inside each container (defaults 9229 / 9230).
+# To attach the debugger from localhost, uncomment the matching inspect port lines in docker-compose.yml.
+# Optional host-side ports (defaults match SERVER_DBG_PORT / WORKER_DBG_PORT):
+# SERVER_DBG_PUBLISH_PORT=9229
+# WORKER_DBG_PUBLISH_PORT=9230
+
+# Redis is internal-only (xtra/worker use redis://redis:6379).
+# To expose on the host, add to docker-compose.yml: ports: ["6380:6379"]

.gitignore

@@ -34,6 +34,9 @@ lerna-debug.log*
 !.env.example
 !**/.env.example
 
+# Local Docker Compose (copy from docker-compose.example.yml)
+docker-compose.yml
+
 # Build/test output
 dist/
 dist-ssr/
@@ -61,3 +64,4 @@ override.tf.json
 *_override.tf.json
 .terraformrc
 terraform.rc
+.pnpm-store

client/Dockerfile.dev

@@ -0,0 +1,12 @@
+FROM node:20-alpine
+
+RUN npm install -g pnpm
+
+WORKDIR /app
+
+COPY client/package.json client/pnpm-lock.yaml client/pnpm-workspace.yaml ./
+RUN pnpm install
+
+EXPOSE 80
+
+CMD ["pnpm", "exec", "vite", "--host", "0.0.0.0", "--port", "80"]

docker-compose.example.yml

@@ -0,0 +1,148 @@
+# Local Docker Compose template.
+#
+# Copy and customize (both files are gitignored once copied):
+#
+#   cp docker-compose.example.yml docker-compose.yml
+#   cp .env.example .env
+#   cp server/.env.example server/.env
+#   cp client/.env.example client/.env
+#
+# Edit .env for host ports, DATABASE_URL, and DEV_UID/DEV_GID. Then:
+#
+#   docker compose up --build
+#
+# API: http://localhost:8080  Frontend: http://localhost:8081  (defaults; see .env)
+
+name: xtra
+
+services:
+  redis:
+    image: redis:7-alpine
+    container_name: xtra_redis
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  xtra:
+    container_name: xtra
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: server/Dockerfile.dev
+      platforms:
+        - linux/amd64
+      args:
+        DEV_UID: ${DEV_UID:-1000}
+        DEV_GID: ${DEV_GID:-1000}
+    cap_add:
+      - NET_BIND_SERVICE
+    security_opt:
+      - seccomp:unconfined
+    env_file:
+      - server/.env
+      - .env
+    environment:
+      DEV_UID: ${DEV_UID:-1000}
+      DEV_GID: ${DEV_GID:-1000}
+      DEBUG: ${DEBUG:-}
+      DEBUG_SERVICE: server
+      SERVER_DBG_PORT: ${SERVER_DBG_PORT:-9229}
+      PORT: "80"
+      REDIS_URL: redis://redis:6379
+      EXTRACTION_FILES_PATH: /data/extractions
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./server:/app
+      - ./common:/common
+      - server-node-modules-amd64:/app/node_modules
+      - extraction-data:/data/extractions
+      - puppeteer-cache:/home/pptruser/.cache/puppeteer
+    ports:
+      - "${SERVER_PORT:-8080}:80"
+      # Optional: publish Node inspect to localhost
+      # - "${SERVER_DBG_PUBLISH_PORT:-${SERVER_DBG_PORT:-9229}}:${SERVER_DBG_PORT:-9229}"
+    # Advanced: use expose instead of ports if you reach services by container IP/DNS
+    # on the Docker network (not localhost). Omit ports and uncomment:
+    # expose:
+    #   - "80"
+    #   - "${SERVER_DBG_PORT:-9229}"
+    depends_on:
+      redis:
+        condition: service_healthy
+    command: ["pnpm", "run", "dev"]
+
+  worker:
+    container_name: xtra_worker
+    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: worker.Dockerfile
+      platforms:
+        - linux/amd64
+    entrypoint: ["sh", "/app/docker-dev-entrypoint.sh"]
+    security_opt:
+      - seccomp:unconfined
+    env_file:
+      - server/.env
+      - .env
+    environment:
+      DEV_UID: ${DEV_UID:-1000}
+      DEV_GID: ${DEV_GID:-1000}
+      DEBUG: ${DEBUG:-}
+      DEBUG_SERVICE: worker
+      WORKER_DBG_PORT: ${WORKER_DBG_PORT:-9230}
+      REDIS_URL: redis://redis:6379
+      EXTRACTION_FILES_PATH: /data/extractions
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./server:/app
+      - ./common:/common
+      - server-node-modules-amd64:/app/node_modules
+      - extraction-data:/data/extractions
+      - puppeteer-cache:/home/pptruser/.cache/puppeteer
+    # Optional: publish Node inspect to localhost
+    # ports:
+    #   - "${WORKER_DBG_PUBLISH_PORT:-${WORKER_DBG_PORT:-9230}}:${WORKER_DBG_PORT:-9230}"
+    # Advanced: expose inspect port on the container network only
+    # expose:
+    #   - "${WORKER_DBG_PORT:-9230}"
+    depends_on:
+      redis:
+        condition: service_healthy
+    command: ["pnpm", "exec", "tsx", "src/worker.ts"]
+
+  xtra_fe:
+    container_name: xtra_fe
+    build:
+      context: .
+      dockerfile: client/Dockerfile.dev
+    env_file:
+      - client/.env
+      - .env
+    environment:
+      VITE_API_URL: ${VITE_API_URL:-http://localhost:8080}
+    volumes:
+      - ./client:/app
+      - ./common:/common
+      - client-node-modules:/app/node_modules
+    ports:
+      - "${FRONTEND_PORT:-8081}:80"
+    # Advanced: use expose instead of ports if you reach the frontend by container IP/DNS
+    # expose:
+    #   - "80"
+    depends_on:
+      - xtra
+    command: ["pnpm", "exec", "vite", "--host", "0.0.0.0", "--port", "80"]
+
+volumes:
+  redis-data:
+  extraction-data:
+  server-node-modules-amd64:
+  client-node-modules:
+  puppeteer-cache:

server/Dockerfile.dev

@@ -0,0 +1,42 @@
+# Dev API server image — same Puppeteer/Chrome setup as worker.Dockerfile;
+# source is bind-mounted and built at container start via docker-dev-entrypoint.sh.
+FROM --platform=linux/amd64 node:20
+
+ARG DEV_UID=1000
+ARG DEV_GID=1000
+
+ENV LANG=en_US.UTF-8
+
+RUN apt-get update \
+  && apt-get install -y wget gnupg \
+  && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \
+  && sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] https://dl-ssl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-freefont-ttf libxss1 dbus dbus-x11 \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* \
+  && groupadd -r pptruser && useradd -rm -g pptruser -G audio,video pptruser
+
+RUN usermod -u "${DEV_UID}" pptruser \
+  && usermod -g "${DEV_GID}" pptruser \
+  && chown -R pptruser:pptruser /home/pptruser
+
+RUN npm install pm2 -g
+RUN npm install pnpm -g
+
+COPY server/package.json server/pnpm-lock.yaml server/pnpm-workspace.yaml /build/app/server/
+RUN cd /build/app/server && pnpm install
+
+USER pptruser
+RUN /build/app/server/node_modules/.bin/rebrowser-puppeteer browsers install chrome
+RUN /build/app/server/node_modules/.bin/rebrowser-puppeteer browsers install chrome-headless-shell
+
+USER root
+RUN mkdir -p /app \
+  && cp -R /build/app/server/node_modules /app/node_modules \
+  && cp /build/app/server/package.json /build/app/server/pnpm-lock.yaml /build/app/server/pnpm-workspace.yaml /app/
+
+WORKDIR /app
+
+ENTRYPOINT ["sh", "/app/docker-dev-entrypoint.sh"]
+CMD ["pnpm", "run", "dev"]

server/docker-dev-entrypoint.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+set -e
+
+DEV_UID="${DEV_UID:-1000}"
+DEV_GID="${DEV_GID:-1000}"
+
+# Enable Node.js inspector when DEBUG matches this service (DEBUG_SERVICE=server|worker).
+# DEBUG=1 (or true/yes/all) attaches all services; DEBUG=server,worker attaches listed ones only.
+debug_inspect_enabled() {
+  [ -n "${DEBUG:-}" ] || return 1
+  [ -n "${DEBUG_SERVICE:-}" ] || return 1
+
+  case "$DEBUG" in
+    1|true|yes|all) return 0 ;;
+  esac
+
+  _old_ifs=$IFS
+  IFS=','
+  for _svc in $DEBUG; do
+    _svc=$(echo "$_svc" | tr -d ' ' | tr '[:upper:]' '[:lower:]')
+    case "$_svc" in
+      server|xtra)
+        [ "$DEBUG_SERVICE" = "server" ] && { IFS=$_old_ifs; return 0; }
+        ;;
+      worker)
+        [ "$DEBUG_SERVICE" = "worker" ] && { IFS=$_old_ifs; return 0; }
+        ;;
+    esac
+  done
+  IFS=$_old_ifs
+  return 1
+}
+
+setup_node_inspect() {
+  debug_inspect_enabled || return 0
+
+  case "$DEBUG_SERVICE" in
+    server) INSPECT_PORT="${SERVER_DBG_PORT:-9229}" ;;
+    worker) INSPECT_PORT="${WORKER_DBG_PORT:-9230}" ;;
+    *) return 0 ;;
+  esac
+
+  export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--inspect=0.0.0.0:${INSPECT_PORT}"
+  echo "Node inspect enabled for ${DEBUG_SERVICE} on 0.0.0.0:${INSPECT_PORT}"
+}
+
+# worker.Dockerfile uses the default pptruser uid; align with xtra/DEV_UID for shared volumes.
+usermod -o -u "$DEV_UID" pptruser
+usermod -g "$DEV_GID" pptruser
+chown -R pptruser: /home/pptruser
+
+chown -R pptruser:pptruser /app
+chown -R pptruser:pptruser /app/db 2>/dev/null || true
+
+mkdir -p /data/extractions /app/dist /home/pptruser/.cache/puppeteer
+chown -R pptruser: /data/extractions
+chown -R pptruser: /home/pptruser/.cache
+
+# xtra and worker share node_modules; serialize install/build to avoid races.
+(
+  flock -w 600 9 || exit 1
+  runuser -u pptruser -- env CI=true pnpm install
+  runuser -u pptruser -- pnpm run build
+  chown -R pptruser:pptruser /app/node_modules /app/dist
+) 9>/tmp/pnpm-install.lock
+
+setup_node_inspect
+
+if [ -n "${NODE_OPTIONS:-}" ]; then
+  exec runuser -u pptruser -- env NODE_OPTIONS="$NODE_OPTIONS" "$@"
+else
+  exec runuser -u pptruser -- "$@"
+fi

server/src/extraction/browser.ts

@@ -99,6 +99,7 @@ export async function getCluster(proxyUrl?: string) {
       ignoreHTTPSErrors: true,
       dumpio: true, // Pipe Chrome process stdout/stderr to console when DEBUG=1
       args: [
+        "--disable-dev-shm-usage",
         "--font-render-hinting=none",
         "--force-gpu-mem-available-mb=4096",
         "--ignore-certificate-errors",