From 539483a6e890864028b471cf3b65a2c7ba5132be Mon Sep 17 00:00:00 2001 From: Alaeddin <15094821+BSalaeddin@users.noreply.github.com> Date: Mon, 18 May 2026 21:39:52 +0100 Subject: [PATCH 1/3] feat(queues): per-target deployment queues with configurable concurrency and cancellation --- apps/dokploy/esbuild.config.ts | 12 +- .../dokploy/server/api/routers/application.ts | 82 ++-- apps/dokploy/server/api/routers/compose.ts | 84 ++-- .../server/api/routers/preview-deployment.ts | 5 +- apps/dokploy/server/api/routers/server.ts | 41 ++ apps/dokploy/server/api/routers/settings.ts | 40 +- .../server/queues/deployments-queue.ts | 360 ++++++++++++++---- apps/dokploy/server/queues/queue-routing.ts | 16 + apps/dokploy/server/queues/queueSetup.ts | 351 +++++++++++++---- apps/dokploy/server/server.ts | 46 ++- .../server/src/utils/process/execAsync.ts | 98 +++-- .../server/src/utils/process/job-context.ts | 138 +++++++ 12 files changed, 1023 insertions(+), 250 deletions(-) create mode 100644 apps/dokploy/server/queues/queue-routing.ts create mode 100644 packages/server/src/utils/process/job-context.ts diff --git a/apps/dokploy/esbuild.config.ts b/apps/dokploy/esbuild.config.ts index fa053d726c..bee31c5c61 100644 --- a/apps/dokploy/esbuild.config.ts +++ b/apps/dokploy/esbuild.config.ts @@ -3,12 +3,20 @@ import esbuild from "esbuild"; const result = dotenv.config({ path: ".env.production" }); +// Keys whose value must be read from process.env at runtime, not baked in at +// build time. Add a key here when an operator should be able to tune it from +// docker-compose env / systemd unit / shell without rebuilding the bundle. +const RUNTIME_ONLY_ENV_KEYS = new Set([ + "DATABASE_URL", + "DEPLOYMENT_QUEUE_CONCURRENCY", + "DEPLOYMENT_SHUTDOWN_GRACE_MS", +]); + function prepareDefine(config: DotenvParseOutput | undefined) { const define = {}; // @ts-ignore for (const [key, value] of Object.entries(config)) { - // Skip DATABASE_URL to allow runtime environment variable override - if (key === "DATABASE_URL") { + if (RUNTIME_ONLY_ENV_KEYS.has(key)) { continue; } // @ts-ignore diff --git a/apps/dokploy/server/api/routers/application.ts b/apps/dokploy/server/api/routers/application.ts index 228c986567..ea8b53dc40 100644 --- a/apps/dokploy/server/api/routers/application.ts +++ b/apps/dokploy/server/api/routers/application.ts @@ -70,9 +70,9 @@ import { import { deploymentWorker } from "@/server/queues/deployments-queue"; import type { DeploymentJob } from "@/server/queues/queue-types"; import { + cancelDeploymentsByApplication, cleanQueuesByApplication, getJobsByApplicationId, - killDockerBuild, myQueue, } from "@/server/queues/queueSetup"; import { cancelDeployment, deploy } from "@/server/utils/deploy"; @@ -338,8 +338,11 @@ export const applicationRouter = createTRPCRouter({ server: !!application.serverId, }; - if (IS_CLOUD && application.serverId) { + if (application.serverId) { jobData.serverId = application.serverId; + } + + if (IS_CLOUD && application.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); @@ -705,8 +708,11 @@ export const applicationRouter = createTRPCRouter({ applicationType: "application", server: !!application.serverId, }; - if (IS_CLOUD && application.serverId) { + if (application.serverId) { jobData.serverId = application.serverId; + } + + if (IS_CLOUD && application.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); @@ -765,7 +771,7 @@ export const applicationRouter = createTRPCRouter({ deployment: ["cancel"], }); const application = await findApplicationById(input.applicationId); - await killDockerBuild("application", application.serverId); + await cancelDeploymentsByApplication(input.applicationId); await audit(ctx, { action: "stop", resourceType: "application", @@ -824,8 +830,11 @@ export const applicationRouter = createTRPCRouter({ applicationType: "application", server: !!app.serverId, }; - if (IS_CLOUD && app.serverId) { + if (app.serverId) { jobData.serverId = app.serverId; + } + + if (IS_CLOUD && app.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); @@ -929,46 +938,43 @@ export const applicationRouter = createTRPCRouter({ }); const application = await findApplicationById(input.applicationId); - if (IS_CLOUD && application.serverId) { - try { - await updateApplicationStatus(input.applicationId, "idle"); - - if (application.deployments[0]) { - await updateDeploymentStatus( - application.deployments[0].deploymentId, - "done", - ); - } + try { + await updateApplicationStatus(input.applicationId, "idle"); + if (application.deployments[0]) { + await updateDeploymentStatus( + application.deployments[0].deploymentId, + "error", + ); + } + if (IS_CLOUD && application.serverId) { await cancelDeployment({ applicationId: input.applicationId, applicationType: "application", }); - await audit(ctx, { - action: "stop", - resourceType: "application", - resourceId: application.applicationId, - resourceName: application.appName, - }); - return { - success: true, - message: "Deployment cancellation requested", - }; - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error - ? error.message - : "Failed to cancel deployment", - }); + } else { + await cancelDeploymentsByApplication(input.applicationId); } - } - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Deployment cancellation only available in cloud version", - }); + await audit(ctx, { + action: "stop", + resourceType: "application", + resourceId: application.applicationId, + resourceName: application.appName, + }); + return { + success: true, + message: "Deployment cancellation requested", + }; + } catch (error) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + error instanceof Error + ? error.message + : "Failed to cancel deployment", + }); + } }), search: protectedProcedure diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index d1fbb14d3a..a8f1fb98de 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -71,9 +71,9 @@ import { import { deploymentWorker } from "@/server/queues/deployments-queue"; import type { DeploymentJob } from "@/server/queues/queue-types"; import { + cancelDeploymentsByCompose, cleanQueuesByCompose, getJobsByComposeId, - killDockerBuild, myQueue, } from "@/server/queues/queueSetup"; import { cancelDeployment, deploy } from "@/server/utils/deploy"; @@ -309,8 +309,8 @@ export const composeRouter = createTRPCRouter({ await checkServicePermissionAndAccess(ctx, input.composeId, { deployment: ["cancel"], }); - const compose = await findComposeById(input.composeId); - await killDockerBuild("compose", compose.serverId); + await findComposeById(input.composeId); + await cancelDeploymentsByCompose(input.composeId); }), loadServices: protectedProcedure @@ -430,8 +430,11 @@ export const composeRouter = createTRPCRouter({ server: !!compose.serverId, }; - if (IS_CLOUD && compose.serverId) { + if (compose.serverId) { jobData.serverId = compose.serverId; + } + + if (IS_CLOUD && compose.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); @@ -478,8 +481,11 @@ export const composeRouter = createTRPCRouter({ descriptionLog: input.description || "", server: !!compose.serverId, }; - if (IS_CLOUD && compose.serverId) { + if (compose.serverId) { jobData.serverId = compose.serverId; + } + + if (IS_CLOUD && compose.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); @@ -1052,49 +1058,45 @@ export const composeRouter = createTRPCRouter({ }); const compose = await findComposeById(input.composeId); - if (IS_CLOUD && compose.serverId) { - try { - await updateCompose(input.composeId, { - composeStatus: "idle", - }); - - if (compose.deployments[0]) { - await updateDeploymentStatus( - compose.deployments[0].deploymentId, - "done", - ); - } + try { + await updateCompose(input.composeId, { + composeStatus: "idle", + }); + if (compose.deployments[0]) { + await updateDeploymentStatus( + compose.deployments[0].deploymentId, + "error", + ); + } + if (IS_CLOUD && compose.serverId) { await cancelDeployment({ composeId: input.composeId, applicationType: "compose", }); - - await audit(ctx, { - action: "stop", - resourceType: "compose", - resourceId: input.composeId, - resourceName: compose.name, - }); - return { - success: true, - message: "Deployment cancellation requested", - }; - } catch (error) { - throw new TRPCError({ - code: "INTERNAL_SERVER_ERROR", - message: - error instanceof Error - ? error.message - : "Failed to cancel deployment", - }); + } else { + await cancelDeploymentsByCompose(input.composeId); } - } - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Deployment cancellation only available in cloud version", - }); + await audit(ctx, { + action: "stop", + resourceType: "compose", + resourceId: input.composeId, + resourceName: compose.name, + }); + return { + success: true, + message: "Deployment cancellation requested", + }; + } catch (error) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + error instanceof Error + ? error.message + : "Failed to cancel deployment", + }); + } }), search: protectedProcedure diff --git a/apps/dokploy/server/api/routers/preview-deployment.ts b/apps/dokploy/server/api/routers/preview-deployment.ts index a45ef80c53..fbe7a03a06 100644 --- a/apps/dokploy/server/api/routers/preview-deployment.ts +++ b/apps/dokploy/server/api/routers/preview-deployment.ts @@ -88,8 +88,11 @@ export const previewDeploymentRouter = createTRPCRouter({ server: !!application.serverId, }; - if (IS_CLOUD && application.serverId) { + if (application.serverId) { jobData.serverId = application.serverId; + } + + if (IS_CLOUD && application.serverId) { deploy(jobData).catch((error) => { console.error("Background deployment failed:", error); }); diff --git a/apps/dokploy/server/api/routers/server.ts b/apps/dokploy/server/api/routers/server.ts index 310363efdd..409ed93b7a 100644 --- a/apps/dokploy/server/api/routers/server.ts +++ b/apps/dokploy/server/api/routers/server.ts @@ -34,6 +34,7 @@ import { apiFindOneServer, apiRemoveServer, apiUpdateServer, + apiUpdateServerDeploymentConcurrency, apiUpdateServerMonitoring, applications, compose, @@ -45,6 +46,7 @@ import { redis, server, } from "@/server/db/schema"; +import { deploymentQueueManager } from "@/server/queues/queueSetup"; export const serverRouter = createTRPCRouter({ create: withPermission("server", "create") @@ -467,6 +469,45 @@ export const serverRouter = createTRPCRouter({ throw error; } }), + updateDeploymentConcurrency: withPermission("server", "create") + .input(apiUpdateServerDeploymentConcurrency) + .mutation(async ({ input, ctx }) => { + const target = await findServerById(input.serverId); + if (target.organizationId !== ctx.session.activeOrganizationId) { + throw new TRPCError({ + code: "UNAUTHORIZED", + message: "You are not authorized to update this server", + }); + } + + const updated = await updateServerById(input.serverId, { + deploymentConcurrency: input.deploymentConcurrency, + }); + + // Apply immediately to the in-process queue. In-flight jobs keep + // running; pending jobs stay queued. + if (!IS_CLOUD) { + try { + await deploymentQueueManager.updateConcurrency( + input.serverId, + input.deploymentConcurrency, + ); + } catch (error) { + console.error( + "Failed to propagate concurrency change to queue manager", + error, + ); + } + } + + await audit(ctx, { + action: "update", + resourceType: "server", + resourceId: input.serverId, + resourceName: target.name, + }); + return updated; + }), publicIp: protectedProcedure.query(async () => { if (IS_CLOUD) { return ""; diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index 9965643f72..0d1b37ca22 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -67,10 +67,15 @@ import { apiServerSchema, apiTraefikConfig, apiUpdateDockerCleanup, + apiUpdateWebServerDeploymentConcurrency, projects, server, } from "@/server/db/schema"; -import { cleanAllDeploymentQueue } from "@/server/queues/queueSetup"; +import { + cleanAllDeploymentQueue, + deploymentQueueManager, + LOCAL_TARGET, +} from "@/server/queues/queueSetup"; import { removeJob, schedule } from "@/server/utils/backup"; import packageInfo from "../../../package.json"; import { appRouter } from "../root"; @@ -360,6 +365,39 @@ export const settingsRouter = createTRPCRouter({ }); return true; }), + /** + * Local counterpart of `server.updateDeploymentConcurrency`: sets how many + * deployments can run in parallel on the Dokploy host itself. Persisted + * on `webServerSettings` and applied to the in-process queue on the same + * tick — no rebuild or service restart needed. + */ + updateDeploymentConcurrency: adminProcedure + .input(apiUpdateWebServerDeploymentConcurrency) + .mutation(async ({ input, ctx }) => { + if (IS_CLOUD) { + return true; + } + await updateWebServerSettings({ + deploymentConcurrency: input.deploymentConcurrency, + }); + try { + await deploymentQueueManager.updateConcurrency( + LOCAL_TARGET, + input.deploymentConcurrency, + ); + } catch (error) { + console.error( + "Failed to propagate local concurrency change to queue manager", + error, + ); + } + await audit(ctx, { + action: "update", + resourceType: "settings", + resourceName: "deployment-concurrency", + }); + return true; + }), updateDockerCleanup: adminProcedure .input(apiUpdateDockerCleanup) .mutation(async ({ input, ctx }) => { diff --git a/apps/dokploy/server/queues/deployments-queue.ts b/apps/dokploy/server/queues/deployments-queue.ts index e92f4c1921..dd68d8821b 100644 --- a/apps/dokploy/server/queues/deployments-queue.ts +++ b/apps/dokploy/server/queues/deployments-queue.ts @@ -10,87 +10,319 @@ import { updateCompose, updatePreviewDeployment, } from "@dokploy/server"; +import { + dokployJobContext, + killJobProcesses, +} from "@dokploy/server/utils/process/job-context"; import { type Job, Worker } from "bullmq"; +import { + CANCEL_CHANNEL, + getQueueName, + getTargetKey, + LOCAL_TARGET, +} from "./queue-routing"; import type { DeploymentJob } from "./queue-types"; import { redisConfig } from "./redis-connection"; -const createDeploymentWorker = () => - new Worker( - "deployments", - async (job: Job) => { - try { - if (job.data.applicationType === "application") { - await updateApplicationStatus(job.data.applicationId, "running"); +export { CANCEL_CHANNEL, getQueueName, getTargetKey, LOCAL_TARGET }; + +const MAX_CONCURRENCY = 10; +const DEFAULT_CONCURRENCY = 1; + +const clampConcurrency = (n: number): number => + Math.max(1, Math.min(Math.floor(n) || DEFAULT_CONCURRENCY, MAX_CONCURRENCY)); + +// Pin queue runtime state to globalThis. Without this, Next.js loading this +// module from both the workspace package and its own runtime produces two +// `workers` Maps, each spinning up its own BullMQ Worker for the same queue. +// Effective concurrency then becomes (concurrency × moduleCopies), and cancel +// can target a job tracked in the other copy's `inflight`. Same reasoning as +// `job-context.ts`'s child-registry pinning. +const QUEUE_STATE_KEY = Symbol.for("dokploy.deploymentQueue.state"); +type QueueState = { + workers: Map>; + inflight: Map; + cancelSubscriber: { quit: () => Promise } | null; + started: boolean; +}; +type GlobalShared = typeof globalThis & { + [QUEUE_STATE_KEY]?: QueueState; +}; +const g = globalThis as GlobalShared; +const queueState: QueueState = + g[QUEUE_STATE_KEY] ?? + (g[QUEUE_STATE_KEY] = { + workers: new Map>(), + inflight: new Map(), + cancelSubscriber: null, + started: false, + }); +const workers = queueState.workers; +const inflight = queueState.inflight; + +const killJobOnAbort = (jobId: string): void => { + if (!jobId) return; + try { + killJobProcesses(jobId); + } catch (error) { + console.error("[deployments] failed to kill job", jobId, error); + } +}; + +const handleJob = async ( + job: Job, + signal: AbortSignal, +): Promise => { + const data = job.data; + const jobId = job.id ?? ""; + const serverId = data.serverId ?? null; + + // On abort, kill *only* this job's process tree — never a global + // `pkill -f "docker build"` (which would kill unrelated concurrent + // builds on the same host). + const onAbort = () => { + killJobOnAbort(jobId); + }; + signal.addEventListener("abort", onAbort, { once: true }); - if (job.data.type === "redeploy") { - await rebuildApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "deploy") { - await deployApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } - } else if (job.data.applicationType === "compose") { - await updateCompose(job.data.composeId, { - composeStatus: "running", + try { + await dokployJobContext.run({ jobId, serverId }, async () => { + if (data.applicationType === "application") { + await updateApplicationStatus(data.applicationId, "running"); + if (data.type === "redeploy") { + await rebuildApplication({ + applicationId: data.applicationId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, }); - if (job.data.type === "deploy") { - await deployCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } else if (job.data.type === "redeploy") { - await rebuildCompose({ - composeId: job.data.composeId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - }); - } - } else if (job.data.applicationType === "application-preview") { - await updatePreviewDeployment(job.data.previewDeploymentId, { - previewStatus: "running", + } else if (data.type === "deploy") { + await deployApplication({ + applicationId: data.applicationId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, + }); + } + } else if (data.applicationType === "compose") { + await updateCompose(data.composeId, { composeStatus: "running" }); + if (data.type === "deploy") { + await deployCompose({ + composeId: data.composeId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, + }); + } else if (data.type === "redeploy") { + await rebuildCompose({ + composeId: data.composeId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, + }); + } + } else if (data.applicationType === "application-preview") { + await updatePreviewDeployment(data.previewDeploymentId, { + previewStatus: "running", + }); + if (data.type === "redeploy") { + await rebuildPreviewApplication({ + applicationId: data.applicationId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, + previewDeploymentId: data.previewDeploymentId, + }); + } else if (data.type === "deploy") { + await deployPreviewApplication({ + applicationId: data.applicationId, + titleLog: data.titleLog, + descriptionLog: data.descriptionLog, + previewDeploymentId: data.previewDeploymentId, }); - - if (job.data.type === "redeploy") { - await rebuildPreviewApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - previewDeploymentId: job.data.previewDeploymentId, - }); - } else if (job.data.type === "deploy") { - await deployPreviewApplication({ - applicationId: job.data.applicationId, - titleLog: job.data.titleLog, - descriptionLog: job.data.descriptionLog, - previewDeploymentId: job.data.previewDeploymentId, - }); - } } - } catch (error) { - console.log("Error", error); + } + }); + } catch (error) { + if (signal.aborted) { + try { + if (data.applicationType === "application") { + await updateApplicationStatus(data.applicationId, "error"); + } else if (data.applicationType === "compose") { + await updateCompose(data.composeId, { composeStatus: "error" }); + } else if (data.applicationType === "application-preview") { + await updatePreviewDeployment(data.previewDeploymentId, { + previewStatus: "error", + }); + } + } catch {} + throw new Error("Deployment aborted by user"); + } + throw error; + } finally { + signal.removeEventListener("abort", onAbort); + } + if (signal.aborted) { + throw new Error("Deployment aborted by user"); + } +}; + +const createWorker = ( + targetKey: string, + concurrency: number, +): Worker => { + const worker = new Worker( + getQueueName(targetKey), + async (job) => { + const ac = new AbortController(); + const key = job.id ?? `${job.queueQualifiedName}:${job.timestamp}`; + inflight.set(key, ac); + try { + await handleJob(job, ac.signal); + } finally { + inflight.delete(key); } }, { autorun: false, + concurrency: clampConcurrency(concurrency), connection: redisConfig, }, ); + worker.on("error", (err) => { + if ((err as { code?: string })?.code === "ECONNREFUSED") { + console.error( + "Make sure you have installed Redis and it is running.", + err, + ); + } + }); + workers.set(targetKey, worker); + return worker; +}; + +/** + * BullMQ's `Worker.run()` only resolves when the worker is closed (it awaits + * the internal main loop). We can't `await` it during boot or every subsequent + * step (bootstrap, shutdown handlers) hangs forever. Fire-and-forget here. + */ +const startIfStopped = (w: Worker): void => { + const running = (w as { isRunning?: () => boolean }).isRunning?.(); + if (running) return; + void w.run().catch((err) => { + console.error("[deployments] worker.run() error", err); + }); +}; + +const ensureWorker = ( + targetKey: string, + concurrency: number = DEFAULT_CONCURRENCY, +): Worker => { + const existing = workers.get(targetKey); + if (existing) return existing; + const worker = createWorker(targetKey, concurrency); + if (queueState.started) startIfStopped(worker); + return worker; +}; + +const startCancelSubscriber = async (): Promise => { + if (queueState.cancelSubscriber) return; + const seedWorker = workers.get(LOCAL_TARGET) ?? createWorker(LOCAL_TARGET, 1); + const client = await seedWorker.client; + const sub = ( + client as unknown as { duplicate: () => unknown } + ).duplicate() as { + subscribe: (channel: string) => Promise; + on: ( + event: "message", + cb: (channel: string, payload: string) => void, + ) => void; + quit: () => Promise; + }; + await sub.subscribe(CANCEL_CHANNEL); + sub.on("message", (_chan, payload) => { + try { + const { jobId } = JSON.parse(payload) as { jobId: string }; + const ac = inflight.get(jobId); + if (ac) ac.abort(); + } catch (err) { + console.error("Cancel subscriber: bad payload", err); + } + }); + queueState.cancelSubscriber = sub; +}; + +const realDeploymentWorker = { + run: async (): Promise => { + queueState.started = true; + ensureWorker(LOCAL_TARGET); + for (const w of workers.values()) startIfStopped(w); + void startCancelSubscriber().catch((err) => { + console.error("[deployments] cancel subscriber failed to start", err); + }); + }, + close: async (_reason?: string): Promise => { + queueState.started = false; + // Stop pulling new jobs and drain in-flight ones gracefully. BullMQ's + // `worker.close()` waits for active handlers to resolve. We deliberately + // do NOT call `ac.abort()` here — that would kill every running deploy + // on a routine Dokploy upgrade (swarm rolling restart sends SIGTERM, + // which calls this path). If the orchestrator follows up with SIGKILL + // after its grace period, the OS handles termination; the spawned + // builds run with `detached:true` so they have their own process group + // and can complete on the host even if our process is gone. + await Promise.all(Array.from(workers.values()).map((w) => w.close())); + workers.clear(); + if (queueState.cancelSubscriber) { + await queueState.cancelSubscriber.quit().catch(() => {}); + queueState.cancelSubscriber = null; + } + }, + cancelJob: async (jobId: string, _reason?: string): Promise => { + const ac = inflight.get(jobId); + if (ac) ac.abort(); + }, + cancelAllJobs: async (_reason?: string): Promise => { + for (const ac of inflight.values()) ac.abort(); + }, + /** + * Live-update a worker's concurrency. The new value applies to the next + * job-fetch decision: BullMQ honours `worker.concurrency` mutation on its + * main loop, so raising the limit lets queued jobs become active + * immediately, and lowering it stops new fetches until in-flight jobs + * drain. **In-flight jobs are NOT interrupted** by this call — they keep + * running to completion. To stop a running job, use `cancelJob`. + */ + setConcurrency: (targetKey: string, concurrency: number): void => { + const safe = clampConcurrency(concurrency); + const worker = ensureWorker(targetKey, safe); + worker.concurrency = safe; + }, + ensureWorker: ( + targetKey: string, + concurrency: number = DEFAULT_CONCURRENCY, + ): void => { + ensureWorker(targetKey, concurrency); + }, + removeWorker: async (targetKey: string): Promise => { + const w = workers.get(targetKey); + if (!w) return; + workers.delete(targetKey); + try { + await w.close(); + } catch (err) { + console.error("[deployments] worker.close error", targetKey, err); + } + }, + hasInflight: (jobId: string): boolean => inflight.has(jobId), +}; -/** No-op worker when Redis is disabled (e.g. IS_CLOUD). Avoids BullMQ connection errors. */ -const noopWorker = { +const noopDeploymentWorker = { run: () => Promise.resolve(), - close: () => Promise.resolve(), - cancelJob: () => Promise.resolve(), - cancelAllJobs: () => Promise.resolve(), + close: (_reason?: string) => Promise.resolve(), + cancelJob: (_jobId: string, _reason?: string) => Promise.resolve(), + cancelAllJobs: (_reason?: string) => Promise.resolve(), + setConcurrency: (_targetKey: string, _concurrency: number) => {}, + ensureWorker: (_targetKey: string, _concurrency?: number) => {}, + removeWorker: (_targetKey: string) => Promise.resolve(), + hasInflight: (_jobId: string) => false, }; export const deploymentWorker = !IS_CLOUD - ? createDeploymentWorker() - : (noopWorker as unknown as Worker); + ? realDeploymentWorker + : noopDeploymentWorker; diff --git a/apps/dokploy/server/queues/queue-routing.ts b/apps/dokploy/server/queues/queue-routing.ts new file mode 100644 index 0000000000..0ee047666f --- /dev/null +++ b/apps/dokploy/server/queues/queue-routing.ts @@ -0,0 +1,16 @@ +import type { DeploymentJob } from "./queue-types"; + +export const LOCAL_TARGET = "__local__"; +export const CANCEL_CHANNEL = "dokploy:deployments:cancel"; + +export const getTargetKey = (data: Pick): string => + data.serverId ?? LOCAL_TARGET; + +/** + * LOCAL_TARGET keeps canary's queue name `deployments` so an upgrade from a + * BullMQ-only canary doesn't strand pending jobs in an orphaned queue. + * Per-server queues use `__` as the separator because BullMQ rejects queue + * names containing `:`. + */ +export const getQueueName = (targetKey: string): string => + targetKey === LOCAL_TARGET ? "deployments" : `deployments__${targetKey}`; diff --git a/apps/dokploy/server/queues/queueSetup.ts b/apps/dokploy/server/queues/queueSetup.ts index 75d59e0795..a1a3761e6f 100644 --- a/apps/dokploy/server/queues/queueSetup.ts +++ b/apps/dokploy/server/queues/queueSetup.ts @@ -1,104 +1,317 @@ -import { IS_CLOUD } from "@dokploy/server"; import { - execAsync, - execAsyncRemote, -} from "@dokploy/server/utils/process/execAsync"; -import type { Job } from "bullmq"; -import { Queue } from "bullmq"; -import { deploymentWorker } from "./deployments-queue"; + findServerById, + getAllServers, + getWebServerSettings, + IS_CLOUD, +} from "@dokploy/server"; +import { type Job, type JobsOptions, type JobType, Queue } from "bullmq"; +import { + CANCEL_CHANNEL, + deploymentWorker, + getQueueName, + getTargetKey, + LOCAL_TARGET, +} from "./deployments-queue"; +import type { DeploymentJob } from "./queue-types"; import { redisConfig } from "./redis-connection"; -/** No-op queue when Redis is disabled (e.g. IS_CLOUD). Avoids BullMQ connection errors. */ -const createNoopQueue = () => ({ - getJobs: () => Promise.resolve([] as Job[]), - add: () => - Promise.resolve({ id: "noop", remove: () => Promise.resolve() } as Job), - close: () => Promise.resolve(), - on: () => {}, -}); +export { LOCAL_TARGET, getTargetKey }; -const myQueue = !IS_CLOUD - ? new Queue("deployments", { connection: redisConfig }) - : (createNoopQueue() as unknown as Queue); +// Pin to globalThis: see deployments-queue.ts QUEUE_STATE_KEY for rationale. +// Without pinning, Next.js loading this module twice produces two `queues` +// Maps + double the Redis connections per target. +const QUEUES_KEY = Symbol.for("dokploy.deploymentQueue.queues"); +const queues: Map> = ( + globalThis as { [QUEUES_KEY]?: Map> } +)[QUEUES_KEY] ?? +((globalThis as { [QUEUES_KEY]?: Map> })[ + QUEUES_KEY +] = new Map>()); -export const getJobsByApplicationId = async (applicationId: string) => { - const jobs = await myQueue.getJobs(); - return jobs.filter((job) => job?.data?.applicationId === applicationId); +const createNoopQueue = () => + ({ + getJobs: () => Promise.resolve([] as Job[]), + getJob: () => Promise.resolve(undefined), + add: () => + Promise.resolve({ id: "noop", remove: () => Promise.resolve() } as Job), + client: Promise.resolve(undefined as unknown as never), + close: () => Promise.resolve(), + on: () => {}, + }) as unknown as Queue; + +const getOrCreateQueue = (targetKey: string): Queue => { + let q = queues.get(targetKey); + if (q) return q; + if (IS_CLOUD) { + q = createNoopQueue(); + } else { + q = new Queue(getQueueName(targetKey), { + connection: redisConfig, + }); + q.on("error", (err) => { + if ((err as { code?: string })?.code === "ECONNREFUSED") { + console.error( + "Make sure you have installed Redis and it is running.", + err, + ); + } + }); + } + queues.set(targetKey, q); + return q; }; -export const getJobsByComposeId = async (composeId: string) => { - const jobs = await myQueue.getJobs(); - return jobs.filter((job) => job?.data?.composeId === composeId); +const getAllQueues = (): Queue[] => Array.from(queues.values()); + +const aggregateJobs = async (states?: JobType[]): Promise => { + const all: Job[] = []; + for (const q of getAllQueues()) { + all.push(...((await q.getJobs(states)) as Job[])); + } + return all; +}; + +/** + * Look up the persisted deploymentConcurrency for a target. Falls back to 1 + * when the row is missing (e.g. a server was deleted between enqueue and + * worker spin-up). Errors are swallowed — concurrency is a tuning knob, not + * a correctness boundary. + */ +const resolveConcurrencyForTarget = async ( + targetKey: string, +): Promise => { + if (IS_CLOUD) return 1; + try { + if (targetKey === LOCAL_TARGET) { + const settings = await getWebServerSettings(); + return settings?.deploymentConcurrency ?? 1; + } + const server = await findServerById(targetKey); + return server?.deploymentConcurrency ?? 1; + } catch { + return 1; + } +}; + +const ensureWorkerForTarget = async (targetKey: string): Promise => { + if (IS_CLOUD) return; + const concurrency = await resolveConcurrencyForTarget(targetKey); + deploymentWorker.ensureWorker(targetKey, concurrency); + deploymentWorker.setConcurrency(targetKey, concurrency); }; -if (!IS_CLOUD) { - process.on("SIGTERM", () => { - myQueue.close(); - process.exit(0); - }); +/** + * Legacy facade preserved for webhook callers (pages/api/deploy/*) that still + * `myQueue.add(...)`. Routes by `serverId` to the correct per-target queue; + * `getJobs` aggregates across all known per-target queues. + */ +export const myQueue = { + add: async (name: string, data: DeploymentJob, opts?: JobsOptions) => { + const targetKey = getTargetKey(data); + const q = getOrCreateQueue(targetKey); + await ensureWorkerForTarget(targetKey); + return q.add(name, data, opts); + }, + getJobs: (states?: JobType[]) => aggregateJobs(states), + close: async () => { + await Promise.all(getAllQueues().map((q) => q.close())); + queues.clear(); + }, + on: () => {}, +} as unknown as Queue; + +const publishCancel = async ( + jobId: string, + targetKey: string, +): Promise => { + const q = getOrCreateQueue(targetKey); + const client = await (q as unknown as { client: Promise }).client; + const publisher = client as { + publish: (channel: string, payload: string) => Promise; + }; + await publisher.publish(CANCEL_CHANNEL, JSON.stringify({ jobId, targetKey })); +}; - myQueue.on("error", (error) => { - if ((error as any).code === "ECONNREFUSED") { +export const deploymentQueueManager = { + enqueue: async (data: DeploymentJob, opts?: JobsOptions) => { + const targetKey = getTargetKey(data); + const q = getOrCreateQueue(targetKey); + await ensureWorkerForTarget(targetKey); + return q.add("deploy", data, opts); + }, + updateConcurrency: async ( + targetKey: string, + concurrency: number, + ): Promise => { + if (IS_CLOUD) return; + deploymentWorker.setConcurrency(targetKey, concurrency); + }, + cancel: async ( + jobId: string, + targetKey: string, + ): Promise<{ canceled: boolean; wasActive: boolean }> => { + if (IS_CLOUD) return { canceled: false, wasActive: false }; + const q = queues.get(targetKey); + if (q) { + const job = await q.getJob(jobId); + if (job) { + const state = await job.getState(); + if (state === "waiting" || state === "delayed") { + await job.remove(); + return { canceled: true, wasActive: false }; + } + } + } + await deploymentWorker.cancelJob(jobId); + await publishCancel(jobId, targetKey); + return { canceled: true, wasActive: true }; + }, + /** + * Drop the in-process Worker + Queue for `targetKey` (called when a + * server is deleted). Cancels any active/pending jobs so cleanup work + * stops, then closes and removes the BullMQ handles. Without this the + * Worker keeps running forever pointed at an orphaned Redis queue. + */ + removeTarget: async (targetKey: string): Promise => { + if (IS_CLOUD) return; + if (targetKey === LOCAL_TARGET) return; + const q = queues.get(targetKey); + if (q) { + try { + const jobs = await q.getJobs([ + "active", + "waiting", + "delayed", + "prioritized", + ]); + for (const job of jobs) { + if (!job?.id) continue; + try { + await deploymentQueueManager.cancel(job.id, targetKey); + } catch {} + } + await q.close(); + } catch (error) { + console.error( + "[deployments] failed to drain queue for removed target", + targetKey, + error, + ); + } + queues.delete(targetKey); + } + try { + await deploymentWorker.removeWorker(targetKey); + } catch (error) { console.error( - "Make sure you have installed Redis and it is running.", + "[deployments] failed to close worker for removed target", + targetKey, error, ); } - }); -} + }, + /** + * Pre-warm queues + workers at boot. LOCAL is always warmed. Each server + * is also warmed so that any jobs persisted in `deployments:` + * Redis queues from a previous process get picked up by their target's + * worker — without this, per-server queues stay orphaned until the user + * triggers a fresh deploy on that server. + */ + bootstrap: async (): Promise => { + if (IS_CLOUD) return; + getOrCreateQueue(LOCAL_TARGET); + await ensureWorkerForTarget(LOCAL_TARGET); + try { + const servers = await getAllServers(); + for (const s of servers) { + if (!s?.serverId) continue; + getOrCreateQueue(s.serverId); + await ensureWorkerForTarget(s.serverId); + } + console.log( + `Deployment queue ready (LOCAL + ${servers.length} server queue(s))`, + ); + } catch (error) { + console.error("Failed to pre-warm per-server deployment workers", error); + } + }, +}; -export const cleanQueuesByApplication = async (applicationId: string) => { - const jobs = await myQueue.getJobs(["waiting", "delayed"]); +export const getJobsByApplicationId = async (applicationId: string) => { + const jobs = await aggregateJobs(); + return jobs.filter((job) => job?.data?.applicationId === applicationId); +}; + +export const getJobsByComposeId = async (composeId: string) => { + const jobs = await aggregateJobs(); + return jobs.filter((job) => job?.data?.composeId === composeId); +}; +export const cleanQueuesByApplication = async (applicationId: string) => { + const jobs = await aggregateJobs(["waiting", "delayed"]); for (const job of jobs) { if (job?.data?.applicationId === applicationId) { await job.remove(); - console.log(`Removed job ${job.id} for application ${applicationId}`); } } }; -export const cleanAllDeploymentQueue = async () => { - deploymentWorker.cancelAllJobs("User requested cancellation"); - return true; -}; - export const cleanQueuesByCompose = async (composeId: string) => { - const jobs = await myQueue.getJobs(["waiting", "delayed"]); - + const jobs = await aggregateJobs(["waiting", "delayed"]); for (const job of jobs) { if (job?.data?.composeId === composeId) { await job.remove(); - console.log(`Removed job ${job.id} for compose ${composeId}`); } } }; -export const killDockerBuild = async ( - type: "application" | "compose", - serverId: string | null, -) => { - try { - if (type === "application") { - const command = `pkill -2 -f "docker build"`; - - if (serverId) { - await execAsyncRemote(serverId, command); - } else { - await execAsync(command); - } - } else if (type === "compose") { - const command = `pkill -2 -f "docker compose"`; +/** + * Self-hosted cancel: abort any in-flight job for this application AND drop + * pending ones. The worker's abort listener tears down only this job's + * process group (LOCAL) or SSH session (REMOTE), so concurrent builds for + * other resources on the same host are untouched. + */ +export const cancelDeploymentsByApplication = async (applicationId: string) => { + const jobs = await aggregateJobs([ + "active", + "waiting", + "delayed", + "prioritized", + ]); + let canceled = 0; + for (const job of jobs) { + if (job?.data?.applicationId !== applicationId) continue; + if (!job.id) continue; + const targetKey = getTargetKey(job.data); + await deploymentQueueManager.cancel(job.id, targetKey); + canceled++; + } + return canceled; +}; - if (serverId) { - await execAsyncRemote(serverId, command); - } else { - await execAsync(command); - } - } - } catch (error) { - console.error(error); +export const cancelDeploymentsByCompose = async (composeId: string) => { + const jobs = await aggregateJobs([ + "active", + "waiting", + "delayed", + "prioritized", + ]); + let canceled = 0; + for (const job of jobs) { + if (job?.data?.composeId !== composeId) continue; + if (!job.id) continue; + const targetKey = getTargetKey(job.data); + await deploymentQueueManager.cancel(job.id, targetKey); + canceled++; } + return canceled; }; -export { myQueue }; +export const cleanAllDeploymentQueue = async () => { + await deploymentWorker.cancelAllJobs("User requested cancellation"); + for (const q of getAllQueues()) { + const jobs = await q.getJobs(["waiting", "delayed"]); + for (const job of jobs) await job.remove(); + } + return true; +}; diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 4de4d76897..efa60e1754 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -16,6 +16,7 @@ import { import { config } from "dotenv"; import next from "next"; import packageInfo from "../package.json"; +import { deploymentWorker } from "./queues/deployments-queue"; import { setupDockerContainerLogsWebSocketServer } from "./wss/docker-container-logs"; import { setupDockerContainerTerminalWebSocketServer } from "./wss/docker-container-terminal"; import { setupDockerStatsMonitoringSocketServer } from "./wss/docker-stats"; @@ -71,8 +72,51 @@ void app.prepare().then(async () => { if (!IS_CLOUD) { console.log("Starting Deployment Worker"); - const { deploymentWorker } = await import("./queues/deployments-queue"); await deploymentWorker.run(); + const { deploymentQueueManager } = await import("./queues/queueSetup"); + await deploymentQueueManager.bootstrap(); + + // Graceful shutdown: abort in-flight deployments so no rows are left + // stuck in "running". `initCancelDeployments` sweeps on the *next* + // boot — this closes the current process's window cleanly. + // + // We bound the drain to DEPLOYMENT_SHUTDOWN_GRACE_MS so a build that + // ignores its abort signal can't prevent the process from exiting + // (systemd / Docker will SIGKILL us after their own grace window; + // we want to exit before that and let initCancelDeployments pick up + // anything still marked running). + const DRAIN_GRACE_MS = Number.parseInt( + process.env.DEPLOYMENT_SHUTDOWN_GRACE_MS ?? "8000", + 10, + ); + const shutdown = async (signal: string) => { + console.log(`Received ${signal}, draining deployment queue…`); + try { + await Promise.race([ + deploymentWorker.close(`${signal} received`), + new Promise((_, reject) => + setTimeout( + () => reject(new Error("drain timeout")), + Number.isFinite(DRAIN_GRACE_MS) + ? Math.max(1000, DRAIN_GRACE_MS) + : 8000, + ).unref?.(), + ), + ]); + } catch (error) { + console.error( + "Deployment drain exceeded grace window or failed; exiting anyway.", + error, + ); + } + process.exit(0); + }; + process.once("SIGTERM", () => { + void shutdown("SIGTERM"); + }); + process.once("SIGINT", () => { + void shutdown("SIGINT"); + }); } } catch (e) { console.error("Main Server Error", e); diff --git a/packages/server/src/utils/process/execAsync.ts b/packages/server/src/utils/process/execAsync.ts index d44e3ccaf8..39de6c9f2c 100644 --- a/packages/server/src/utils/process/execAsync.ts +++ b/packages/server/src/utils/process/execAsync.ts @@ -1,43 +1,66 @@ import { exec, execFile } from "node:child_process"; -import util from "node:util"; import { findServerById } from "@dokploy/server/services/server"; import { Client } from "ssh2"; import { ExecError } from "./ExecError"; +import { + getCurrentJob, + jobMarker, + trackLocalChild, + trackSshClient, +} from "./job-context"; // Re-export ExecError for easier imports export { ExecError } from "./ExecError"; -const execAsyncBase = util.promisify(exec); +/** + * When we're inside a deployment job, prepend a marker (`: DOKPLOY_JOB_ID=…`) + * so the spawned shell's argv carries an identifier we can grep for, and + * spawn the child detached (own process group) so cancel can `kill -PGID` + * the entire build tree — shell, docker compose, docker build — in one shot. + */ +const tagCommand = (command: string): string => { + const ctx = getCurrentJob(); + if (!ctx) return command; + return `: ${jobMarker(ctx.jobId)};\n${command}`; +}; -export const execAsync = async ( +export const execAsync = ( command: string, options?: { cwd?: string; env?: NodeJS.ProcessEnv; shell?: string }, ): Promise<{ stdout: string; stderr: string }> => { - try { - const result = await execAsyncBase(command, options); - return { - stdout: result.stdout.toString(), - stderr: result.stderr.toString(), - }; - } catch (error) { - if (error instanceof Error) { - // @ts-ignore - exec error has these properties - const exitCode = error.code; - // @ts-ignore - const stdout = error.stdout?.toString() || ""; - // @ts-ignore - const stderr = error.stderr?.toString() || ""; - - throw new ExecError(`Command execution failed: ${error.message}`, { - command, - stdout, - stderr, - exitCode, - originalError: error, - }); - } - throw error; - } + const ctx = getCurrentJob(); + const tagged = tagCommand(command); + return new Promise((resolve, reject) => { + const child = exec( + tagged, + { + ...options, + // Own process group so cancel can `kill -PGID` the whole tree. + detached: !!ctx, + } as Parameters[1], + (error, stdout, stderr) => { + if (error) { + const codeRaw = (error as { code?: unknown }).code; + const exitCode = typeof codeRaw === "number" ? codeRaw : undefined; + reject( + new ExecError(`Command execution failed: ${error.message}`, { + command, + stdout: typeof stdout === "string" ? stdout : stdout.toString(), + stderr: typeof stderr === "string" ? stderr : stderr.toString(), + exitCode, + originalError: error, + }), + ); + return; + } + resolve({ + stdout: typeof stdout === "string" ? stdout : stdout.toString(), + stderr: typeof stderr === "string" ? stderr : stderr.toString(), + }); + }, + ); + if (ctx) trackLocalChild(ctx.jobId, child); + }); }; interface ExecOptions { @@ -61,8 +84,7 @@ export const execAsyncStream = ( command, stdout: stdoutComplete, stderr: stderrComplete, - // @ts-ignore - exitCode: error.code, + exitCode: (error as { code?: number }).code, originalError: error, }), ); @@ -88,7 +110,7 @@ export const execAsyncStream = ( }); childProcess.on("error", (error) => { - console.log(error); + console.error("execAsyncStream error", error); reject( new ExecError(`Command execution error: ${error.message}`, { command, @@ -150,13 +172,15 @@ export const execAsyncRemote = async ( let stdout = ""; let stderr = ""; + const tagged = tagCommand(command); + const ctx = getCurrentJob(); return new Promise((resolve, reject) => { const conn = new Client(); + if (ctx) trackSshClient(ctx.jobId, conn); - sleep(1000); conn .once("ready", () => { - conn.exec(command, (err, stream) => { + conn.exec(tagged, (err, stream) => { if (err) { onData?.(err.message); reject( @@ -188,6 +212,14 @@ export const execAsyncRemote = async ( ); } }) + .on("error", (err: Error) => { + reject( + new ExecError( + `Remote stream error: ${err.message}`, + { command, stdout, stderr, serverId, originalError: err }, + ), + ); + }) .on("data", (data: string) => { stdout += data.toString(); onData?.(data.toString()); diff --git a/packages/server/src/utils/process/job-context.ts b/packages/server/src/utils/process/job-context.ts new file mode 100644 index 0000000000..758f104f18 --- /dev/null +++ b/packages/server/src/utils/process/job-context.ts @@ -0,0 +1,138 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import type { ChildProcess } from "node:child_process"; +import type { Client } from "ssh2"; + +/** + * Per-deployment job context that flows through every async call + * (deployApplication → getBuildCommand → execAsync / execAsyncRemote) + * via Node's AsyncLocalStorage. + * + * NOTE on the `Symbol.for(...)` singleton dance below: the dokploy app + * bundles `@dokploy/server` as an *external* dependency, so this file + * gets loaded twice in production — once for the app's own code, once + * for the workspace package. Without pinning the instances on + * `globalThis`, the AsyncLocalStorage / child registries would be two + * separate copies, and context set in the worker handler would be + * invisible to `execAsync`. Cancel would then have nothing to kill. + */ +export interface JobContext { + jobId: string; + /** null = LOCAL (Dokploy host); otherwise the remote serverId. */ + serverId: string | null; +} + +const STORE_KEY = Symbol.for("dokploy.jobContext.store"); +const LOCAL_KEY = Symbol.for("dokploy.jobContext.localChildren"); +const REMOTE_KEY = Symbol.for("dokploy.jobContext.remoteSshClients"); + +type GlobalShared = typeof globalThis & { + [STORE_KEY]?: AsyncLocalStorage; + [LOCAL_KEY]?: Map>; + [REMOTE_KEY]?: Map>; +}; + +const g = globalThis as GlobalShared; + +export const dokployJobContext: AsyncLocalStorage = + g[STORE_KEY] ?? (g[STORE_KEY] = new AsyncLocalStorage()); + +const localChildren: Map> = g[LOCAL_KEY] ?? +(g[LOCAL_KEY] = new Map()); + +const remoteSshClients: Map> = g[REMOTE_KEY] ?? +(g[REMOTE_KEY] = new Map()); + +export const getCurrentJob = (): JobContext | undefined => + dokployJobContext.getStore(); + +/** Marker injected into the deploy command line for `pkill -f` matching. */ +export const jobMarker = (jobId: string): string => `DOKPLOY_JOB_ID=${jobId}`; + +export const trackLocalChild = (jobId: string, child: ChildProcess): void => { + let set = localChildren.get(jobId); + if (!set) { + set = new Set(); + localChildren.set(jobId, set); + } + set.add(child); + const cleanup = () => { + const s = localChildren.get(jobId); + s?.delete(child); + if (s && s.size === 0) localChildren.delete(jobId); + }; + child.once("exit", cleanup); + child.once("close", cleanup); +}; + +export const trackSshClient = (jobId: string, client: Client): void => { + let set = remoteSshClients.get(jobId); + if (!set) { + set = new Set(); + remoteSshClients.set(jobId, set); + } + set.add(client); + const cleanup = () => { + const s = remoteSshClients.get(jobId); + s?.delete(client); + if (s && s.size === 0) remoteSshClients.delete(jobId); + }; + client.once("end", cleanup); + client.once("close", cleanup); +}; + +/** + * Kill every process this job spawned. + * + * LOCAL: SIGKILL the process group (PGID = child.pid; the shell was + * spawned with `detached: true`), tearing down sh, `docker compose` + * and `docker build` in one shot. + * + * REMOTE: destroy the ssh client (force-close the socket); the + * stream error triggers promise rejection in execAsyncRemote, + * propagating cancellation through the handler. + * + * Limitation: BuildKit RUN-step containers are owned by dockerd and + * live outside our process group, so an in-flight RUN step runs to + * natural completion before the build aborts. Sub-second RUN-step + * cancel requires BuildKit's gRPC Cancel API with session-ID + * tracking, which is out of scope. + */ +export const killJobProcesses = ( + jobId: string, +): { local: number; remote: number } => { + let local = 0; + let remote = 0; + const localSet = localChildren.get(jobId); + const sshSet = remoteSshClients.get(jobId); + if (localSet) { + for (const child of localSet) { + if (!child.pid) continue; + try { + // Try the process group first (works when the child was + // spawned with detached:true so PGID = child.pid). If the + // group doesn't exist (ESRCH), fall back to killing the + // shell's PID — that closes its stdio, which docker + // compose / docker build CLIs treat as a cancel signal. + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + local++; + } catch { + // Process already gone — nothing more to do. + } + } + localChildren.delete(jobId); + } + if (sshSet) { + for (const client of sshSet) { + try { + client.destroy(); + remote++; + } catch {} + } + remoteSshClients.delete(jobId); + } + return { local, remote }; +}; From 5e761426cd332ecce02b5da2b7b6de65a26509b2 Mon Sep 17 00:00:00 2001 From: Alaeddin <15094821+BSalaeddin@users.noreply.github.com> Date: Mon, 18 May 2026 21:40:06 +0100 Subject: [PATCH 2/3] feat(ui): deployment concurrency settings and self-hosted cancel controls --- .../deployments/show-deployments.tsx | 6 +- .../deployments/show-queue-table.tsx | 4 +- .../actions/change-concurrency-modal.tsx | 179 ++++++++++++++++++ .../servers/actions/show-dokploy-actions.tsx | 2 + .../deployment-concurrency-section.tsx | 148 +++++++++++++++ .../settings/servers/setup-server.tsx | 5 + 6 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 apps/dokploy/components/dashboard/settings/servers/actions/change-concurrency-modal.tsx create mode 100644 apps/dokploy/components/dashboard/settings/servers/deployment-concurrency-section.tsx diff --git a/apps/dokploy/components/dashboard/application/deployments/show-deployments.tsx b/apps/dokploy/components/dashboard/application/deployments/show-deployments.tsx index ccf2564b06..b8f3aab4f8 100644 --- a/apps/dokploy/components/dashboard/application/deployments/show-deployments.tsx +++ b/apps/dokploy/components/dashboard/application/deployments/show-deployments.tsx @@ -75,8 +75,6 @@ export const ShowDeployments = ({ }, ); - const { data: isCloud } = api.settings.isCloud.useQuery(); - const { mutateAsync: rollback, isPending: isRollingBack } = api.rollback.rollback.useMutation(); const { mutateAsync: killProcess, isPending: isKillingProcess } = @@ -121,7 +119,7 @@ export const ShowDeployments = ({ // Check for stuck deployment (more than 9 minutes) - only for the most recent deployment const stuckDeployment = useMemo(() => { - if (!isCloud || !deployments || deployments.length === 0) return null; + if (!deployments || deployments.length === 0) return null; const now = Date.now(); const NINE_MINUTES = 10 * 60 * 1000; // 9 minutes in milliseconds @@ -141,7 +139,7 @@ export const ShowDeployments = ({ const elapsed = now - startTime; return elapsed > NINE_MINUTES ? mostRecentDeployment : null; - }, [isCloud, deployments]); + }, [deployments]); useEffect(() => { setUrl(document.location.origin); }, []); diff --git a/apps/dokploy/components/dashboard/deployments/show-queue-table.tsx b/apps/dokploy/components/dashboard/deployments/show-queue-table.tsx index 22b132f166..150636143c 100644 --- a/apps/dokploy/components/dashboard/deployments/show-queue-table.tsx +++ b/apps/dokploy/components/dashboard/deployments/show-queue-table.tsx @@ -71,7 +71,6 @@ export function ShowQueueTable(props: { embedded?: boolean }) { undefined, { refetchInterval: 3000 }, ); - const { data: isCloud } = api.settings.isCloud.useQuery(); const utils = api.useUtils(); const { mutateAsync: cancelApplicationDeployment, @@ -157,8 +156,7 @@ export function ShowQueueTable(props: { embedded?: boolean }) { — )} - {isCloud && - row.state === "active" && + {row.state === "active" && (d?.applicationId != null || d?.composeId != null) && ( + + + + + ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/actions/show-dokploy-actions.tsx b/apps/dokploy/components/dashboard/settings/servers/actions/show-dokploy-actions.tsx index 7d63de210d..3937a4a0d2 100644 --- a/apps/dokploy/components/dashboard/settings/servers/actions/show-dokploy-actions.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/actions/show-dokploy-actions.tsx @@ -14,6 +14,7 @@ import { api } from "@/utils/api"; import { ShowModalLogs } from "../../web-server/show-modal-logs"; import { TerminalModal } from "../../web-server/terminal-modal"; import { GPUSupportModal } from "../gpu-support-modal"; +import { ChangeConcurrencyModal } from "./change-concurrency-modal"; export const ShowDokployActions = () => { const { mutateAsync: reloadServer, isPending } = @@ -61,6 +62,7 @@ export const ShowDokployActions = () => { + { + const serverQuery = api.server.one.useQuery( + { serverId }, + { enabled: !!serverId }, + ); + const update = api.server.updateDeploymentConcurrency.useMutation(); + + const currentConcurrency = serverQuery.data?.deploymentConcurrency; + // Only poll while the section is actually mounted; stops the moment a parent unmounts the tab. + const [isMounted, setIsMounted] = useState(false); + useEffect(() => { + setIsMounted(true); + return () => setIsMounted(false); + }, []); + const queueQuery = api.deployment.queueList.useQuery(undefined, { + enabled: isMounted, + refetchInterval: isMounted ? 3000 : false, + }); + const liveCounts = useMemo(() => { + const snapshots = queueQuery.data ?? []; + let active = 0; + let pending = 0; + for (const s of snapshots) { + const target = s.data.serverId ?? null; + if (target !== serverId) continue; + if (s.state === "active") active++; + else pending++; + } + return { active, pending }; + }, [queueQuery.data, serverId]); + + const [value, setValue] = useState(currentConcurrency ?? 1); + + useEffect(() => { + if (typeof currentConcurrency === "number") { + setValue(currentConcurrency); + } + }, [currentConcurrency]); + + const isDirty = + typeof currentConcurrency === "number" && value !== currentConcurrency; + const isValid = Number.isInteger(value) && value >= MIN && value <= MAX; + + const save = async () => { + if (!isValid) { + toast.error(`Concurrency must be an integer between ${MIN} and ${MAX}`); + return; + } + try { + await update.mutateAsync({ + serverId, + deploymentConcurrency: value, + }); + await serverQuery.refetch(); + toast.success("Deployment concurrency updated"); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Failed to update concurrency", + ); + } + }; + + return ( + + + Deployment Concurrency + + Number of deployments this server may run in parallel. Other servers + are unaffected. + + + +
+ + + Live: {liveCounts.active} active · {liveCounts.pending} queued + +
+ setValue(Number(event.target.value))} + className="max-w-xs" + /> +

+ 1 = one deploy at a time + (safest). 2–{MAX} = run that many + side-by-side; only raise if the host has spare CPU/RAM. +

+ + + + Each concurrent build uses roughly one CPU core and 2 GB of RAM + while active. Saved value applies immediately; in-flight deployments + are not interrupted. + + +
+ +
+ + + ); +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/setup-server.tsx b/apps/dokploy/components/dashboard/settings/servers/setup-server.tsx index 570955fc8e..a5ce326505 100644 --- a/apps/dokploy/components/dashboard/settings/servers/setup-server.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/setup-server.tsx @@ -27,6 +27,7 @@ import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; import { ShowDeployment } from "../../application/deployments/show-deployment"; import { type LogLine, parseLogs } from "../../docker/logs/utils"; +import { DeploymentConcurrencySection } from "./deployment-concurrency-section"; import { EditScript } from "./edit-script"; import { GPUSupport } from "./gpu-support"; import { SecurityAudit } from "./security-audit"; @@ -320,6 +321,10 @@ export const SetupServer = ({ serverId, asButton = false }: Props) => { /> + + {!isBuildServer && ( + + )} From 69ce0df61b22155ba38c7a8f65ef96f30b3c3d05 Mon Sep 17 00:00:00 2001 From: Alaeddin <15094821+BSalaeddin@users.noreply.github.com> Date: Mon, 18 May 2026 21:40:19 +0100 Subject: [PATCH 3/3] feat(schema): add deploymentConcurrency column to server and webServerSettings --- .../__test__/queues/global-state.test.ts | 114 ++++++++++++++++++ apps/dokploy/__test__/queues/routing.test.ts | 43 +++++++ .../server/update-server-config.test.ts | 1 + .../drizzle/0167_concurrent_deployments.sql | 1 + .../0168_local_deployment_concurrency.sql | 1 + apps/dokploy/drizzle/meta/0167_snapshot.json | Bin 0 -> 450954 bytes apps/dokploy/drizzle/meta/_journal.json | 16 ++- packages/server/src/db/schema/server.ts | 6 + .../src/db/schema/web-server-settings.ts | 19 ++- 9 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 apps/dokploy/__test__/queues/global-state.test.ts create mode 100644 apps/dokploy/__test__/queues/routing.test.ts create mode 100644 apps/dokploy/drizzle/0167_concurrent_deployments.sql create mode 100644 apps/dokploy/drizzle/0168_local_deployment_concurrency.sql create mode 100644 apps/dokploy/drizzle/meta/0167_snapshot.json diff --git a/apps/dokploy/__test__/queues/global-state.test.ts b/apps/dokploy/__test__/queues/global-state.test.ts new file mode 100644 index 0000000000..a0ad191c31 --- /dev/null +++ b/apps/dokploy/__test__/queues/global-state.test.ts @@ -0,0 +1,114 @@ +import { afterEach, describe, expect, it } from "vitest"; + +/** + * Regression test for the Next.js double-load concurrency bug. + * + * Symptom: with `deploymentConcurrency=2` set, peak active jobs in BullMQ was 4. + * Root cause: `workers`/`inflight`/`queues` Maps in `deployments-queue.ts` and + * `queueSetup.ts` were module-scoped, not pinned to `globalThis`. Next.js + * loaded each file from both the workspace package AND its own runtime, + * producing two `workers` Maps in the same Node process. Each constructed an + * independent BullMQ Worker against the same queue, doubling effective + * concurrency. + * + * Fix: pin queue runtime state behind `Symbol.for(...)` keys on globalThis, + * matching the pattern already used by `packages/server/src/utils/process/ + * job-context.ts` for the AsyncLocalStorage child registry. + * + * These tests verify the *pattern itself* — the bare module imports trigger + * heavy side-effects (BullMQ, DB barrel) that are hard to mock cleanly in unit + * tests, but the symbol-keyed pinning is a pure pattern we can lock in here. + */ + +const QUEUE_STATE_KEY = Symbol.for("dokploy.deploymentQueue.state"); +const QUEUES_KEY = Symbol.for("dokploy.deploymentQueue.queues"); + +type GlobalAny = typeof globalThis & Record; + +describe("queue runtime state pinning pattern", () => { + afterEach(() => { + delete (globalThis as GlobalAny)[QUEUE_STATE_KEY]; + delete (globalThis as GlobalAny)[QUEUES_KEY]; + }); + + it("Symbol.for() returns the same key across calls (cross-realm sharing primitive)", () => { + const a = Symbol.for("dokploy.deploymentQueue.state"); + const b = Symbol.for("dokploy.deploymentQueue.state"); + expect(a).toBe(b); + // Sanity: a fresh local symbol is NOT shared. + const local = Symbol("dokploy.deploymentQueue.state"); + expect(local).not.toBe(a); + }); + + it("nullish-coalesce assignment (??=) preserves the first writer's object", () => { + // Simulate the deployments-queue.ts pattern: + // const queueState = g[KEY] ?? (g[KEY] = { workers: new Map(), ... }); + const g = globalThis as GlobalAny; + const initial = { workers: new Map(), inflight: new Map() }; + const a = + (g[QUEUE_STATE_KEY] as typeof initial | undefined) ?? + ((g[QUEUE_STATE_KEY] = initial), initial); + const b = + (g[QUEUE_STATE_KEY] as typeof initial | undefined) ?? + ((g[QUEUE_STATE_KEY] = { workers: new Map(), inflight: new Map() }), + g[QUEUE_STATE_KEY] as typeof initial); + expect(a).toBe(b); + expect(a.workers).toBe(b.workers); + }); + + it("a second module load (with cleared cache) reuses globalThis-pinned Maps", () => { + // First "module load" puts state on globalThis. + const moduleA = (() => { + const g = globalThis as GlobalAny; + return ((g[QUEUE_STATE_KEY] as { workers: Map }) ?? + (g[QUEUE_STATE_KEY] = { + workers: new Map(), + inflight: new Map(), + cancelSubscriber: null, + started: false, + })) as { workers: Map }; + })(); + moduleA.workers.set("srv-1", "worker-instance-A"); + + // Second "module load" — different lexical scope, same globalThis. + const moduleB = (() => { + const g = globalThis as GlobalAny; + return ((g[QUEUE_STATE_KEY] as { workers: Map }) ?? + (g[QUEUE_STATE_KEY] = { + workers: new Map(), + inflight: new Map(), + cancelSubscriber: null, + started: false, + })) as { workers: Map }; + })(); + + // CRITICAL: same workers Map across both "loads". + expect(moduleB.workers).toBe(moduleA.workers); + expect(moduleB.workers.get("srv-1")).toBe("worker-instance-A"); + }); + + it("without pinning, two module instances would each have their own Map", () => { + // Sanity: prove the bug shape this fix prevents. + const moduleA = { workers: new Map() }; + const moduleB = { workers: new Map() }; + moduleA.workers.set("srv-1", "worker-A"); + moduleB.workers.set("srv-1", "worker-B"); + // Two separate Maps. BullMQ would construct two Workers for the same + // queue, doubling effective concurrency. + expect(moduleA.workers).not.toBe(moduleB.workers); + expect(moduleA.workers.get("srv-1")).not.toEqual( + moduleB.workers.get("srv-1"), + ); + }); + + it("symbol keys are stable (cross-package Realm-safe)", () => { + // Symbol.for() keys are interned in the global symbol registry, which + // is shared across realms. This is what makes the fix work for + // Next.js's workspace-package double-load: both copies of the module + // resolve to the SAME symbol via Symbol.for(), so they hit the same + // globalThis property. + expect(QUEUE_STATE_KEY).toBe(Symbol.for("dokploy.deploymentQueue.state")); + expect(QUEUES_KEY).toBe(Symbol.for("dokploy.deploymentQueue.queues")); + expect(QUEUE_STATE_KEY).not.toBe(QUEUES_KEY); + }); +}); diff --git a/apps/dokploy/__test__/queues/routing.test.ts b/apps/dokploy/__test__/queues/routing.test.ts new file mode 100644 index 0000000000..cb0e837987 --- /dev/null +++ b/apps/dokploy/__test__/queues/routing.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { + CANCEL_CHANNEL, + getQueueName, + getTargetKey, + LOCAL_TARGET, +} from "../../server/queues/queue-routing"; + +describe("queue routing", () => { + it("routes jobs without a serverId to LOCAL_TARGET", () => { + expect(getTargetKey({ serverId: undefined })).toBe(LOCAL_TARGET); + expect(getTargetKey({} as { serverId?: string })).toBe(LOCAL_TARGET); + }); + + it("routes jobs with a serverId to that server's key", () => { + expect(getTargetKey({ serverId: "srv-abc" })).toBe("srv-abc"); + }); + + it("preserves canary's 'deployments' queue name for LOCAL_TARGET (upgrade-safe)", () => { + expect(getQueueName(LOCAL_TARGET)).toBe("deployments"); + }); + + it("namespaces remote queues per server (uses __ since BullMQ rejects :)", () => { + expect(getQueueName("srv-abc")).toBe("deployments__srv-abc"); + expect(getQueueName("srv-xyz")).toBe("deployments__srv-xyz"); + }); + + it("never produces a queue name containing : (BullMQ requirement)", () => { + expect(getQueueName(LOCAL_TARGET)).not.toContain(":"); + expect(getQueueName("srv-with-colons-disallowed")).not.toContain(":"); + }); + + it("uses a stable cross-process cancel channel name", () => { + expect(CANCEL_CHANNEL).toBe("dokploy:deployments:cancel"); + }); + + it("treats null/undefined serverId identically (no NaN keys)", () => { + expect(getTargetKey({ serverId: undefined })).toBe(LOCAL_TARGET); + expect(getTargetKey({ serverId: null as unknown as undefined })).toBe( + LOCAL_TARGET, + ); + }); +}); diff --git a/apps/dokploy/__test__/traefik/server/update-server-config.test.ts b/apps/dokploy/__test__/traefik/server/update-server-config.test.ts index e07f34ade9..6e87a86ff9 100644 --- a/apps/dokploy/__test__/traefik/server/update-server-config.test.ts +++ b/apps/dokploy/__test__/traefik/server/update-server-config.test.ts @@ -65,6 +65,7 @@ const baseSettings: WebServerSettings = { cleanupCacheApplications: false, cleanupCacheOnCompose: false, cleanupCacheOnPreviews: false, + deploymentConcurrency: 1, createdAt: null, updatedAt: new Date(), }; diff --git a/apps/dokploy/drizzle/0167_concurrent_deployments.sql b/apps/dokploy/drizzle/0167_concurrent_deployments.sql new file mode 100644 index 0000000000..23815932d1 --- /dev/null +++ b/apps/dokploy/drizzle/0167_concurrent_deployments.sql @@ -0,0 +1 @@ +ALTER TABLE "server" ADD COLUMN "deploymentConcurrency" integer DEFAULT 1 NOT NULL; \ No newline at end of file diff --git a/apps/dokploy/drizzle/0168_local_deployment_concurrency.sql b/apps/dokploy/drizzle/0168_local_deployment_concurrency.sql new file mode 100644 index 0000000000..d0839dbda9 --- /dev/null +++ b/apps/dokploy/drizzle/0168_local_deployment_concurrency.sql @@ -0,0 +1 @@ +ALTER TABLE "webServerSettings" ADD COLUMN "deploymentConcurrency" integer DEFAULT 1 NOT NULL; \ No newline at end of file diff --git a/apps/dokploy/drizzle/meta/0167_snapshot.json b/apps/dokploy/drizzle/meta/0167_snapshot.json new file mode 100644 index 0000000000000000000000000000000000000000..60b41a8a8d9e91948b867408329f3b6d36595e4d GIT binary patch literal 450954 zcmeIbTXP*pjxM^MC*u5v4!(Hi#I&a`bK8D%OYX~VkH<%{XJW!z!7wk9*petUN!eC+ z^j|;c;42Wy8xpIKRkdnWv4SCy>ypT}Ze5!xTzI8#Lf4_T8-#y>mef-utYI94k zUv#6r-Tj%qky5{-R%czi+xu1lN5=Bq?tS--E28`&=M>sp}JgZ1pCCA3`ZVY$DJtYHs7$8CLh4YRicpIN<*$)WE^ zrm&v>NLbx`+&6l#Dk)fFFF9;!h+Pp{zuNtF_xbJ%vVCNyDtmhma&<}`1kY&Zu=Bj6 zvA`Cy+D@SxWv^%Ti+u@r8+$}qUjncDbn1DxR$-mG?CgNH6-%dLJwwU4^Xo>z-iP3O z(ux-}8^aLPsXPn>*@C|$1ivO6WBvIJ$ztoSs2^1y!b{Vxbqu~pEY;6H>Jg8|^-*6p z(y)%MNh`zlhj;Ui{=z#$IJghZhl%PK+*?CeqrDE4^L0)09V4=GP3svn9d?QArT?qy zXFef*hn;&#ZC+3-_H~Cgd4JedmVW8f=XXk04Sc=gLp-HEUVR*eMvKfom0(U zb1(Khi*@>28r@rxa*1H1ec3i+h{ixO2n899-4+}hVj}4%zWW%tlD&KU`y@m4r26e= zrp23%AUwgo-Z|D%n?1WvXwk>S5wOW&V+m&GovncX!L2rOpz&>FrwI-0{eDIi5K)up zn>Nb76?o~g1AO39K}_zd8|~M`D|s|1NQh1ak+&;~VUe$nkCo%YGS^unG;Z2O%4Z_TG*J)lw2`v%9Kkait6 zhbL%zt5+?i(8jr<{g4d=><=98hda4{t8O@7sM9W!uQw(55?#p(o zYfp>TL%{Mi@gb~{(~s*kb81^0!KUWH6Z^TP#<`&>#n;dh3Cr7iaGN{+74)ljo#6<~ zX}9xzL$eQ_%#rYG%RW>eF1{jVrxfvKzu{Z@t7TQvaIR7J;Jw^6N<##Zg_eB~d`GYw zQt1$C4+GE3dyg8gR!Dx?-hpi4--{-R=TiVNjfa3;YT*Pf^6Shx zB@d3GW7(s$tb9Ot{Dx;+>Qo*IhI;0X*G|%@EDgi99lD;m<3&)X@-Ps@87pK>AeJPj z;6jW`+Xt|3C--F5==e}vS{wnpTtpH$2Tjh^dA#H$g>@j7bB-2^IYhogh)ar>r1WBr zm-jHFH+$a2Cy{0zAfqGnX?p?Z%6?!kU5UEx&4zY)X2ub*dkC2rTOI_T$__gw;IrjH zAfuY?dfls)>!Umm0(&EzLxt49PDc!@wofSI=z8l?!neGLL!Qb;zT69%Pwbrk63uIC zRBRu;a=wF<=J6bOh~~+mXtTVU7#IHY+blYBF#_8M&LM4*TO_ zo;pNO-cSui$Zl;9S$gQwR;5&Lp1I942fH0v?91}Z$r>Tnig)*sw=ZQQSV~LxVg^5( z7p2g0&G_g$Q7=|H6(4TLv!Bh^c1VM^zT8hQqxS?e=gkdv*HTTzTi}8=58D|AUG(uc)r^Rc8&1 zH&=D~+g$CNt9Z@+;y%I$nYt$5Jx+=b*<<#jz0J|SIg0lmW6x1P+S?q}=LnS&d4GwB z+3@|%K6;+HkQm8#lmlVsWxOLDo!4aKs9o?~Sv_0TeXMpD-MWza6i=w~{S*56eD^;) zjQlhDNuNa9!S|UVR8hU?sxrc67_R5STnCLp?qG4{Omz5;TTYHP`v(q}vZmK3kdhnr7#WBL_Ln{PmFf{6qO%#$wOL5Wbck zBws4mmf~xDww+49zX4nj!SB9jF}7S+z6X3*u4`TE>y~dBt*KKe_0g6Cd&nHvG*iQq z?-W{=b11U9{A$d%UDc14dUv7aGVlqWFl{Z;tar@v3E}gW>Kn+-HXaTNE!U0EIa$q; z;f9-+k?U4FGY}*v$QQIM>_m-<54ol0v%;Rkj%2HNCH%p^>r#octk;BE`>FGif{)jZ z^ZY`t+cX5H_e#|4A$Ta#c`Nr~Z6|ey_h`?4&Ert(*WOv_-Urz;*=>jH%5wB|-xg$V z+n42%1XPZ7U#5+^<1AXKl<<@7JE1P9SW=-O-Ipttev4gwwh@c9YBKfTF{7TO&?@J5+dCg} z?_8g6zWq}7j)(>4nRBLB&Uc_yiw4?j<&=35x+c({0k=j8S9?(lEp^(!X!(8!t~`?By|?iVwHr*?U{#)-#s&*GAH5;uhj}_q%rFP_+6!bSiqEHF=*O|6rNv zx-oy(vp3D#G77a7>dC#ftw+TF(9IUsotE2Y#_HFaVZQ}i_f3X%6kZFgik$ZoSKzhw z3;WjRKpz(S7u7xjht!`u(Bil;uLAqe?uK?@AHi+m!{s~O>XtoPmv+3+w0FaJ=5J|_ zA~#%fuYI1CtJPq4^Pc7evx1!M*B@_*?ni^YhcAieB@T8%m0N(=CH>eI5^lM=+r9kg zuhlWoBo)}yc_v7p*e@Ijo<`crKI;6%n1@>DJX`kOaHwR-_^i1QUW0dTlc6=rP`-v~ zQguS-Y_Pgq5*NH6ZSY6>>jUuv{SrIy}ov_6`AJ(q^--j6iu>xzt>sE^RGHT!YLsJ8@{dD zNuAP~@spUT@Q@ZEGy(fJd4AjKBI#7hxfQLpd<*dh zk^^`vTW-9H+pZZSj%uJL0Bq;hD4b5khui#7IEj>KYp%9x@D5Cm0DG-zoyyV>3{e*! zhI$m&i=j?sZ5ZarYMeP{p0ZAV^WTl(zk_w!-(SbyF8O&XYYhGU;bUaf%h66} zUx@y;`n2lxx^bW5oFn;E{B7@<^UW!-Lrw%@b$rqTaskotrYt*c!8z z%P6adZxyIT!Q0kNo0|1ROJ6P0{mI03)c zKTY#Zy!*QGdbYViv}PJ5z#0oHy>Hu*|gGlK{}T2s+l1?N4EaZ<`wOpna`Gv zCp_Q2vB2NBWwLxI^3ZhYjZYIkqV@Bk#V=s*zoVT3?ks7EUewx`QajylS$l#uJ3xOL zrd2*s*d2^*1&B>#^||e+FHQY99zM(!Uy*zuZ{2SnJghhu2o~WL z>6Lxa&>%ti{<=oTdWz%0E}*Wh{n|>zb_Al0KQwTdv|BeidqFbSmpc z?#FIzKc`bOUw41K+>@Wr+?dOxPNfC8c`A*!N@dD1zoE!BZWOtodhw{eYM*q_k=#$Q z1yy^0gFX@K{hoe{bHzCd((rEi|&FNatx%^cmMfXdZ zaomcIDkXl~#iG@M;*ur*zwhN9JEfbZ_*{x+Nv{RbrAqzRwA;RE@;9a^209+}kV}j^ zK3>p2oZYb6qQB`hr`5COFaP!K7u}Dh73-E|irMe!+3(0cAFdH%O4gI)T6L+>ig8vv zvBHMx<3PuQz1(fHB(6W%eX{!}`lov~>wJ$JU7%&+4Bbn*LHPr5f6HEeA3e`Jj6v%u z*-J0!NnqD8H=WRvz3<|X@aY|Xj{EqoG`)ryhK-3Dnf`fB_1mNF48a{ydqcOGGY34O z-?#M6YuaJAW%9Q4;o*m4kjo*f%9q_LiRuhP^;%5HgW!@^bszG`7WkSp+hunfDf}Ui z>2G-Q-;!>FpOWuV{G4$lhKPzp$cxG&FJ0Ps*)~Gdo6n-8NHaox)FocL!H2u zT}a$^-6&z#CfSfu8PStNp11d)bv!#UpM`hZdAnjB@ycRJ!GqiKtUF|uzM&tS6KLVG zy#2b-LaeVX${$*`AhfJVDSXZFO89*ep=Aq7hxc!{?nL3=c}~0pFZ4@_K)fZt0Q9vV zu{x!7BM9$3stzM=(jLLlsd$iUr?9SQ&5O>ad)ckO{)o+aXANQvdenKz5XJJ#f=Mp^P%>LyM4ah?al~k-}$i0!vvz)?hWPVPK+pZg>Gt$wJYec5~ zIHj%Ej+Eq*hD;9c?j?9w1C6DjW{p?Ve686ah~}V5IcI&f+YmK6)^n8FPajJk8pmfx zgtf>Te{C+2u>z~uZ$*4hEwP3@?p7m?e*1NfPQ`kXd^V5Aitu?)*oNbXGx7=E?*2qb zB3jvYf^Hu@52nYH&C!nO6V*!c>i8H{ouUWV<=I{F^ZzI1y6`>9emSqvc{u3(W{_!9tv{l8q>~yDV>LdUb(Ui!LX#ZG$gI@j=4^qb)HptW{~}P zNxP|WG-_YE)^jd@o$80eQ;ILVCr^)mm*A!U(9n)oBCkFvv5l!cG$hk?X*iZCy@!Wp zxk8PBX-Rcy$XeqabKR4hd(z4wo(#5r@N=S-``i2ZHtYTyjs#+H>&aZgr}rS&q$Br{@C|j zE?`SFn*9_#h|V3wm(#}9m^FJnWMB9q)v*oqoYu3p^Eo;d>lu#GZnws(VU8T6$B`{6 z`1x~KV*F46fGbn|M0SS6xQ1D`HJa8FAL9(Q^Y(8@!*I>Ur-bSlI|iNN+7U&5(7RoD z_96UPr?qYrk>`&4G4Ckf?GgP$w$&@LEl^z*H9q{ZzMrB8(Ld1+hfnfYtx!p1x|(=C zNylCkyE5PhorxMMI92pv5UTyCS{UUzba}|4o)G6$NJXI`d}>o$H!DMoUxGYOLyJq%I$4KWL~4OZqzuJn(HkOL9Rw;-RL2+_wGI#GR!KfQ(8BI z?4N!|{%M@TLOvu{k3-eCi$Uv6f65+&kr9B11a`g4In52f(?xgwkw^ZN{)Qz#MB@9< z^dP%DdlP;@>?&|B^>z@JbxP|-5WALepI3`VwWQ!dZ0<8Vtsc~0syF=A>q_f4g|T<( zu8R`-_wjvbdgfv*ddGL#uyvNzCbq^gMHX3@1md@z$atzu0}c`{nLm=>LCi zm=Sm}vG0CGQZ{rpELzr+=*=qktJ0BOv zsNvSTmirwv8Xm;w)5L94es8ZwQ@J#yFDWV)oV47^(9nV*-V8AA|@DLe$kkR>t%uh4$o zXq6`l+fb{~SvPvOvFF%lE}#GymCsjS9%l5cguICkf<@nVxfyX-m`9oQm!3SlJaM;=R4)vbX5B|H}8pU zER8Fslb$ihuv`h+@Bb;~r+jA`i5h=w4@tuIqu07#q>|r%z1k%1lvb>ox3TtF?fofa zZko}}Ep_@+O}Dz~{mrq)jX#pwN_S57$t~G<-*-g{KAck>!*=Sc(~LXF|Wm%hL7Z7$V(lk zL$5AVOAhEC^R7P3#1k4R>{gsqzoAS9mY>`kf0(GwxTBg|rg99YTZe4i4lP^xa}QY= zYb(>C)~By!7M#-V9=SWUUM!iEOp;q_{g$2+r|IzIcxvQ_ z@EAi*wM8r4!E?)fc?appEveB^`dg79r1J>4gEO11Xnw<7>fgqF&|K}#<^;Vp@O!fR zlFrb5K^RJo2=;QYd;*g3^W>CHXf~0#&tv|SsJy4$Aa?Yp^xj)qJ8Hah()Ecm6Z{SE zOxd=aMPzO9bE8tdwAHjHOI*m=s*Qv>_^R8*dTl;#RgMxgneNypVC9sB)hF_%?nq7~ z_X!?&4@gFK=&+drfFv(z?mj1uPk6dCNI2U=~wZ26AmO8=ja?{)vY5$f-tS z&6uS*rruQ`ZM|lkV7;T7?R#1mPzmjnR)h~^1GMm%MeAW<*tV9I_#NIF2`iyBV`HwN zsWq4n4v<^7%#jCiap zfjl?HA3uw;f^3aBHd6gOG~;0<*(-8Kv!Am{3WwyZL*oT+m{_STa+tUGV0A_kf_yBT z*}5hjV6~mX&mOSOH6=8Mwob5ZBT*e_F<4}N9xEwW&%Sh-WM4S$k8JV#r>)nF6I=JF zc#PQpSHxZLI{Ue+MrRq=wM!*O?GU>PnRIQ|1)^(`{hHHS28v>#_%z!qdT!ju^<{@3 zWAKP{G_sXj;<5Yad9dXCYn+$B8NB7J7O}1SO>;JnPQ`;?JB6pDkB|u+X9UN2$%mf{ z;cn~s^$7VVA9FHf_4ePzl&{|Hvs$NQJy~uK45MSL{OF!H_gn@6p= z1D#2FCLo0`RXj?N9>OXu%bGg+E#hJI`s~1|{<50^c z)=t>uH-~8YZMqXedGDLhfI5pLRTS{Twh?SE2-yP4rBJO+FWkC!%YvZH7 zZlw7Jwy#N^-|ldV2{{`#jhTxUt!1DntG>R@G zw4C+uoar7oZfT8lY;@XpU2XX2T=BS*m6qz%`{=D1OV~bFWZ$x$`6J1RRFd`Zola%V zxam9tKR{d?SCe~9#^G(%^nReXW-L()`;OM_CxpQ@{rmCbs@)$yjfMxarB-jTs*{WD z(;r2fng`FJEAFA!6sa=OzQ->qECa8Aw8kf8b#KjXIiWQ~tFf&JS9RoI*L zPo?H$+g^UkvDiFQ+q>%{viIEj{=KFFY+6J1@3kIz$Dr{fR?ugYoQ8szZ|v-V!|v5} z{;Am)+O6xc9@9P_S7zegSZ3pQ>iz4_$+lJC?EL)Gcybbtayr?FdznKX#_E6bV|j1h z)j1WX5v_f~(0m%5y0=$LHs{W?(VsBvv+C=O_DNssX!%~r_?&EQUlIGC5Xa>kA*33z zey_}QUdVXFtUJB2<=CSwHJB{k9^0=b-Z87s&Fgq+T7u)@lV&c~WT5Vw&D&eEIyNVT zJ)YI;cvx|L1KiGXMbTBc2ilKWjgIvc$Aeu!U0*x&r#R`nU_UhYz~DVYJDslB#^Xjt z!ZTXEYMd2X`p`+W@^6X5aGu7`TQxe?a~uzE^;sJrJwO>ZyGy;(lieruqwjGwZw+$| zs_8u^X+}i#<)D16I;Cac-paM{J;>iA(MeC7MW^CHPNVRMeDAiE7FIQpfa9&RX`xY* z2+rJiNs@BbARVbDX`kI19qWl?K3MSZlomA z`?!;(*CdNAlBiShAh*Cw6etbrIa2Hf)X8&J8Z&98~;ij<7v@dH@f9Hqj9um zU%DH*A9hc>6BjR zGeA#*nqS&1TGmtLxon&<&((Y|nk&))*Rv=@D(A*Hev)%VYyY)sBG3zWG=KGv79WNDgF>6q~R2YD##@(h(K{b_6tNclJ&(>`9-`{VUL z*W)4u(@53$BdN*Pu~f2(c|~fqhy79dPWoz0`N?-xX;0~^xZk-XBp&Z{auN43hct}U ze@zCCb&cWY4m(!iKYk~D6?3E4_RcM`)_&)e0!C$ODG0OCYU!jaRDVk=pOx1BmnH?e z|KXHQI_4)9wD-o8i?TJdI` ztpTc8Zh`9l9_NyUFtX`nAoMW@?~l{_(s(}iB&PLd(^o7N<8Xd!zjN__Jl^RfAns)j z85pboniL#%|D1avbZ>HO&%;KV`i^LQ6!11L`PL@n6iM3ZfxadjF4Z`RrWqw} z5@)=5WfG?cLoS~+-+BI>Ce3TezU#;RczS#@ubz2B?_ARl;;=6ovha{`{^DaMlwW7$ z?L9L(|2dc3DglSrIsYA@+;SdH+ARZ{z>hw3^!(VimD{4VZWLn+SgO2xb*x5X-Kdp& zB`;@r_GbpIHciH6v;Fi0`^mpG~L$H?fQ>cCEdeEKwY*^5U)~K}=I<)lZQe^zv zGt;J5`_i?ZGq=m;Tg6+^-LmH26LJtqK%=p4PQ$I)u+5Q=*p8b{W!=c3 zLNiVV-R%B@kVCHC_uctE{|=>2Y266I!@*pKJ!7jasZ&`ua%aSAsMY?S?4h=GwxqCb z#2~4!=|`Ts4K>#Lyk|9|7I}X+0@pJ8uy*Yks1C;gqEQ-IjZlK|9}bo{|*2>)yR0EgVi-V+4kklk3W**h_>tebV3rO$y@0! z#M!1aJ1TuWzFcYa)A#Z*#beN)m*Z7|=Tk1#(XhEb&dWTmfS?)3Vibt!G_eO}p!M?IZn#JvpR%2YaojPF}V#_E5GQqj++ zd`cEw?9)C~*P>gp@Y3%+R#vR>Wvg$W>G#HZhIu8l-!n}oyJ`Qm`9AbYH|?mFDCe3d z5NlrQWh~zrUt04%xrh8+dnEJ9Tt3$6q$}@fp4Bnsks)JPqU4BIRQ$;}V--Bi>8tUj zCGLx<$M4j8pI1WSF-|8PaW6x-vej-JtN$fRh34%)?P7j$B%yp~dc&@|35q^2quLc|+gR8qTLtKka;`$f876VTg>^a^m!}om%@+{--?MS&o3Epe0UH zqg6gp$XOpdgIQ{KTYZjqdNIwo|ErqE%aPCYn=v<7BOM!^kfm+)Ivzgo+`jEjH6V_5 z(@+t;q+pHwc<>5H=Qr>N@x}C9)UJI7Xx+MeWR$dT(IQ#RL$Wm{!b8TuZ zXxh7M*gscPb^Uw#J=T3AZL92YoYVsru#S-Zhg$UWp=ZrCtX)`uX@v;b)j#h3e)kVF zYE6RwR=lg(iHFGqFZXh%H2S(E)jvHY{%x;K4kV`O6^6<=G@P|G^cleQw^i>rN_g(<}T>cad$N>x|{sQA55pKW)w4 zj=d`IUm>eJZdJEPN^*Sa#cZANnNX9kmLAU>3QcG!TjW)+M20D$v5Q(sF4CS>x4s^@OVM{Zm6FK8XtB)a&K$D?%E(PCwE$I@WV6pEG0I z<(p2*Td>oqc#vzS@Ra5q^*Ug2@~o%PUwemkT>2U^pr5Vv-xH1cmRGw!lV9U<_lQ=^ zYw~j7w)fUZ)joQj*=AY0qGx?gb7r-iVt&M4@ve%91V9_NS0vK>a@NxykA7*ztUP(^ z%~psrV5Az)GB68vo0rqVU7e4J+hFTHq2IS8#jo${bNJV{*aoXpT{p72Zf)E3TeQ}V zB6RXrWl=mG@Lg&CgLdg>iNEUBYp&#Sxofx0U=>4y)aZG5bqrd^XRpcHW+74Av0&z} z31Po|CSHXbY7aZ;>&~`l*(25|SxYrM4?XxlSl&acR)(T+O2pg9d|{5jqa z&1?6D@J7A_YAXInypA=bB}<}4$Ai6ikmk;gu*R(O`{!f{KB3IH|Jh;epV3eHEV`wc zzpMFH4-u;lkrurwQEKb!-)XH%HeTP>tcB=t8tvU5-x7NQH|a>_vN!^rVq@=`@!hRj z5~B9kU43@w9LS0IhYov(m(DLKN7$A4^u31NW(+ye`^A2#)_3iFZo^~J@!;4_;Qg`X zM}L;-~OEJ?2V?N;;pNn_P0@_M^sKX^W-$E3mXs-G-Z=`)Ts z_Ic#A-?L07w`u=1%v-ae(v>-^@B6aBrXAIB$aAh0bt^TtgSHcqttTRpss97*$Z+QC zx{H`1Gp*)TF0-|2?Mt08Ro<390$yLeTGyggK9Q`}OTY6n&CuFTM9RGKZ?1ETI$+y< zm*t6T!AjH7_zcy3w|J@dwjkC+H7zWd*R-*16XpI#@f+E#V0TP)rp3~%y7%jo20uHV z;O*{DWILbJ3fHziXf&+RA0O7aZ3SMc%PwwSg5?X(e0(m(vqoq%JeaN3Kc3HoHVf1H zVs(p$cbuuT|CclUM6`_vsNt)4oCQUgr23*|9<-AO$5)394^8m$bAR{2E1 z8XVi&=#vkjWxPnce1)@ZjcpfdTIgHA|?>ifV`vZuo86kbgw zC4G9+p8>NTXYHk4oca^moU2QLjM8qKa#=c_eViseY+N4l1+m-x8Cd{`AV?iSP5%2< zg~I%xA{J_CY&i_AISi~ha3-^6bJcuC+@_S~X}v_~m8HtNLkUmH?aPrv2Zi&fy0-?-ZSDar8Lu6lV(Rqi?^>q(X`%h3GKyA=dp z!(kX&wAPIxe1h-pS7O|qjjAsGl|-lFK~7F?&tF;2C&$Q&ecSCaT2An$tvz@R-OIG= zyM8Te_BkXCyZwLcBo}{uMmv9ac`jqZU2-0f)b*->8!X z+X=POq>@o9_j_o2=J?oUBA1kT^7q2-?DT%Or&%WQnHHm{+cT%M!n|tGhld!G%}I6~ zqUQAp)~@bz<>x9foX!VZpPSYgK=VEZ<+t&-yvpIPV5jU}asS(>f%A zfy=F4wOw&&8Ow*vbuK6JzC;M)Cm+svJRAOa!Li$4`kbT~K8n0l!;vhh>0`irj(;XJ z@RZf-RNNRp`%_v8+C7J{bl0g`&$xW=O^g9zAiM{BP27sz7)O?e^Q%qy;UJ71(G{I? z=bdhA-Q#%9bczoL(f=k5a7JE)xAgOBH=aJQ=spB=wY_`IQVv!_9-CoK)oCpQMLVjH z+fel=((yEmFY5gzU?+{x!rcjPvyDHz^Y5ohGCD~5F4$rQ_d7Lxix<6ACxfU}jy?be^Qu3$r z?#2DKdG|^R<>=Sq-D|g2L(4;=6Z|_?uj6??*^~1XdA8f`{Lb53qdp!lns_~PEo*y< zVKr^l%masd(L)vqY#^MdxZHi){eM%{xIqI2Nrcshs`52T>*l`nsd4Y(E&cpRHo(ot z{9@03Jjb!5xMn+;Lq5GUVIO-}#mEWr@| zfly<=HHv3b^UUoroU-$2XPj!2{i#?3x8(1G-?;6T(ft%Xh_1B`pN;|FYD|*veSRd%tU}!>A@!^bUsbl8#MfRcja8UiSTe(rg%`d4jk};iiqjyg0^83-a z(Wdqgkc>IUE6a>tOkv%KArcu^GW)vIjV-cUr{qC!sd3Hx>3>J^{Ppe;{ljj43_MGU z4-wbT$pU=S#XQGDR-^e4u+_8vo_1b6uEOG9!TZqkU^^DQb@^GYND2@q^LtoO@8)}I zi^?~Ty6QLBO+F>BBYZY*8f>=x6g~6(C$h+K9>d;;q<*IO9AnruS-om|)6mkl-$E7{ zpIU{UhgNz``stkfUqevsN7Z`9d?&j+WO1`Q`>U`oK<4-mL~Uy8MiQBR938kMsfT<$ zCmww<2+uw=*NrN2C~;C95in@cv5@Uccirg1OM?jQ5SiC#tQ$4XI=dvUyd8vGjn2B! zyC#0SyN|#Qvx@4J){P)6A=JVA9a)pSc7IKnBAN$j9wN*ADSHq;>()W`IxNTLgf(+}!Tc5$(H0P4AC2Sk8Zvqxa0rMo34~|BT7_BRP%=GCCinoN`L@#u*Qy1=DxZ zSK~=eYp?ni;;nKi47p8U~$l$a@C!Q8DMHt4ez20(W3t z)4KO(`rVQt=%H;%*ERJI@8)fnQGK_2Ok=_x9995sedO3+uq-TEsix+Z`__vT6|1(F zhH>9}C$WbHV(0WP&Wo*a529Vc*9$!0=ir@N*fHv@f!~wem$Xa&f-n@_B^iwY$@qD4 zN_i!5&b7WGY8PZX)#A0~X}7#xIeSx`cT0O_z30u+(P)%UAXdcq8L+bdu=>JEoy68!_ z+hksD4Q}B0FEaL!P~Ajn2Ax(mt)_UlXj8WDfz!Aso1!{Tl16QaGFA z;kmiXY5_@*3@vFJ44#YU@U_&zTo5o_%Pp8`Zw&UA)NlrMqr)wKFhY zBsCiAMy+=K#fw~x&bra7pMLS;SEsaY1pA+M@gmxvvIpVq9F5qA@TKlMo8v)tdG;pq z#>5Xe>=a_1(z+2W&k=d0yrkekY;I4CyeyM$~NS6n@P1}vuhZ&X4N0POi zp8YwYs2}3&ArHA9m2hU~gyON`IgIf~avUczbRNigMLTg6 zYd7ulapWiNg-<%g@6`L%pNr!H;jCTCjrmjCd>yF75)zNjZa+jW;$G&EhOzoT4jJgT zo^dW0&Z69Q*%OzPkr-FN=L%;b`$=CF<8wogh*tep_k2{$yQvO8C5?_4q+})!J(uL8 zZpqkoGAW-w+#Gg%CzHT|FNhCo`+_nn9)4zNQ*yv`Cz7Ps*3(F7`*P&r9KhH+7umyE zeU7*H3+IWv`<2kr8tK^Rgx!YK>v;I28nedtfg1uU!1E;2xb_;qbgZ_MNSO1$p}9^6 zmuTxeuhsdvjqN1OF#Lv}0Lf>hq-g1&CVlC@HL)_wqY%!YL3h za&<}`1n16TXkAzOoYn9m+J~kG+p*|v=Sg@!zyB5$?~L!K=$Y^3Wuf&u`Qb&iAJugu zYo7q|B5G4xH9b)&1D1@R)O(O5TXwR0d|}*o($_k=uEa?K}zOvelzdo(Azsc}c+|&2xKV`eeyy zak=ego4+HjD}Ru=g}RBU$6rwP`TMQxXD^ppv^@N}-E6az+@o#)&X`jl`wTIoN^(4a7BeG^PoGvKO zD9E_>$-kTfiq9wL_}P?@b+X+kl&`QSgvvGDxPg0Ip_$|!5afA(PkIXfLmq8xLE~G> z;OES=Km%*{&xitU@sPTgxujx5nzZ75oMjO{(N4+4Cpfgvhs#Xd!y!)Y$kW<*2f2!+ zA|9*Veuyl@y=Xm!-(>l_ou|5_r*-1?<;Z0<9zRD>cF0zr1-7+ zvfu1JklnG*K9Tmvn*)+du5A^)r1b#aaAf}9(Rn=o`Ja-42mkrJ7W?*r`cs_H{jOsD zUw%Ai>PJndqW8)6t8kw9*q4XgbF$pAH)!XzlES(XgSCeB4ZGC(Di?Hrel{;}zizZ- zKY^POFLyzodp)u>Iv(_f&^X?;#X>;ywc(7Ao^|;Ntu(%s3VQ}#soV93M#FlhqZUr6y>h>Hq1&2>qacCHqwTfMq}-$iN~iE zv9$f#(fUC9y+4y3-ilV*e%)wsb)!3~2*Ww_ZJ6m)){PwSx}X){wvo^B_Uk|^x88Ig zJu;7AP5(sy_SMgNPF69nx*{IE-sLs7^8VP95UgPFkz=Jrd`@C(;PcvNwfDJAdz_4I zdz-p$gW7F)L^l6OWLS$W^SJCNm>xybM{84po!A;%*85|frEvnd;n~hYCLK+P5bmo} zIsQnnMjv-|`;nL{GR8^LlU zA>Y59l4c8g!t~Wxt4G`e(~RG#_c;gm$0M7L_v2pX;QX=rUk=}wd+^h&)^z@QEbbrH z^Bgjej(9pLNc)*X4#pe7k|Y6RMLkvK@Hp=${5>A`dwZLQ<7Jdna=Y{~52ugOJ7_G! z;(tS#Bf+Nz-7c@=zsKWpp)N8duWP-{!{sunDY;zwn1{>9=zTd{u6Z#}DPxNBhr~l# zzB3lz=RGmq{9U{MIV2$;;dJtl_cVu8j5&to$VTd|KJ7fR?EQ}a9!nDZz0JY*VU*Kx zeCT5iULU9TDcImTqvv8@$lh@ z&{q^=M2*`T-v@38sBoTs-rgGZ(y>D97w08$o6@_FFzUc z+@BmpL>H${hvewzw<84ZT?T@p0f*tY-O@1(zw%BE?<$(}VM%e#{D=yMsM}ILwHL=a zr8VPMyPd>~UX9M0vE%&?A~)OpBx0M@SKe^Th?loQ%NB0h?kC~78UO5Wu)(Z((RK-; zrEm3&Z@_rc*}<1&2mcV|Ci?B*J~XZ8JD&Z-7^r?pyNWmTujM>pjATq_-RN!iljK$N zTKTZNUBtUmloZy@d-YpIyvWrlc@UhtDrPI^=^4pSNed$D{e6^YudP??SJ! z^C?XawtdFfhu*r*4_}c-2UP?7Cx-XmCgPQ${S^Dm4_h!NyX}4w;D6FZOHqRay8!I- zCA#xr_jLDT_wF0|*1D0kZxr$JsZDL&NcOo)#EWMin(Icj?>!=3Wc$)xH@e7$MYJZ? zcU~kl8tX=_cCUyRxf-2yqgTIG#EV~@(z+3Z&GMEaw7(;}1(j)0*$Un&)Tg}|B+LCN zdl264ClSjV^|ZG8NyHC$JBU|tf3i-=!<)<83&GCjlk?%6et-V~@&{829>nJM#7K65 zHD=f}@YsMy#NWK#PqN)lQkRi;-TKLA_iBVSIGDaTR6@DMj-NN%cq+l79^gWaaJ7Q@sdviYZ8#YD!n&x6Ze4mPJ2mT@#n*8`^eL_NJTtWyM4-iY5kOa92oa9hXjn(|8ekru08a1 zi*vsC9NQtzxa6baSYs@UY3}>*8#09NrEZqwHRqIdJt47B*z~U{Zu%AJ`a3$QjaqN0 zA9hWi5V7_9*=(tVu(h{OGEz+RTEZCVvr3iW9Ce)+mVVOcyo+qZON9uU?$;~N8l|mD zE|}`HQQR+UJAK5m>f0k{3gzfS>#;3v*=JaNj<-L_eOR^m4bPkBFO(Flk&lg5Kix1M zUL3oJN8+X_+aqu9!AiVaXM5tgx zEN{quu4#l)Z|3bv-kwV5r#-i6eT_!>1VImut;r6n_kSb}c1Ab#;8e^jx-0R9tgzZl zhiA@pzfR?ls0F=b^*WyZhvsL=2)TNG+;l3|6Xe`>JU8cV>p_UxtaClMB)k86+LP5a zM7b8kZA!kLY8?o@wygnq`*P$#7v>T<^d_yt>i=7s(>sbAKJ6^oAG-hjdQ7KsNYoCk z|K*91Uy3}_cFY0=4_?O-uhq4_{v;=z{{TCt)PC^Pw?J-vNlrnK5nXsrG0NJ?UD~o~tCFv$vS#Cc+w9qrLOJ@d zhAiEpg?+%5Y^y&bj#LVbLn3x)YnJ0*$Ypp%Yvy%TCIXM-D`rVy&3LhA9hN_0x^*2= z$E&2^!RuJ8e{0NIw&uSS@c|&Dt)zG#)web;8={s$ALOemW{8$dMj}H(xb(Q4cyTWtZ$qiMgCU*y_Md&oRF zxqTF#zV!WjSJINgIuOfo)G*xa|8lO6(KpFf6nvBEEz14Z(mDkV0n`RZHX_z7?&E}h zYv08FE}X-ujfb^OZ47N|s!gpNYtR|XURP6n?}?L!mWM4XXZ<{=E8V$Wa=IorK=FY$pQHQawgl5kEwBUF~M^fJ$2J_|OrZ64~f z@WaOy@p3n84(gOP{OkCmiu|jOx}PgNOXa^7Ik2d0^U!x5hDky?rzx#%4(vI=8UX*> zBU+g+JI_Gtitx5uPc&NF9MtC^&KvlKc2%5#hSL?Wl>8cLAH8iZ>T^-sGk!z6gm)wz z{@K`1(K8?8P&?l0%)VU(?k9uBLj2%@@(}B>gZgelqv4s6p_%((M`iUo-tPC7vM!akn(?4Q#_qr?PJ3v=fnD&$~ur-Kgbfh`fm9?bnSK zTS{C*piMGM3hPF!%<1)VSV>{sh@sLLGQQjRD{Z}Qr0O{gULLDcS~r4AuRas3ENh-L zTQ;=x*=K1^pqJ0m*4CJpZ^=w(zAC;D+ubh z0##Iw%U@3;KWU%bNBW*Mj`<}g9qV**llC)*)QmTRrOHg|+r{kx@RJ~)F8+HOISG9+ z{qVZh_x!RE#yFi^gg)kwiE(;gsyyVr`_nEOjO+*8td+i+Mk-oIFu!cXBcDz>;$G&E zkFok+s)XdRnp5)3$iC(SNl5%^8VMAlV`2{OLvq(FL@LjuO=d#T)Ci?g0~JB$2WzW(ks5;FG4=9ePv ziKde%t+zR(YwU3>Rnqdv{wevn5Rbhhd+ZHaV)3hKB&zHoey2Wy`6VYF=X4Si_cDj1 zjMe{AB_z(pIPGF+sAm=anno%-z0NNQGTP~+Kzf)%0>f?shzcucDW>Aq#7S`CSY7uKyEkiVx`frMn5)U;gpHh>v)*KyZ4Ut$9Y5DfVizS z+NEQJv*_n!x8u$w&l%HIr^GD9P>1W zPJgpL#<&w9_lWejLduqbAwY{dF|G;^_uX?QPdFJ?6-2p zkFlnGoV{J+@shXGR@#$JQ@cLyx6NiNDU_oRGM#HOL^^)zbg$Kl*D_>fLC?n4l6Zxz z)??csasQq=^=!dNYiq<~qZIVD)$4dT)$Ww`S|Bw#)>AAUJ9yBxyQ$xkUdUHPwk`5^ zx3v2dzY%(ETP5@M<;d5vM(Q`4c{RUI<&dZydd+^q1V5^F3+*Ho4qKZy8vIS;*H*%=+I7PtdlCE_WC3j(Go$JXSBO5x_D! z61OQida9K(^x7tydHZtYYgsK*pSvd}5?XrD3CMm!@gTn8s!#jly+O(vy>_GR$?g-% zT6jn62Hzv~2}zT#J9Ww@-JRR8=)UeK)TJ@-XdaKpt=%aFy{~nw+^El%d%ZoJp11d4 zwbXl;#TVh@Ozc(>kpN-2TR84)$d2pslo1sFMxHxc9)#!D*V=L zMAk&H-fYE8qoL2yv1z}+p+2ePyU^MHJoJ79i`KePj6HFeje9m0rU6r*JDY}u%7j2lhu712}wshon)l_ zu(w0gjnhfQcq2FtNyyRDSF-r24{=^UA62>D$)27y`%7A}bX|2y-Ur;3^=JCslFJ`f z4y=jnNAUJ(N#u2RrsFl`tbNr*avoD3$ah1e_59%<0Z}flrwDiRmTdxVVu@k(#)pM+BT#4 zJZp2j{0t#vINf}d&eG?jMFylr$urZ%1-MV1NpG%;JR`R*N6i{!idd(3DiwyW?!LnMtTHd1NnVDma;pE6_ynP7oCa=P^ zogM6w+Gn-*xlP+Wx?&I6v~}LF+YGr|S4L&++v0mhH@_zAD}9#O(#^SF?38E5&iI5h z?Dg(3t&qqwfLx33_?-H}iD+cb;ay8VfX26E&q=O?(13;ZGok>00N?hl`+@uBpT+yU zLwIv^{Q=Hh@+lFFANJG>T3_QcjXD45d%?72$rV%X3FE$*=e;Xn{_ISpd9T$10n6}390hlJm^9g6Uhc(quLU!e4m@Pp?c)j@xx^$a(I567{jxNbbL`Xj&X zifp%Fq4~K{r?U*~g0C^Rf1tHB3y`(>1#ulJJ;2ZTht7)$KT``2>U5TYo>((J?AVTn z=NG3k{g~zLmw{C)r?C%H;-VfKF0^bx=sU{IvG;vbc<@_MLN4dostK^x#JA2vKw>Kmt9N{-W~tC zP^06)aK0UED|+W-y}chzs%>fy0m&g8$mqos){U6%kA6zhi?D{iv=RS)43E{0Y`CpLQI=r#7{9BN?B^d(_DnGH_oE;?O=c*NrMNT##!E zpL(bpIvm-)bk~h8cD#sQ50QC|#=231#ylg`kO4jfxf-2yqjycn-QC9|hFL{*O6x`t zQGVQy_dBvCdF}q1Fh!IC(maGC`&0HHeAcal?0Z>`%?WGhf>3@-qem5pr}XzFtwBTZ z?L*Up?D8ymoSZ>UJN#F}v}c{tx)H>#C7cIu@gI~FJc!NhLzRe{HD=f}@D|3mD*aMD zFISIWZgoxSHz4TPpAhPZF&POK$P>rSTcW=ug=OHi&%d9? z&^6gTO!j)5XL)LHeeJW_`;e6O3LRnR(#pXu`7{ek8|{)voI$K1CdJ%PHB49FG^Rn7)&~ z8dr*1d)A*#)+piclq1V0x4B!FlX%2-`;;HkvFzi?xR*I(Vyym;LlSa*qOWb3uQ%PE z^tRhUp>}3@MmX#VbSiOGg-4yG23gbYVai9v+?$^fkTXz{?>xM2L%x!wS>!o=OVZ}9 z)sT){Ki$LYhK66dxX*u$SL_v`NO)o*LzK;~QH~(MH5i4XuZ#ArR{UI`t>k)Z>yW8@H^y#HDZyvSgUXTXI z-8_HkeEMA3woRMa)DEfF0uNceju)@z?9&#{64!aMMtOW%%^IzCngel(dS!;(I?*Kj zFg~cTnxB#_QkRVSr>9-~r9CS=wt?!sKsxK6remDrx?%x_9BEYI z*737x4e<=sdSqRebD69g&nC$MzoHqk@&~P1*>=V%(_xKSvr_|~Yu18hmqII@(7r3&ZOZ;Mu6grELf}J# zobRJ&Jy*V4;FC93G!w6BBvwoKqK@MRyDiVbzmvD3kzTK)Ma$x#ocE43t1FjoO|qxo zabk6ftv-pY%$?tGsBzhxBzwwyXPS@B)@fmf8}ChZ{A)Rw~hIwVbmUBK-KI=`%dGMo@ zx3@;U6e~)9h7+=BwkYP$x{6k)Iq8kay(38+b`MlZL7yq!yDp#G^c#n~dmiMSX*VFP z89UgtI3Ks&TqNtbedY%dt-94CG|C*M?dBrEMt+wVNd$cv@)qom+?RkS)cgr}|D?0A zwwsH9gl3zKf$Eo?4|BV5MmCOvZ|sr1c2AT&Vk3Q<@%r9E;v|+Y{Sg`|oD*&iHI^Jt%&nY?n}hvvFb?RzVe7umja*Nv`rQTDSCLF9bXL ziXw@K;`;LhN(vst=JsB-=3RSld*WEpc5@N)m!R&8M3px(c%{6g;NjKza)bBzSaTe; z$jRZdgKz1iXPw_`pVi(+9$V|_)Q}BlQ`bjpw;6IIwAig=i5V~a^01_lfD{DW?K8ypNIX~fv0PE@_NSYKBJCT65>&_=4;{el1lj$v8Mk(yHm;WhdsnO6SjV@p9+{oEj0*gbe4hLzP+FR z6gc6{wn8}V;+5Bug5DQ%`_j3sJcHoHtE8}IymFoNEy*a(>w8hF(ODLD)@Zf54R(Do zo!V9}u%%ugwBTzx+w^;ia>44tj@r+qIu(7Mbba`QdXBB1Z^-&%+olhdb!Eey(~P{O z73#vMv8SU_uhQ0LliC&7^fBo3JJdD}>%7&g-#X20R?S{LjBhyR{(3Dy_FdKHC9M=V zbgda@6xMv%-mVxCdruSS#Gc?9N;a(_zTjGqbNIrpI!&JQH>49k&y-X9@)@+q_3m-E zGF*0R0IVG#wz|5&wc96{q-(%~2MZAib+fQX3r(F_Otw){366V}1ku^FVj+2uXX>pZtcRJC`8b_a;nkrpw`;{>HW%!`gP(hj_;JTt=B2`RiZVmr-$W>*}_g#?q)#z z8lF@jQNJfkry48-u|S5*WNOOb~TTLnHU>zY-)ev5yOIco9HDm98cd372r zhb!_li+q*Vc(6!=Wd^;+(iB>I_&)fPtX5O#omTCyb7$LLz@la0k!z3Tp~$ky8TFTq zmBuw1>qbrNEyPCJIIg6yZp0*mv<Fb*OH%>xA&m6)Vr_R)>`Cb;NF10 zbUXFB_p83%mJ~c_FLAYiUxag9E_R{xKqA=IhK2w4mB>D zlSHrP=S}m`*}HE&2Bq7OHl;N3la6X%^hfWH*VmjEMT1QfhZ=t*#}SFrd8zbSoRTjG zcM-%^bpBrc8uQdz%OK}gopw4V{X0YAw)rS*PS<579ivTQh^(aj%poJ=jbLfAk?(ZU z`UL5#aiyZQXI<{aUpVC&5|4N~X^4B7Ll(yBe`%5s?_yCYfN>ahjv?2=fiBCE-phO( zF5{VwyQPOYIQxoLbv?r$+QYWKj)R+X%TeFO#H-mM>Tu|zSo%2x$p1O%0eI};Nv_4p z%X0{2%Tl6)of^j&HMzrC7v$i*CmR+SP_=RJ3YhoJa^{b_rq9k)%BPT>%lMh}vYWU1 z951^>v{D+e@x*pY3f9PvCo&;8=b8mGtI0KUS$Trq%V8yjHRGju)x3ylG}ern<+t!+ zXVF>%hPm`=l6wg6X1Zz|k9YGh z2Nu}U>imf|s?BS7&h(q{d>;~>kb7tKI^HfA`NQy7N{(Xd>53X1>nVa0Qx7=d^_F$e!;ht&qqVhK1l?>FSg`_zk1-lr$KsB*byl z<(}1)YJ{lF<+og7B)|Dsx7I5eI+fNL4dduTVO_Hsuaf@REQqHO<1JZU;JekjUZaZL~zHB-Re1r^g#FP-P-v)y-HULpFyjX2t;7~$V& zt^bUEVm{V=MESe(Mf$~p9H%z-5F2Hj5!m`hqV8#&JeA#+$bD@WV$&KDyRY@QFDhu^ zla423ChR*Ubw!f0?2KF;m6R`ViXycUcm&Wj^>bXm;lja*s+m&1}m2V7Ox#au<@!XV%iDm}cke07V zh7mtzUYB+{x9Zg32}Rz)5BC--$2I&mlukwOlQq-vSfzrSHQMpAV`9&9JKDi}Ei|mr zA0O5}9!<}Vd1{D+mTSfb+_{}H3s!=UyHZc==cHHt`cI>=W{!JJ>jmtzZ|Q&Brq+g< zPGuRmwQ`%CE23Lh@wy+Mph9JZ8}ee}Owbio^@f*;W08O8vOt!Bq^!TX)aLK%yDjV& zpap;GG@;0^{@v#tYL|r63u?vdr=6(}OEtD;>pm#pUjJ158m)OKahnfBL#FTE_s~8NUEY^pKLas@({88smUe20o*=Ry zS}vawVKu%U zj9!h-y3vDgE7rRbH}Tr)H9E^cPxPXX^YU1e7qh(mGO%iu)0&Oi%5nSXtsBh?l3JYf z#NN<@cr`feTW#;rw?Atpf^Q-91+c$H>(%xz1bQK#;e@PnJ^^I69pami9z@UQoFY4b z%^RU1qsfujy*1vmSH>wlcp6vk?{D9Dhuv$Xqh#~DE-STh+O&qqPOZmHmg3WESe<@G zw$S;<2*I#b>d>A&P44SjzVwwbHw@2t(j}#Df7V(ycP3PAcd8)|?fPtxm9uo8ETi=& z^4me8ko|_!1hrj8>DR2aFF9oM$KG`Q(GnT4ipI`{dP_AUcwRX+6)M8%n4(-O?ZOER%>T|q2kTe5kI|(53%D<(C5PYJ3y~*n~ zx3y+-Yz_+1a;sO3heAtD<7q1&Z~{uG5F}^v+j7@68rJxa2Q`scA7(W_J8K-Twz8?M z0mmF&jT@);1n`Pdk1z4wqBa}iHE-MLjM=x%5w%@^eFYc2iTl>1b-d51A!6BKqJ8C7 z@pKoq?<8`q$2s;wS6!}d&cTA2?QPh0LEgJQ)BOtbf$V-xDR(J8@#C?0`qH&zYwn$p zEfw{V^Ew}}tMjtUdG zkLpsW0??`ubSmAHmuc7&(r&);SLx%r`nEmNx>TV*qpf(oVm8|B}LM? z3Jgxg;B;|KKLuG1njU%rb~UO5Ym_{=KeXHJLgm|Vk{EYMgiIoj{b5t{%)`(s#KZYt zw-WSc*!&Beu`vX*hnIlCfMGmq(h-w(<j$16K(AI<0d>q#*muH00HT|^lSlZfy)Z9EZ46oD9Hb?#rJbr%uV%%~n z9{iTdX;@R^dn($o%jx*goBy<3D+lV8|Ggd}oyyWMY+D&{q8&FJ`FSt2^dSXao#UPW z!N(e{Zb^Ug@#LTXp4r%H{+iYocq4yHcw;Y(yB|@n&yRgSs@9Vpn=RcsJFS|7@7u%< z=?`6XW>}cKf(4IZ#n+_PK^-H}LeSgYpM;P46{ha>nal&w>hR6}XQvI28vv`*uf=R?o_UZp_dBYBaCi+Cbe}I8z?R#q zE+%l(-I9K>`-YIkEk(#r!Be&1+L!JU5w^yxdyPJ)yptzX5C4C5;EvDeC*>A>&hmFP znHV!$T&I65rvlmDfS;V}_n~rWoXzT&eaTz2Xj*eUcOGm&La*C{HT+?p?t_i>nz#fz zGu*O?9EMl)8h3B&`-n2T)uK^y$5i>Q@hE;1J;xuKY!I7T`BdN}-69k{*1F>LvFI#| zJ=c*KrKZPt^l3(*C)XmhJR~|{RkwN_51*JXz9KGz+|(q6IRbf=0UOBFNw>2RsS8xB zC&(iZL%HhExM5Fv+3nwcYQQUPy=I)mXTba3Z<1Vlk2Kckc(5Bn<0-8pFYhf8mJd`6 z3HPtHYQy1tYayW9rk{|Uzw501)cfF%Z!(=A@%jo|*Bg0z>sgkrF;ebp$E-$Y8Q2AW z_pRd`FTbYU|8x5BuTXh=4_Xf`)&Va(vbnzK@{>4A;g`d>%}GM z*)K`oV(jo>J?gxO@O}Gvcpp6vmUH{xTlE0E=|l3+^Y$Lh=GFtPnCYv1+dk5w<-u}3 zS#CuUr+gv7FFMQrwlV6fQ&~51@ZKSUhx&eJjkswv){Yw1+ZV)jZIb$?Tdza=wWIZc zr1Q_@`)frjZNF}`VAEZYM7CLIO32-=y2Lw^Gjkj zYoFEL=QeE>%ML$6o4aJcn^qD_qb@0uwl`X@b*5u)Q-Y4>xDRr1((!$#X37O$u-+d_ zuEsfN`OM4{Oh*&yxE+2Zu-qS$X2RJj>~tk!a2%D+E&kT_r>M!C?tRO5rjob3XYMC| z*B;L?NLN1M>EtT!X%2}Ra}3LsrP60{+GR2YUr7FXDk&N^l4Fpc^a-Yup0uAiWM{k) zELn15-xs7rUdMk=CoR$&_prbBfeas^nWAqLh%e2kjQ1(plLO~Mbb^P~q zQX##GRLJXE@5dn(GRi5XLi(6TD#qx2xl)n(qPda``?ll1r;-YP@5dk&Vbs$}Md)J= zsTimCr~=1 zw2wN^8Y1X76n*8a;C=0}GQx5=r+@KTalJP_H-l4{Eqx(J=?P^}KkceANF-FMFzC6Q z`8d*(I-gS2w@b?Bc}o9Fbrk%4M?ZM}7t~`dH@iIf)`wQ9F{U~*dPSPIHueyCMW#eJ z=ZA{mYSi3x@(?Q2TvC;Y_@*4s>AijQtmn$n9D~-nPV-^rQMcNEe(zym#PwSkV)M=Hamoz(AU;TPHwDjP+ypt82CBt=>kiQ5mb4Tkv zteUoa682MEH=?)1aWC&}!ygDm@I+iU&7-?5TI)tJJ}dT^q!>0V)bzFHV=)Hv~4Zk335dZ$~kN^C( z1a=gdP3#MA@ArDJ9$z%fh(*hKzT8&)6O9BsakgiFb4(Q`F&}^3{T~`}yz9X0&+gxs zr{wqWKeW>;(na9p4_(`T?EXg->^aQ|cIhu^AI&QPxBFL`asC$e^RV{${eRy53w}1 zc}GtoPXs^21w9*ok5M}(9Nyo*BToT-LKY8f3;ymeX!nEt4_?8mf+t}A2Q++2u^);u zTU_ZRr2xOeIgJ(dBQNhqfVyzpKkh`(KE(+CM%;kvaWNx6U(Fp_7mJ&4D&!UjqAY0?^+|4GsU zKm27UhnM7E9&@gA>f1b7-b>+0?9YN+0ZFWRA-3<|D_A^`)86LDnj^@IoORvqd3{Yf z%?eGj9bqor?7w`6_~_8`fmlCa{~}QL)+|(@N3@ z|9nO@nHclbtxg|%9MOAQv|qvKBMo#Z;iU@cEqfV zCu?m>Y0%@x`|45R#4z^P?>lMZwNF1A3vCnpo!B8_MXkx@3GFapC&N?lzN@+{&;O44 zgVz-HQ27+B*YF*97_c+JDP~XGYZ@8u(A?h(YZvI6b7L%=1M&JJ#j?tFT3tKl5_y_X z6ZBJhZrmAhLw)X#o|e3zw=Qu!EH~6=5_dAx?{++)-6TGLNr=?_aq=zpRn+@B>u7;n z*ek!kUsxM;dKVr4V(p3b0p`><Z>`ER~{o8S- z_|Q2{ImSQ3wf1w&Z@f39qg45t$5n$Nz^s2dz&=vxR25tGkil&aZNuQm#AwYNEfZ>S;re$_g$3~# zL_%QKBKvXZss*{{_>o_q;i<~jbIe0u{iWCLsUwa(d+?49MVs{jJg#p$k1M1DUV86d zYag9CX5biX1&*`=Hb71S& z9ck^UI@+Ngg7uxa7P0lc{T4KI8lN<;)9I^G$F#3G6TG}#)Xz8jDb2w_x3p^WKDmxN zBDYxEQLP(Z<@z^y1~E@Jro_j0ua!^D-XCxTWX;wc*q6Z*^_*6$`naG$?Qg(lz^N6i ze$brQRn+OhJBTbGteI$0|7M&iQ2!R@0QX1Mzk!hfd)TML^dCHYcVw}>Y*)rIdBG_)MH2|Bn21U=R-MC{JfB< z{~+WrCw>hZeD*q>IS}Ms80#|2b05wp48!sXAqTq``U@F)eiZ8z=D-gtz-!XozbCH1 zGw}|y90{)($S9DTrHA2&j50)GE{MmW9sD@fDU_q1USoPK#lE(dSi|-zh+TQesm=YQ zLuU%}1n)!39F=Y7m>rG`yfoH-XtbfLQ#@B~9Vna)BLeIX0U%l7WPn0 z!Y89%kev%%>es=w_VXafo`gDfh~VSg*FH{@&*MCr)UFJ%t^7NebRQ2gnK+VD`yO_t zkf++`eQKE?Zwi%8;=Jy3G^l`z-8eE#O3lM3jKVz$@|~Cl?*+vEhG$0~F@KLUB=YA9 LXyPx generateAppName("server")), enableDockerCleanup: boolean("enableDockerCleanup").notNull().default(false), + deploymentConcurrency: integer("deploymentConcurrency").notNull().default(1), createdAt: text("createdAt").notNull(), organizationId: text("organizationId") .notNull() @@ -176,6 +177,11 @@ export const apiUpdateServer = createSchema command: z.string().optional(), }); +export const apiUpdateServerDeploymentConcurrency = z.object({ + serverId: z.string().min(1), + deploymentConcurrency: z.number().int().min(1).max(10), +}); + export const apiUpdateServerMonitoring = createSchema .pick({ serverId: true, diff --git a/packages/server/src/db/schema/web-server-settings.ts b/packages/server/src/db/schema/web-server-settings.ts index f36f75660a..db880eba78 100644 --- a/packages/server/src/db/schema/web-server-settings.ts +++ b/packages/server/src/db/schema/web-server-settings.ts @@ -1,5 +1,12 @@ import { relations } from "drizzle-orm"; -import { boolean, jsonb, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { + boolean, + integer, + jsonb, + pgTable, + text, + timestamp, +} from "drizzle-orm/pg-core"; import { createInsertSchema } from "drizzle-zod"; import { nanoid } from "nanoid"; import { z } from "zod"; @@ -106,6 +113,11 @@ export const webServerSettings = pgTable("webServerSettings", { cleanupCacheOnCompose: boolean("cleanupCacheOnCompose") .notNull() .default(false), + // How many deployments can run in parallel on the local Dokploy host. + // Kept in the DB so operators can tune it from the Web Server settings + // UI without a rebuild or service restart. The DEPLOYMENT_QUEUE_CONCURRENCY + // env var still acts as a cold-boot default on a fresh install. + deploymentConcurrency: integer("deploymentConcurrency").notNull().default(1), createdAt: timestamp("created_at").defaultNow(), updatedAt: timestamp("updated_at").notNull().defaultNow(), }); @@ -128,6 +140,7 @@ export const apiUpdateWebServerSettings = createSchema.partial().extend({ sshPrivateKey: z.string().optional(), enableDockerCleanup: z.boolean().optional(), logCleanupCron: z.string().optional().nullable(), + deploymentConcurrency: z.number().int().min(1).max(10).optional(), metricsConfig: z .object({ server: z.object({ @@ -211,6 +224,10 @@ export const apiUpdateWhitelabeling = z.object({ whitelabelingConfig: whitelabelingConfigSchema, }); +export const apiUpdateWebServerDeploymentConcurrency = z.object({ + deploymentConcurrency: z.number().int().min(1).max(10), +}); + export const apiUpdateWebServerMonitoring = z.object({ metricsConfig: z .object({