From 70d153e1f96abe675481cc2f061ceea8ea662768 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 3 May 2026 00:45:37 +0800 Subject: [PATCH 1/4] chore: add local service bootstrap --- README.md | 36 +++++++ dev-services.compose.yml | 30 ++++++ package.json | 4 + scripts/dev-services.js | 207 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 dev-services.compose.yml create mode 100644 scripts/dev-services.js diff --git a/README.md b/README.md index ac3637a34c..41b83284ec 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,42 @@ pnpm --filter=@examples/helloworld-typescript run dev pnpm --filter=site run dev ``` +### Local External Services + +Some DAL, ORM, Redis, and ecosystem benchmark paths need local MySQL and Redis services. Start the repository-aligned Docker services before running those tests on a clean machine: + +```bash +pnpm run dev:services:start +``` + +This starts MySQL 8 and Redis 7, matching the main CI service versions, and creates the databases used by local DAL/ORM/e2e fixtures: `test`, `apple`, `banana`, `test_runtime_datasource`, `test_runtime_dao`, `test_dal_plugin`, `test_dal_standalone`, `cnpmcore`, and `cnpmcore_unittest`. + +Useful commands: + +```bash +pnpm run dev:services:status +pnpm run dev:services:stop +pnpm run dev:services:reset +``` + +The default host ports are `127.0.0.1:3306` for MySQL and `127.0.0.1:6379` for Redis. If either port is already used, the start command stops before changing containers. Keep using the existing service if it is compatible with CI, or stop it and run the command again. You can change Docker host ports with `EGG_DEV_SERVICES_MYSQL_PORT` and `EGG_DEV_SERVICES_REDIS_PORT`; however, the full DAL/ORM/Redis local test path still expects the default host ports. + +Image overrides are available for compatibility checks: + +```bash +EGG_DEV_SERVICES_MYSQL_IMAGE=mysql:5.7 pnpm run dev:services:start +EGG_DEV_SERVICES_REDIS_IMAGE=redis:7 pnpm run dev:services:start +``` + +Run `pnpm run dev:services:reset` before switching MySQL image families, for example between MySQL 8 and MySQL 5.7, because MySQL data directories are not downgrade-compatible across major versions. + +Current hard-coded service assumptions: + +- Redis plugin fixtures under `plugins/redis/test/fixtures/apps/**/config.*` use `127.0.0.1:6379`; skipped Redis plugin tests become runnable when that port is available. +- DAL runtime tests in `tegg/core/dal-runtime/test/DataSource.test.ts` and `tegg/core/dal-runtime/test/DAO.test.ts` use local MySQL on port `3306`. +- DAL module fixtures in `tegg/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml` and `tegg/standalone/standalone/test/fixtures/dal-*/module.yml` use local MySQL on port `3306`. +- ORM fixtures in `tegg/plugin/orm/test/fixtures/prepare.js` and `tegg/plugin/orm/test/fixtures/apps/orm-app/config/config.default.ts` use local MySQL on port `3306`. + ## Documentations - [Documentations](https://eggjs.org/) diff --git a/dev-services.compose.yml b/dev-services.compose.yml new file mode 100644 index 0000000000..78a8c48667 --- /dev/null +++ b/dev-services.compose.yml @@ -0,0 +1,30 @@ +name: ${EGG_DEV_SERVICES_PROJECT:-eggjs_dev_services} + +services: + mysql: + image: ${EGG_DEV_SERVICES_MYSQL_IMAGE:-mysql:8} + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: test + ports: + - '127.0.0.1:${EGG_DEV_SERVICES_MYSQL_PORT:-3306}:3306' + healthcheck: + test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-uroot', '--silent'] + interval: 5s + timeout: 3s + retries: 30 + volumes: + - mysql-data:/var/lib/mysql + + redis: + image: ${EGG_DEV_SERVICES_REDIS_IMAGE:-redis:7} + ports: + - '127.0.0.1:${EGG_DEV_SERVICES_REDIS_PORT:-6379}:6379' + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 5s + timeout: 3s + retries: 30 + +volumes: + mysql-data: diff --git a/package.json b/package.json index 7063b24ae8..0bd98b4849 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,10 @@ "test:cov": "ut run test -- --coverage", "preci": "ut run pretest --workspaces --if-present", "ci": "ut run test -- --coverage", + "dev:services:start": "node scripts/dev-services.js start", + "dev:services:stop": "node scripts/dev-services.js stop", + "dev:services:status": "node scripts/dev-services.js status", + "dev:services:reset": "node scripts/dev-services.js reset", "site:dev": "cd site && npm run dev", "site:build": "cd site && npm run build", "puml": "puml . --dest ./site", diff --git a/scripts/dev-services.js b/scripts/dev-services.js new file mode 100644 index 0000000000..8ab891cd9b --- /dev/null +++ b/scripts/dev-services.js @@ -0,0 +1,207 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import net from 'node:net'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const rootDir = dirname(dirname(fileURLToPath(import.meta.url))); +const composeFile = join(rootDir, 'dev-services.compose.yml'); +const mysqlPort = Number.parseInt(process.env.EGG_DEV_SERVICES_MYSQL_PORT || '3306', 10); +const redisPort = Number.parseInt(process.env.EGG_DEV_SERVICES_REDIS_PORT || '6379', 10); +const waitTimeout = Number.parseInt(process.env.EGG_DEV_SERVICES_WAIT_TIMEOUT || '90', 10); + +const databaseNames = [ + 'test', + 'apple', + 'banana', + 'test_runtime_datasource', + 'test_runtime_dao', + 'test_dal_plugin', + 'test_dal_standalone', + 'cnpmcore', + 'cnpmcore_unittest', +]; + +function dockerCompose(args, options = {}) { + const result = spawnSync('docker', ['compose', '-f', composeFile, ...args], { + cwd: rootDir, + encoding: 'utf8', + stdio: options.stdio ?? 'pipe', + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0 && !options.allowFailure) { + const output = [result.stdout, result.stderr].filter(Boolean).join('\n').trim(); + throw new Error(output || `docker compose ${args.join(' ')} failed`); + } + return result; +} + +function runDocker(args, options = {}) { + const result = spawnSync('docker', args, { + cwd: rootDir, + encoding: 'utf8', + stdio: options.stdio ?? 'pipe', + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0 && !options.allowFailure) { + const output = [result.stdout, result.stderr].filter(Boolean).join('\n').trim(); + throw new Error(output || `docker ${args.join(' ')} failed`); + } + return result; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function portIsFree(port) { + return await new Promise((resolve) => { + const server = net.createServer(); + server.once('error', () => resolve(false)); + server.once('listening', () => { + server.close(() => resolve(true)); + }); + server.listen(port, '127.0.0.1'); + }); +} + +function runningServices() { + const result = dockerCompose(['ps', '--status', 'running', '--services'], { allowFailure: true }); + if (result.status !== 0) { + return new Set(); + } + return new Set(result.stdout.trim().split(/\s+/).filter(Boolean)); +} + +function publishedEndpoint(service, containerPort) { + const result = dockerCompose(['port', service, String(containerPort)], { allowFailure: true }); + if (result.status !== 0) { + return undefined; + } + const line = result.stdout.trim().split(/\r?\n/).filter(Boolean).at(0); + const match = line?.match(/^(.+):(\d+)$/); + if (!match) { + return undefined; + } + return { + host: match[1], + port: Number.parseInt(match[2], 10), + }; +} + +async function assertPortAvailable(service, port, containerPort, running) { + if (running.has(service)) { + const published = publishedEndpoint(service, containerPort); + if (published?.host === '127.0.0.1' && published.port === port) { + return; + } + + const current = published ? `${published.host}:${published.port}` : 'an unknown host port'; + throw new Error( + [ + `${service} is already running for this compose project on ${current}, but this run requested 127.0.0.1:${port}.`, + 'Re-run with the same EGG_DEV_SERVICES_* port override, or run pnpm run dev:services:reset before changing ports.', + ].join('\n'), + ); + } + if (await portIsFree(port)) { + return; + } + + throw new Error( + [ + `Port ${port} is already in use, so the ${service} container was not started.`, + 'If the existing service is compatible, keep using it and run the tests directly.', + `Otherwise stop that service, or choose another Docker host port with EGG_DEV_SERVICES_${service.toUpperCase()}_PORT.`, + 'Most DAL/ORM/Redis test fixtures still default to 127.0.0.1:3306 or 127.0.0.1:6379, so the default ports are required for the full local test path.', + ].join('\n'), + ); +} + +async function waitFor(command, label) { + const deadline = Date.now() + waitTimeout * 1000; + let lastOutput = ''; + + while (Date.now() < deadline) { + const result = dockerCompose(['exec', '-T', ...command], { allowFailure: true }); + lastOutput = [result.stdout, result.stderr].filter(Boolean).join('\n').trim(); + if (result.status === 0) { + return; + } + await sleep(1000); + } + + throw new Error(`${label} was not ready within ${waitTimeout}s.\n${lastOutput}`); +} + +async function initMysql() { + await waitFor(['mysql', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-uroot', '--silent'], 'MySQL'); + + const sql = databaseNames.map((name) => `CREATE DATABASE IF NOT EXISTS \`${name}\`;`).join(' '); + dockerCompose(['exec', '-T', 'mysql', 'mysql', '-uroot', '-e', sql], { stdio: 'inherit' }); +} + +async function waitRedis() { + await waitFor(['redis', 'redis-cli', 'ping'], 'Redis'); +} + +async function start() { + runDocker(['compose', 'version']); + + const running = runningServices(); + await assertPortAvailable('mysql', mysqlPort, 3306, running); + await assertPortAvailable('redis', redisPort, 6379, running); + + dockerCompose(['up', '-d'], { stdio: 'inherit' }); + await initMysql(); + await waitRedis(); + + console.log(`Local services are ready: MySQL 127.0.0.1:${mysqlPort}, Redis 127.0.0.1:${redisPort}`); +} + +function stop() { + dockerCompose(['down'], { stdio: 'inherit' }); +} + +function reset() { + dockerCompose(['down', '-v'], { stdio: 'inherit' }); +} + +function status() { + dockerCompose(['ps'], { stdio: 'inherit' }); +} + +async function main() { + const command = process.argv[2] || 'start'; + switch (command) { + case 'start': + case 'up': + await start(); + break; + case 'stop': + case 'down': + stop(); + break; + case 'reset': + reset(); + break; + case 'status': + case 'ps': + status(); + break; + default: + console.error(`Unknown command: ${command}`); + console.error('Usage: node scripts/dev-services.js '); + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; +}); From b3db4dfb8196a0d5404f761dcb8a24c6d6f44cba Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 3 May 2026 10:37:09 +0800 Subject: [PATCH 2/4] docs: use utoo for dev services commands --- README.md | 14 +++++++------- scripts/dev-services.js | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 41b83284ec..947061f041 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ pnpm --filter=site run dev Some DAL, ORM, Redis, and ecosystem benchmark paths need local MySQL and Redis services. Start the repository-aligned Docker services before running those tests on a clean machine: ```bash -pnpm run dev:services:start +utoo run dev:services:start ``` This starts MySQL 8 and Redis 7, matching the main CI service versions, and creates the databases used by local DAL/ORM/e2e fixtures: `test`, `apple`, `banana`, `test_runtime_datasource`, `test_runtime_dao`, `test_dal_plugin`, `test_dal_standalone`, `cnpmcore`, and `cnpmcore_unittest`. @@ -78,9 +78,9 @@ This starts MySQL 8 and Redis 7, matching the main CI service versions, and crea Useful commands: ```bash -pnpm run dev:services:status -pnpm run dev:services:stop -pnpm run dev:services:reset +utoo run dev:services:status +utoo run dev:services:stop +utoo run dev:services:reset ``` The default host ports are `127.0.0.1:3306` for MySQL and `127.0.0.1:6379` for Redis. If either port is already used, the start command stops before changing containers. Keep using the existing service if it is compatible with CI, or stop it and run the command again. You can change Docker host ports with `EGG_DEV_SERVICES_MYSQL_PORT` and `EGG_DEV_SERVICES_REDIS_PORT`; however, the full DAL/ORM/Redis local test path still expects the default host ports. @@ -88,11 +88,11 @@ The default host ports are `127.0.0.1:3306` for MySQL and `127.0.0.1:6379` for R Image overrides are available for compatibility checks: ```bash -EGG_DEV_SERVICES_MYSQL_IMAGE=mysql:5.7 pnpm run dev:services:start -EGG_DEV_SERVICES_REDIS_IMAGE=redis:7 pnpm run dev:services:start +EGG_DEV_SERVICES_MYSQL_IMAGE=mysql:5.7 utoo run dev:services:start +EGG_DEV_SERVICES_REDIS_IMAGE=redis:7 utoo run dev:services:start ``` -Run `pnpm run dev:services:reset` before switching MySQL image families, for example between MySQL 8 and MySQL 5.7, because MySQL data directories are not downgrade-compatible across major versions. +Run `utoo run dev:services:reset` before switching MySQL image families, for example between MySQL 8 and MySQL 5.7, because MySQL data directories are not downgrade-compatible across major versions. Current hard-coded service assumptions: diff --git a/scripts/dev-services.js b/scripts/dev-services.js index 8ab891cd9b..e268714d15 100644 --- a/scripts/dev-services.js +++ b/scripts/dev-services.js @@ -105,7 +105,7 @@ async function assertPortAvailable(service, port, containerPort, running) { throw new Error( [ `${service} is already running for this compose project on ${current}, but this run requested 127.0.0.1:${port}.`, - 'Re-run with the same EGG_DEV_SERVICES_* port override, or run pnpm run dev:services:reset before changing ports.', + 'Re-run with the same EGG_DEV_SERVICES_* port override, or run utoo run dev:services:reset before changing ports.', ].join('\n'), ); } From dc18e082977f73e90c0043fed2634b5e326e2f99 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 3 May 2026 10:49:10 +0800 Subject: [PATCH 3/4] chore: tighten dev services helper --- scripts/dev-services.js | 64 +++++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/scripts/dev-services.js b/scripts/dev-services.js index e268714d15..037064465f 100644 --- a/scripts/dev-services.js +++ b/scripts/dev-services.js @@ -7,9 +7,6 @@ import { fileURLToPath } from 'node:url'; const rootDir = dirname(dirname(fileURLToPath(import.meta.url))); const composeFile = join(rootDir, 'dev-services.compose.yml'); -const mysqlPort = Number.parseInt(process.env.EGG_DEV_SERVICES_MYSQL_PORT || '3306', 10); -const redisPort = Number.parseInt(process.env.EGG_DEV_SERVICES_REDIS_PORT || '6379', 10); -const waitTimeout = Number.parseInt(process.env.EGG_DEV_SERVICES_WAIT_TIMEOUT || '90', 10); const databaseNames = [ 'test', @@ -23,6 +20,35 @@ const databaseNames = [ 'cnpmcore_unittest', ]; +function readPositiveInteger(name, defaultValue) { + const rawValue = process.env[name]; + if (!rawValue) { + return defaultValue; + } + + const value = Number(rawValue); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer, got ${rawValue}`); + } + return value; +} + +function readPort(name, defaultValue) { + const port = readPositiveInteger(name, defaultValue); + if (port > 65535) { + throw new Error(`${name} must be between 1 and 65535, got ${port}`); + } + return port; +} + +function readConfig() { + return { + mysqlPort: readPort('EGG_DEV_SERVICES_MYSQL_PORT', 3306), + redisPort: readPort('EGG_DEV_SERVICES_REDIS_PORT', 6379), + waitTimeout: readPositiveInteger('EGG_DEV_SERVICES_WAIT_TIMEOUT', 150), + }; +} + function dockerCompose(args, options = {}) { const result = spawnSync('docker', ['compose', '-f', composeFile, ...args], { cwd: rootDir, @@ -105,7 +131,7 @@ async function assertPortAvailable(service, port, containerPort, running) { throw new Error( [ `${service} is already running for this compose project on ${current}, but this run requested 127.0.0.1:${port}.`, - 'Re-run with the same EGG_DEV_SERVICES_* port override, or run utoo run dev:services:reset before changing ports.', + 'Re-run with the same EGG_DEV_SERVICES_* port override, or use `utoo run dev:services:reset` before changing ports.', ].join('\n'), ); } @@ -123,7 +149,7 @@ async function assertPortAvailable(service, port, containerPort, running) { ); } -async function waitFor(command, label) { +async function waitFor(command, label, waitTimeout) { const deadline = Date.now() + waitTimeout * 1000; let lastOutput = ''; @@ -139,18 +165,20 @@ async function waitFor(command, label) { throw new Error(`${label} was not ready within ${waitTimeout}s.\n${lastOutput}`); } -async function initMysql() { - await waitFor(['mysql', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-uroot', '--silent'], 'MySQL'); +async function initMysql(waitTimeout) { + await waitFor(['mysql', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-uroot', '--silent'], 'MySQL', waitTimeout); const sql = databaseNames.map((name) => `CREATE DATABASE IF NOT EXISTS \`${name}\`;`).join(' '); dockerCompose(['exec', '-T', 'mysql', 'mysql', '-uroot', '-e', sql], { stdio: 'inherit' }); } -async function waitRedis() { - await waitFor(['redis', 'redis-cli', 'ping'], 'Redis'); +async function waitRedis(waitTimeout) { + await waitFor(['redis', 'redis-cli', 'ping'], 'Redis', waitTimeout); } async function start() { + const { mysqlPort, redisPort, waitTimeout } = readConfig(); + runDocker(['compose', 'version']); const running = runningServices(); @@ -158,8 +186,20 @@ async function start() { await assertPortAvailable('redis', redisPort, 6379, running); dockerCompose(['up', '-d'], { stdio: 'inherit' }); - await initMysql(); - await waitRedis(); + try { + await initMysql(waitTimeout); + await waitRedis(waitTimeout); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + [ + message, + 'The compose stack is still running so Docker health status and logs can be inspected.', + 'After fixing the issue, run `utoo run dev:services:reset` to clean up before starting again.', + ].join('\n'), + { cause: err }, + ); + } console.log(`Local services are ready: MySQL 127.0.0.1:${mysqlPort}, Redis 127.0.0.1:${redisPort}`); } @@ -196,7 +236,7 @@ async function main() { break; default: console.error(`Unknown command: ${command}`); - console.error('Usage: node scripts/dev-services.js '); + console.error('Usage: node scripts/dev-services.js '); process.exitCode = 1; } } From 5e4dd6d620e902b90d1c58745ee5ab62e8ef3252 Mon Sep 17 00:00:00 2001 From: killa Date: Sun, 3 May 2026 11:46:48 +0800 Subject: [PATCH 4/4] docs: document session redis dev service assumption --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 947061f041..ed27699833 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ Run `utoo run dev:services:reset` before switching MySQL image families, for exa Current hard-coded service assumptions: - Redis plugin fixtures under `plugins/redis/test/fixtures/apps/**/config.*` use `127.0.0.1:6379`; skipped Redis plugin tests become runnable when that port is available. +- Session Redis fixtures under `plugins/session/test/fixtures/redis-session/config/config.default.js` use `127.0.0.1:6379`. - DAL runtime tests in `tegg/core/dal-runtime/test/DataSource.test.ts` and `tegg/core/dal-runtime/test/DAO.test.ts` use local MySQL on port `3306`. - DAL module fixtures in `tegg/plugin/dal/test/fixtures/apps/dal-app/modules/dal/module.yml` and `tegg/standalone/standalone/test/fixtures/dal-*/module.yml` use local MySQL on port `3306`. - ORM fixtures in `tegg/plugin/orm/test/fixtures/prepare.js` and `tegg/plugin/orm/test/fixtures/apps/orm-app/config/config.default.ts` use local MySQL on port `3306`.