Skip to content

Commit 5b6a544

Browse files
najamansarirkistner
najamansari
and
rkistner
authored
Dynamically load connection modules for reduced memory usage (#418)
* Dynamically load connection modules for reduced memory usage Done in response to #410. Refactors module loading to dynamically import database connection and storage modules (MySQL, PostgreSQL) based on the configuration, rather than registering them all upfront. Signed-off-by: Najam Ahmed Ansari <[email protected]> * Incorporating PR feedback (cleanups) * Refactoring module loader and its test cases - Split modules between storage and connection. - Cleaner code that now checks if given connection/storage types can be handled by us or not. - Error raised if any connection types cannot be handled or if any imports fail. - Explicit test case that simulates import failure for a module. Signed-off-by: Najam Ahmed Ansari <[email protected]> * Removing MongoDB exclusions Signed-off-by: Najam Ahmed Ansari <[email protected]> * Some cleanup. * Move core module loading logic to service-core. * Skip docker login on PRs from forks. --------- Signed-off-by: Najam Ahmed Ansari <[email protected]> Co-authored-by: Ralf Kistner <[email protected]>
1 parent 291e0e5 commit 5b6a544

File tree

11 files changed

+203
-19
lines changed

11 files changed

+203
-19
lines changed

‎.changeset/perfect-plants-report.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@powersync/service-image': minor
3+
---
4+
5+
Dynamically load connection modules for reduced memory usage

‎.github/workflows/test.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ jobs:
1919
uses: actions/checkout@v5
2020

2121
- name: Login to Docker Hub
22+
if: github.event_name != 'pull_request'
2223
uses: docker/login-action@v3
2324
with:
2425
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -48,6 +49,7 @@ jobs:
4849
- uses: actions/checkout@v5
4950

5051
- name: Login to Docker Hub
52+
if: github.event_name != 'pull_request'
5153
uses: docker/login-action@v3
5254
with:
5355
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -95,6 +97,7 @@ jobs:
9597
- uses: actions/checkout@v5
9698

9799
- name: Login to Docker Hub
100+
if: github.event_name != 'pull_request'
98101
uses: docker/login-action@v3
99102
with:
100103
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -171,6 +174,7 @@ jobs:
171174
- uses: actions/checkout@v5
172175

173176
- name: Login to Docker Hub
177+
if: github.event_name != 'pull_request'
174178
uses: docker/login-action@v3
175179
with:
176180
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -244,6 +248,7 @@ jobs:
244248
- uses: actions/checkout@v4
245249

246250
- name: Login to Docker Hub
251+
if: github.event_name != 'pull_request'
247252
uses: docker/login-action@v3
248253
with:
249254
username: ${{ secrets.DOCKERHUB_USERNAME }}

‎packages/service-core/src/modules/loader.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { ResolvedPowerSyncConfig } from '../util/util-index.js';
2+
import { AbstractModule } from './AbstractModule.js';
3+
4+
interface DynamicModuleMap {
5+
[key: string]: () => Promise<AbstractModule>;
6+
}
7+
8+
export interface ModuleLoaders {
9+
storage: DynamicModuleMap;
10+
connection: DynamicModuleMap;
11+
}
12+
/**
13+
* Utility function to dynamically load and instantiate modules.
14+
*/
15+
export async function loadModules(config: ResolvedPowerSyncConfig, loaders: ModuleLoaders) {
16+
const requiredConnections = [...new Set(config.connections?.map((connection) => connection.type) || [])];
17+
const missingConnectionModules: string[] = [];
18+
const modulePromises: Promise<AbstractModule>[] = [];
19+
20+
// 1. Map connection types to their module loading promises making note of any
21+
// missing connection types.
22+
requiredConnections.forEach((connectionType) => {
23+
const modulePromise = loaders.connection[connectionType];
24+
if (modulePromise !== undefined) {
25+
modulePromises.push(modulePromise());
26+
} else {
27+
missingConnectionModules.push(connectionType);
28+
}
29+
});
30+
31+
// Fail if any connection types are not found.
32+
if (missingConnectionModules.length > 0) {
33+
throw new Error(`Invalid connection types: "${[...missingConnectionModules].join(', ')}"`);
34+
}
35+
36+
if (loaders.storage[config.storage.type] !== undefined) {
37+
modulePromises.push(loaders.storage[config.storage.type]());
38+
} else {
39+
throw new Error(`Invalid storage type: "${config.storage.type}"`);
40+
}
41+
42+
// 2. Dynamically import and instantiate module classes and resolve all promises
43+
// raising errors if any modules could not be imported.
44+
const moduleInstances = await Promise.all(modulePromises);
45+
46+
return moduleInstances;
47+
}

‎packages/service-core/src/modules/modules-index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
export * from './ModuleManager.js';
22
export * from './AbstractModule.js';
3+
export * from './loader.js';

‎packages/service-core/test/src/module-loader.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js';
2+
import { describe, expect, it, vi } from 'vitest';
3+
4+
interface MockConfig {
5+
connections?: { type: string }[];
6+
storage: { type: string };
7+
}
8+
9+
class MockMySQLModule extends AbstractModule {
10+
constructor() {
11+
super({ name: 'MySQLModule' });
12+
}
13+
async initialize(context: ServiceContextContainer): Promise<void> {}
14+
async teardown(options: TearDownOptions): Promise<void> {}
15+
}
16+
class MockPostgresModule extends AbstractModule {
17+
constructor() {
18+
super({ name: 'PostgresModule' });
19+
}
20+
async initialize(context: ServiceContextContainer): Promise<void> {}
21+
async teardown(options: TearDownOptions): Promise<void> {}
22+
}
23+
class MockPostgresStorageModule extends AbstractModule {
24+
constructor() {
25+
super({ name: 'PostgresStorageModule' });
26+
}
27+
async initialize(context: ServiceContextContainer): Promise<void> {}
28+
async teardown(options: TearDownOptions): Promise<void> {}
29+
}
30+
const mockLoaders = {
31+
connection: {
32+
mysql: async () => {
33+
return new MockMySQLModule();
34+
},
35+
postgresql: async () => {
36+
return new MockPostgresModule();
37+
}
38+
},
39+
storage: {
40+
postgresql: async () => {
41+
return new MockPostgresStorageModule();
42+
}
43+
}
44+
};
45+
46+
describe('module loader', () => {
47+
it('should load all modules defined in connections and storage', async () => {
48+
const config: MockConfig = {
49+
connections: [{ type: 'mysql' }, { type: 'postgresql' }],
50+
storage: { type: 'postgresql' }
51+
};
52+
53+
const modules = await loadModules(config as any, mockLoaders);
54+
55+
expect(modules.length).toBe(3);
56+
expect(modules[0]).toBeInstanceOf(MockMySQLModule);
57+
expect(modules[1]).toBeInstanceOf(MockPostgresModule);
58+
expect(modules[2]).toBeInstanceOf(MockPostgresStorageModule);
59+
});
60+
61+
it('should handle duplicate connection types (e.g., mysql used twice)', async () => {
62+
const config: MockConfig = {
63+
connections: [{ type: 'mysql' }, { type: 'postgresql' }, { type: 'mysql' }], // mysql duplicated
64+
storage: { type: 'postgresql' }
65+
};
66+
67+
const modules = await loadModules(config as any, mockLoaders);
68+
69+
// Expect 3 modules: mysql, postgresql, postgresql-storage
70+
expect(modules.length).toBe(3);
71+
expect(modules.filter((m) => m instanceof MockMySQLModule).length).toBe(1);
72+
expect(modules.filter((m) => m instanceof MockPostgresModule).length).toBe(1);
73+
expect(modules.filter((m) => m instanceof MockPostgresStorageModule).length).toBe(1);
74+
});
75+
76+
it('should throw an error if any modules are not found in ModuleMap', async () => {
77+
const config: MockConfig = {
78+
connections: [{ type: 'mysql' }, { type: 'redis' }],
79+
storage: { type: 'postgresql' }
80+
};
81+
82+
await expect(loadModules(config as any, mockLoaders)).rejects.toThrowError();
83+
});
84+
85+
it('should throw an error if one dynamic connection module import fails', async () => {
86+
const config: MockConfig = {
87+
connections: [{ type: 'mysql' }],
88+
storage: { type: 'postgresql' }
89+
};
90+
91+
const loaders = {
92+
connection: {
93+
mysql: async () => {
94+
throw new Error('Failed to load MySQL module');
95+
}
96+
},
97+
storage: mockLoaders.storage
98+
};
99+
100+
await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module');
101+
});
102+
});

‎service/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,4 @@
2828
"npm-check-updates": "^16.14.4",
2929
"ts-node": "^10.9.1"
3030
}
31-
}
31+
}

‎service/src/entry.ts‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,6 @@ import { container, ContainerImplementation } from '@powersync/lib-services-fram
22
import * as core from '@powersync/service-core';
33

44
import { CoreModule } from '@powersync/service-module-core';
5-
import { MongoModule } from '@powersync/service-module-mongodb';
6-
import { MongoStorageModule } from '@powersync/service-module-mongodb-storage';
7-
import { MySQLModule } from '@powersync/service-module-mysql';
8-
import { PostgresModule } from '@powersync/service-module-postgres';
9-
import { PostgresStorageModule } from '@powersync/service-module-postgres-storage';
105
import { startServer } from './runners/server.js';
116
import { startStreamRunner } from './runners/stream-worker.js';
127
import { startUnifiedRunner } from './runners/unified-runner.js';
@@ -17,14 +12,7 @@ container.registerDefaults();
1712
container.register(ContainerImplementation.REPORTER, createSentryReporter());
1813

1914
const moduleManager = new core.modules.ModuleManager();
20-
moduleManager.register([
21-
new CoreModule(),
22-
new MongoModule(),
23-
new MongoStorageModule(),
24-
new MySQLModule(),
25-
new PostgresModule(),
26-
new PostgresStorageModule()
27-
]);
15+
moduleManager.register([new CoreModule()]);
2816
// This is a bit of a hack. Commands such as the teardown command or even migrations might
2917
// want access to the ModuleManager in order to use modules
3018
container.register(core.ModuleManager, moduleManager);

‎service/src/runners/server.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { container, logger } from '@powersync/lib-services-framework';
22
import * as core from '@powersync/service-core';
3+
34
import { logBooting } from '../util/version.js';
5+
import { DYNAMIC_MODULES } from '../util/modules.js';
46

57
/**
68
* Starts an API server
@@ -9,12 +11,18 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) {
911
logBooting('API Container');
1012

1113
const config = await core.utils.loadConfig(runnerConfig);
14+
15+
const moduleManager = container.getImplementation(core.modules.ModuleManager);
16+
const modules = await core.loadModules(config, DYNAMIC_MODULES);
17+
if (modules.length > 0) {
18+
moduleManager.register(modules);
19+
}
20+
1221
const serviceContext = new core.system.ServiceContextContainer({
1322
serviceMode: core.system.ServiceContextMode.API,
1423
configuration: config
1524
});
1625

17-
const moduleManager = container.getImplementation(core.modules.ModuleManager);
1826
await moduleManager.initialize(serviceContext);
1927

2028
logger.info('Starting service...');

‎service/src/runners/stream-worker.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { container, logger } from '@powersync/lib-services-framework';
22
import * as core from '@powersync/service-core';
3+
34
import { logBooting } from '../util/version.js';
5+
import { DYNAMIC_MODULES } from '../util/modules.js';
46

57
/**
68
* Configures the replication portion on a {@link serviceContext}
@@ -20,15 +22,20 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) =
2022
logBooting('Replication Container');
2123

2224
const config = await core.utils.loadConfig(runnerConfig);
25+
26+
const moduleManager = container.getImplementation(core.modules.ModuleManager);
27+
const modules = await core.loadModules(config, DYNAMIC_MODULES);
28+
if (modules.length > 0) {
29+
moduleManager.register(modules);
30+
}
31+
2332
// Self-hosted version allows for automatic migrations
2433
const serviceContext = new core.system.ServiceContextContainer({
2534
serviceMode: core.system.ServiceContextMode.SYNC,
2635
configuration: config
2736
});
28-
2937
registerReplicationServices(serviceContext);
3038

31-
const moduleManager = container.getImplementation(core.modules.ModuleManager);
3239
await moduleManager.initialize(serviceContext);
3340

3441
// Ensure automatic migrations

‎service/src/runners/unified-runner.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as core from '@powersync/service-core';
33

44
import { logBooting } from '../util/version.js';
55
import { registerReplicationServices } from './stream-worker.js';
6+
import { DYNAMIC_MODULES } from '../util/modules.js';
67

78
/**
89
* Starts an API server
@@ -11,14 +12,19 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig)
1112
logBooting('Unified Container');
1213

1314
const config = await core.utils.loadConfig(runnerConfig);
15+
16+
const moduleManager = container.getImplementation(core.modules.ModuleManager);
17+
const modules = await core.loadModules(config, DYNAMIC_MODULES);
18+
if (modules.length > 0) {
19+
moduleManager.register(modules);
20+
}
21+
1422
const serviceContext = new core.system.ServiceContextContainer({
1523
serviceMode: core.system.ServiceContextMode.UNIFIED,
1624
configuration: config
1725
});
18-
1926
registerReplicationServices(serviceContext);
2027

21-
const moduleManager = container.getImplementation(core.modules.ModuleManager);
2228
await moduleManager.initialize(serviceContext);
2329

2430
await core.migrations.ensureAutomaticMigrations({

0 commit comments

Comments
 (0)