RocksDB storage #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

kriszyp wants to merge 123 commits into main from rocksdb

components/componentLoader.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -38,6 +38,7 @@ @@
     import { PluginModule } from './PluginModule.ts';
     import { platform } from 'node:os';
     import { getEnvBuiltInComponents } from './Application.ts';
+    import { RocksDatabase } from '@harperfast/rocksdb-js';
     const CF_ROUTES_DIR = resolvePath(env.get(CONFIG_PARAMS.COMPONENTSROOT));
     let loadedComponents = new Map<any, any>();
@@ Expand Down Expand Up / @@ -74,7 +75,7 @@ @@
     	});
     }
     export const TRUSTED_RESOURCE_PLUGINS = {
     	REST, // for backwards compatibility with older configs
     	rest: REST,
     	graphql: graphqlQueryHandler,
@@ Expand Down Expand Up / @@ -116,16 +117,20 @@ @@
     	return new Promise<void>((resolve, reject) => {
     		// Create timeout to avoid deadlocks
     		const timeout = setTimeout(() => {
-    			Status.primaryStore.unlock(componentDirectory, 0);
+    			store.unlock(componentDirectory);
     			reject(new Error('symlinking harperdb module timed out'));
     		}, 10_000);
-    		if (
-    			// Get lock for this component
-    			Status.primaryStore.attemptLock(componentDirectory, 0, () => {
-    				clearTimeout(timeout);
-    				resolve();
-    			})
-    		) {
+    		const callback = () => {
+    			clearTimeout(timeout);
+    			resolve();
+    		};
+    		const store = Status.primaryStore;
+    		const lockAcquired = store.tryLock(componentDirectory, callback);
+    		if (!lockAcquired) {
+    			clearTimeout(timeout);
+    		} else {
     			try {
     				// validate node_modules directory exists
     				const nodeModulesDir = join(componentDirectory, 'node_modules');
@@ Expand All / @@ -151,7 +156,7 @@ @@
     				resolve();
     			} finally {
     				// finally release the lock
-    				Status.primaryStore.unlock(componentDirectory, 0);
+    				store.unlock(componentDirectory);
     			}
     		}
     	});
@@ Expand All / @@ -175,12 +180,14 @@ @@
     			throw new Error(`Invalid timeout value for ${scope.name}. Expected a number, received: ${typeof timeout}`);
     		}
     		let whenResolved, timer;
-    		if (
-    			!Status.primaryStore.attemptLock(scope.name, 0, () => {
-    				clearTimeout(timer);
-    				whenResolved(sequentiallyHandleApplication(scope, plugin));
-    			})
-    		) {
+    		const callback = () => {
+    			clearTimeout(timer);
+    			whenResolved(sequentiallyHandleApplication(scope, plugin));
+    		};
+    		const store = Status.primaryStore;
+    		const lockAcquired = store.tryLock(scope.name, callback);
+    		if (!lockAcquired) {
     			return new Promise((resolve, reject) => {
     				whenResolved = resolve;
     				timer = setTimeout(() => {
@@ Expand All / @@ -206,7 +213,7 @@ @@
     				),
     			]);
     		} finally {
-    			Status.primaryStore.unlock(scope.name, 0);
+    			Status.primaryStore.unlock(scope.name);
     			clearTimeout(loadTimeout);
     		}
     	});
@@ Expand Down @@

config/configUtils.js

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -14,13 +14,14 @@ const { handleHDBError } = require('../utility/errors/hdbError.js');
  
    const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.js');

    const { server } = require('../server/Server.ts');

    const { getBackupDirPath } = require('./configHelpers.ts');

    const { PACKAGE_ROOT } = require('../utility/packageUtils');

    const { DATABASES_PARAM_CONFIG, CONFIG_PARAMS, CONFIG_PARAM_MAP } = hdbTerms;

    const UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized';

    const CONFIG_INIT_MSG = 'Config successfully initialized';

    const BACKUP_ERR = 'Error backing up config file';

    const EMPTY_GET_VALUE = 'Empty parameter sent to getConfigValue';

    const DEFAULT_CONFIG_FILE_PATH = path.join(__dirname, '../../static', hdbTerms.HDB_DEFAULT_CONFIG_FILE);

    const DEFAULT_CONFIG_FILE_PATH = path.join(PACKAGE_ROOT, 'static', hdbTerms.HDB_DEFAULT_CONFIG_FILE);

    const CONFIGURE_SUCCESS_RESPONSE =

    	'Configuration successfully set. You must restart Harper for new config settings to take effect.';

dataLayer/harperBridge/ResourceBridge.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -66,7 +66,7 @@ @@
     		}
     		return table.search(
     			{
     				conditions: searchObject.conditions,
     				//set the operator to always be lowercase for later evaluations
     				operator: searchObject.operator ? searchObject.operator.toLowerCase() : undefined,
@@ Expand Down Expand Up / @@ -124,7 +124,7 @@ @@
     		});
     	}
     	async createAttribute(createAttributeObj) {
     		await getTable(createAttributeObj).addAttributes([
     			{
     				name: createAttributeObj.attribute,
@@ Expand All / @@ -144,15 +144,7 @@ @@
     			const deleteRecord = (key, record, version): Promise<void> => {
     				record = { ...record };
     				delete record[property];
-    				return Table.primaryStore
-    					.ifVersion(key, version, () => Table.primaryStore.put(key, record, version))
-    					.then((success) => {
-    						if (!success) {
-    							// try again with the latest record
-    							const { value: record, version } = Table.primaryStore.getEntry(key);
-    							return deleteRecord(key, record, version);
-    						}
-    					});
+    				return Table.primaryStore.put(key, record, version);
     			};
     			for (const { key, value: record, version } of Table.primaryStore.getRange({ start: true, versions: true })) {
     				resolution = deleteRecord(key, record, version);
@@ Expand Down Expand Up / @@ -459,7 +451,7 @@ @@
     	}
     	resetReadTxn(schema, table) {
-    		getTable({ schema, table })?.primaryStore.resetReadTxn();
+    		getTable({ schema, table })?.primaryStore.resetReadTxn?.();
     	}
     	async deleteAuditLogsBefore(deleteObj) {
@@ Expand Down @@

dataLayer/schemaDescribe.js

-Original file line number
+Diff line change
@@ Expand Up / @@ -185,7 +185,7 @@ async function descTable(describeTableObject, attrPerms) { @@
     	}
     	let db_size;
     	try {
-    		db_size = (await fs.stat(tableObj.primaryStore.env.path)).size;
+    		db_size = (await fs.stat(tableObj.primaryStore.path ?? tableObj.primaryStore.env.path)).size;
     	} catch (error) {
     		logger.warn(`unable to get database size`, error);
     	}
@@ Expand Down @@

integrationTests/apiTests/tests/8_deleteTests.mjs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -292,7 +292,6 @@ describe('8. Delete Tests', () => {
  
    			})

    			.expect((r) => assert.equal(r.body.message, "table 'drop_schema.drop_table' successfully created.", r.text))

    			.expect(200);

    		await setTimeout(2000);

    	});

    	it('Confirm correct attributes', () => {

    @@ -717,15 +716,15 @@ describe('8. Delete Tests', () => {
  
    	it('create schema drop_attr', () => {

    		return req()

    			.send({ operation: 'create_schema', schema: 'drop_attr' })

    			.expect((r) => assert.ok(r.body.message.includes('successfully created'), r.text))

    			.expect(200);

    			.expect(200)

    			.expect((r) => assert.ok(r.body.message.includes('successfully created'), r.text));

    	});

    	it('create table test', async () => {

    		await req()

    			.send({ operation: 'create_table', schema: 'drop_attr', table: 'test', hash_attribute: 'id' })

    			.expect((r) => assert.ok(r.body.message.includes('successfully created'), r.text))

    			.expect(200);

    			.expect(200)

    			.expect((r) => assert.ok(r.body.message.includes('successfully created'), r.text));

    		await setTimeout(2000);

    	});

package.json

-Original file line number
+Diff line change
@@ Expand Up / @@ -109,9 +109,9 @@ @@
     		"chai": "^6.2.0",
     		"chai-as-promised": "^8.0.2",
     		"chai-integer": "^0.1.0",
+    		"eventsource": "^3.0.5",
     		"fast-glob": "^3.3.3",
     		"fs-extra": "^11.3.2",
-    		"eventsource": "^3.0.5",
     		"globals": "^16.5.0",
     		"intercept-stdout": "0.1.2",
     		"mkcert": "^3.2.0",
@@ Expand All / @@ -138,6 +138,8 @@ @@
     		"@fastify/compress": "~6.5.0",
     		"@fastify/cors": "~9.0.1",
     		"@fastify/static": "~7.0.4",
+    		"@harperfast/extended-iterable": "^1.0.1",
+    		"@harperfast/rocksdb-js": "^0.1.3",
     		"@turf/area": "6.5.0",
     		"@turf/boolean-contains": "6.5.0",
     		"@turf/boolean-disjoint": "6.5.0",
@@ Expand Down Expand Up / @@ -172,7 +174,7 @@ @@
     		"json2csv": "5.0.7",
     		"jsonata": "1.8.7",
     		"jsonwebtoken": "9.0.3",
-    		"lmdb": "3.4.4",
+    		"lmdb": "3.5.1",
     		"lodash": "4.17.21",
     		"mathjs": "11.12.0",
     		"micromatch": "^4.0.8",
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

RocksDB storage #37

Uh oh!

Diff view

Diff view

There are no files selected for viewing

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

Uh oh!

Uh oh!

RocksDB storage #37

Are you sure you want to change the base?

Uh oh!

RocksDB storage #37

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

GitHub Actions / Build Harper (Node.js v20)

GitHub Actions / Build Harper (Node.js v22)

GitHub Actions / Build Harper (Node.js v24)

GitHub Actions / Unit Test (Node.js v22)

GitHub Actions / Unit Test (Node.js v24)

GitHub Actions / Unit Test (Node.js v20)

Uh oh!

Uh oh!