feat: default stream concurrency to 32 (up from 16)

Author: kirillgroshkov

src/adapter/file/localFile.persistence.plugin.ts

@@ -71,7 +71,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   }
 
   async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
-    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 16 })
+    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 32 })
   }
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {

src/commondao/common.dao.model.ts

@@ -321,7 +321,7 @@ export interface CommonDaoStreamOptions<IN>
 
   /**
    * When chunkSize is set - this option controls how many chunks to run concurrently.
-   * Defaults to 16, "the magic number of JavaScript concurrency".
+   * Defaults to 32.
    */
   chunkConcurrency?: number
 }

src/commondao/common.dao.ts

@@ -920,7 +920,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
    * "Streaming" is implemented by buffering incoming rows into **batches**
    * (of size opt.chunkSize, which defaults to 500),
    * and then executing db.saveBatch(chunk) with the concurrency
-   * of opt.chunkConcurrency (which defaults to 16).
+   * of opt.chunkConcurrency (which defaults to 32).
    */
   streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
     this.requireWriteAccess()
@@ -936,7 +936,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
     const { beforeSave } = this.cfg.hooks!
 
-    const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt
+    const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt
 
     return [
       transformMap<BM, DBM>(
@@ -1019,7 +1019,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     let deleted = 0
 
     if (opt.chunkSize) {
-      const { chunkSize, chunkConcurrency = 16 } = opt
+      const { chunkSize, chunkConcurrency = 32 } = opt
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),

src/kv/commonKeyValueDao.ts

@@ -224,7 +224,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }
@@ -248,7 +248,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }

yarn.lock

@@ -884,9 +884,9 @@
     zod "^3.20.2"
 
 "@naturalcycles/nodejs-lib@^13.0.1", "@naturalcycles/nodejs-lib@^13.0.2", "@naturalcycles/nodejs-lib@^13.1.1":
-  version "13.19.0"
-  resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.19.0.tgz#c5705a2636718e85b6aef8cb40fe19500d18d5ea"
-  integrity sha512-kc7Sx95amibbj6jHTej7UC1z1BVNBrSraEurcCWbKBnJWBAsp2f52yc5stnehOwD9J8cXm7DJtWdVdwwArpkiw==
+  version "13.20.0"
+  resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.20.0.tgz#8d97cef2f685b387559d5be78d91f942d411d92f"
+  integrity sha512-qUmYEK2m58N2MwMXri+zqL/bJvc58FqBx1aV31Xtu6BiGMyRUZrqls+SuYCUW+/oJj9ULWFcBL08AHSbLnWC9A==
   dependencies:
     "@naturalcycles/js-lib" "^14.0.0"
     "@types/js-yaml" "^4.0.9"