fix: make most of db-lib tests pass, fix some bugs

Author: kirillgroshkov

package.json

@@ -12,7 +12,7 @@
   },
   "devDependencies": {
     "@naturalcycles/dev-lib": "^12.7.5",
-    "@types/node": "^16.10.2",
+    "@types/node": "^17.0.5",
     "dotenv": "^10.0.0",
     "jest": "^27.2.4"
   },

readme.md

@@ -3,12 +3,33 @@
 > MySQL client implementing CommonDB interface
 
 [![npm](https://img.shields.io/npm/v/@naturalcycles/mysql-lib/latest.svg)](https://www.npmjs.com/package/@naturalcycles/mysql-lib)
-[![](https://circleci.com/gh/NaturalCycles/mysql-lib.svg?style=shield&circle-token=123)](https://circleci.com/gh/NaturalCycles/mysql-lib)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
 
-# Packaging
+# Quick start
 
-- `engines.node >= 10.13`: Latest Node.js LTS
-- `main: dist/index.js`: commonjs, es2018
-- `types: dist/index.d.ts`: typescript types
-- `/src` folder with source `*.ts` files included
+todo
+
+# Develop
+
+Install MySQL server+client on Mac:
+
+```shell
+
+brew install mysql
+
+# start
+
+mysql.server start
+
+# secure
+
+mysql_secure_installation
+
+# connect
+
+mysql -uroot
+
+# Default options are read from the following files in the given order:
+# /etc/my.cnf /etc/mysql/my.cnf /opt/homebrew/etc/my.cnf ~/.my.cnf
+
+```

src/mysql.db.ts

@@ -10,7 +10,7 @@ import {
   RunQueryResult,
 } from '@naturalcycles/db-lib'
 import {
-  _filterNullishValues,
+  _filterUndefinedValues,
   _mapKeys,
   _mapValues,
   _Memo,
@@ -136,7 +136,7 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
   async close(): Promise<void> {
     const pool = this.pool()
     // eslint-disable-next-line @typescript-eslint/await-thenable
-    await promisify(pool.end.bind(pool))
+    await promisify(pool.end.bind(pool))()
     this.cfg.logger.log('closed')
   }
 
@@ -191,14 +191,24 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
     _opt: MysqlDBOptions = {},
   ): Promise<RunQueryResult<OUT>> {
     const sql = dbQueryToSQLSelect(q)
-    const rows = await this.runSQL<ROW[]>({ sql })
+    const rows = (await this.runSQL<ROW[]>({ sql })).map(
+      row => _mapKeys(_filterUndefinedValues(row, true), k => mapNameFromMySQL(k)) as any,
+    )
+
+    // edge case where 0 fields are selected
+    if (q._selectedFieldNames?.length === 0) {
+      return {
+        rows: rows.map(_ => ({} as any)),
+      }
+    }
+
     return {
-      rows: rows.map(r => _mapKeys(_filterNullishValues(r, true), k => mapNameFromMySQL(k)) as any),
+      rows,
     }
   }
 
   async runSQL<RESULT>(q: QueryOptions): Promise<RESULT> {
-    if (this.cfg.logSQL) this.cfg.logger.log(q.sql, q.values)
+    if (this.cfg.logSQL) this.cfg.logger.log(...[q.sql, q.values].filter(Boolean))
 
     return await new Promise<RESULT>((resolve, reject) => {
       this.pool().query(q, (err, res) => {
@@ -232,7 +242,7 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
         new Transform({
           objectMode: true,
           transform(row, _encoding, cb) {
-            cb(null, _filterNullishValues(row, true))
+            cb(null, _filterUndefinedValues(row, true))
           },
         }),
       )
@@ -248,13 +258,13 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
 
     // Stringify object values
     const rows = _rows.map(row =>
-      _mapValues(row, v => {
+      _mapValues(row, (_k, v) => {
         return v && typeof v === 'object' && !Buffer.isBuffer(v) ? JSON.stringify(v) : v
       }),
     )
 
     // inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
-    const sqls = insertSQL(table, rows, 'INSERT', this.cfg.logger)
+    const sqls = insertSQL(table, rows, 'REPLACE', this.cfg.logger)
 
     for await (const sql of sqls) {
       await this.runSQL({ sql })

src/query.util.ts

@@ -41,16 +41,16 @@ export function dbQueryToSQLDelete(q: DBQuery<any>): string {
  */
 export function insertSQL(
   table: string,
-  records: Record<any, any>[],
+  rows: Record<any, any>[],
   verb: 'INSERT' | 'REPLACE' = 'INSERT',
   logger: CommonLogger = console,
 ): string[] {
   // INSERT INTO table_name (column1, column2, column3, ...)
   // VALUES (value1, value2, value3, ...);
 
   // eslint-disable-next-line unicorn/no-array-reduce
-  const fieldSet = records.reduce((set: Set<string>, rec) => {
-    Object.keys(rec).forEach(field => set.add(field))
+  const fieldSet = rows.reduce((set: Set<string>, row) => {
+    Object.keys(row).forEach(field => set.add(field))
     return set
   }, new Set<string>())
   const fields = [...fieldSet]
@@ -63,7 +63,7 @@ export function insertSQL(
     `VALUES\n`,
   ].join(' ')
 
-  const valueRows = records.map(rec => {
+  const valueRows = rows.map(rec => {
     return `(` + fields.map(k => mysql.escape(rec[k])).join(',') + `)`
   })
 

src/schema/mysql.schema.util.ts

@@ -106,7 +106,7 @@ export function mysqlTableStatsToJsonSchemaField<T = any>(
       s.properties[name] = { instanceof: 'Buffer' }
     } else if (t.startsWith('tinyint') || t.includes('(1)')) {
       s.properties[name] = { type: 'boolean' } as JsonSchemaBoolean
-    } else if (t.startsWith('int(')) {
+    } else if (t === 'int' || t.startsWith('int(')) {
       s.properties[name] = { type: 'integer' } as JsonSchemaNumber
     } else if (t.startsWith('float')) {
       s.properties[name] = { type: 'number' } as JsonSchemaNumber

src/test/mysql.manual.test.ts

@@ -1,10 +1,12 @@
 import { DBQuery } from '@naturalcycles/db-lib'
 import {
+  createTestItemDBM,
   createTestItemsDBM,
   getTestItemSchema,
   runCommonDaoTest,
   runCommonDBTest,
   TEST_TABLE,
+  TestItemDBM,
 } from '@naturalcycles/db-lib/dist/testing'
 import { deflateString, inflateToString, requireEnvKeys } from '@naturalcycles/nodejs-lib'
 import { MysqlDB } from '../mysql.db'
@@ -51,6 +53,24 @@ test('getTableSchema', async () => {
   expect(getTestItemSchema()).toMatchObject(schema)
 })
 
+test('saveBatch overwrite', async () => {
+  const items = createTestItemsDBM(1)
+  // should not throw
+  await db.saveBatch(TEST_TABLE, items)
+  await db.saveBatch(TEST_TABLE, items)
+})
+
+test('null values', async () => {
+  const item3 = {
+    ...createTestItemDBM(3),
+    k2: null,
+  }
+  await db.saveBatch(TEST_TABLE, [item3])
+  const item3Loaded = (await db.getByIds<TestItemDBM>(TEST_TABLE, [item3.id]))[0]!
+  console.log(item3Loaded)
+  // expect(item3Loaded.k2).toBe(null)
+})
+
 test('emojis', async () => {
   const items = createTestItemsDBM(5).map(r => ({ ...r, k1: `😣` }))
   // should not throw
@@ -97,7 +117,7 @@ test('stringify objects', async () => {
   await db.createTable(TEST_TABLE, getTestItemSchema(), { dropIfExists: true })
   await db.saveBatch(TEST_TABLE, [item])
   const { rows } = await db.runQuery(new DBQuery(TEST_TABLE))
-  // console.log(records)
+  // console.log(rows)
   expect(rows).toEqual([
     {
       ...item,

src/test/setupJest.ts

@@ -0,0 +1,5 @@
+import { jestLog, jestLogger } from '@naturalcycles/dev-lib/dist/testing'
+
+// Patch console functions so jest doesn't log it so verbose
+console.log = console.warn = jestLog
+console.error = jestLogger.error.bind(jestLogger)