diff --git a/bfx-report-ui b/bfx-report-ui index a68255e32..5525aea9c 160000 --- a/bfx-report-ui +++ b/bfx-report-ui @@ -1 +1 @@ -Subproject commit a68255e3243738fedde64de5a75a00557639d8c1 +Subproject commit 5525aea9c9e349cf14bf7ce4e1aeab8296de80b8 diff --git a/package-lock.json b/package-lock.json index 89c7b358b..49a55c05c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bitfinex/bfx-reports-framework", - "version": "5.1.0", + "version": "5.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@bitfinex/bfx-reports-framework", - "version": "5.1.0", + "version": "5.2.0", "license": "Apache-2.0", "dependencies": { "@bitfinex/bfx-facs-db-better-sqlite": "git+https://github.com/bitfinexcom/bfx-facs-db-better-sqlite.git", @@ -16,9 +16,8 @@ "@bitfinex/lib-js-util-base": "git+https://github.com/bitfinexcom/lib-js-util-base.git", "bignumber.js": "9.1.2", "csv": "5.5.3", - "grenache-nodejs-ws": "git+https://github.com/bitfinexcom/grenache-nodejs-ws.git", + "grenache-nodejs-ws": "1.0.0", "inversify": "6.0.1", - "lodash": "4.17.23", "mathjs": "14.8.1", "moment": "2.29.4", "puppeteer": "24.1.0", @@ -27,11 +26,11 @@ }, "devDependencies": { "@bitfinex/bfx-report-express": "git+https://github.com/bitfinexcom/bfx-report-express.git", - "bfx-api-mock-srv": "git+https://github.com/bitfinexcom/bfx-api-mock-srv.git", + "bfx-api-mock-srv": "2.0.0", "chai": "4.3.4", "concurrently": "9.2.1", "cross-env": "10.1.0", - "grenache-grape": "git+https://github.com/bitfinexcom/grenache-grape.git", + "grenache-grape": "1.0.0", "mocha": "11.1.0", "nodemon": "3.1.14", "standard": "17.0.0", @@ -235,8 +234,8 @@ } }, "node_modules/@bitfinex/bfx-report": { - "version": "5.1.0", - "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report.git#762548c2441e95c81f0f86765399984440531cd3", + "version": "5.2.0", + "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report.git#77b84149109290ce8b076d5eac145fe4b0039173", "license": "Apache-2.0", "dependencies": { "@bitfinex/bfx-facs-deflate": "git+https://github.com/bitfinexcom/bfx-facs-deflate.git", @@ -270,8 +269,8 @@ } }, "node_modules/@bitfinex/bfx-report-express": { - "version": "3.1.0", - "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report-express.git#510163a94336f889364da7b248103df804ef3a05", + "version": "3.1.1", + "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report-express.git#e3d966cbaf1f05af4d1ed84ed25b79bae86ba88a", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -1403,9 +1402,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", - "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz", + "integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -5645,9 +5644,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, "node_modules/lodash.defaults": { diff --git a/package.json b/package.json index 2fc3219fa..97d0bbe00 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@bitfinex/bfx-reports-framework", - "version": "5.1.0", + "version": "5.2.0", "description": "Bitfinex reports framework", "main": "worker.js", "engines": { @@ -15,9 +15,8 @@ "@bitfinex/lib-js-util-base": "git+https://github.com/bitfinexcom/lib-js-util-base.git", "bignumber.js": "9.1.2", "csv": "5.5.3", - "grenache-nodejs-ws": "git+https://github.com/bitfinexcom/grenache-nodejs-ws.git", + "grenache-nodejs-ws": "1.0.0", "inversify": "6.0.1", - "lodash": "4.17.23", "mathjs": "14.8.1", "moment": "2.29.4", "puppeteer": "24.1.0", @@ -26,11 +25,11 @@ }, "devDependencies": { "@bitfinex/bfx-report-express": "git+https://github.com/bitfinexcom/bfx-report-express.git", - "bfx-api-mock-srv": "git+https://github.com/bitfinexcom/bfx-api-mock-srv.git", + "bfx-api-mock-srv": "2.0.0", "chai": "4.3.4", "concurrently": "9.2.1", "cross-env": "10.1.0", - "grenache-grape": "git+https://github.com/bitfinexcom/grenache-grape.git", + "grenache-grape": "1.0.0", "mocha": "11.1.0", "nodemon": "3.1.14", "standard": "17.0.0", diff --git a/workers/api.framework.report.wrk.js b/workers/api.framework.report.wrk.js index 4f21625c5..1fe06a69a 100644 --- a/workers/api.framework.report.wrk.js +++ b/workers/api.framework.report.wrk.js @@ -36,7 +36,8 @@ const argv = require('yargs') .argv const { - PDFBufferUnderElectronCreationError + PDFBufferUnderElectronCreationError, + PDFBufferUnderFrameworkCreationError } = require('@bitfinex/bfx-report/workers/loc.api/errors') const appDeps = require('./loc.api/di/app.deps') @@ -208,7 +209,10 @@ class WrkReportFrameWorkApi extends WrkReportServiceApi { await wsTransport.start() processorQueue.on('error:base', (err, job) => { - if (!(err instanceof PDFBufferUnderElectronCreationError)) { + if ( + !(err instanceof PDFBufferUnderElectronCreationError) && + !(err instanceof PDFBufferUnderFrameworkCreationError) + ) { return } diff --git a/workers/loc.api/di/factories/helpers/get-migration-file-metadata.js b/workers/loc.api/di/factories/helpers/get-migration-file-metadata.js index fd105ed4a..02db0c6bd 100644 --- a/workers/loc.api/di/factories/helpers/get-migration-file-metadata.js +++ b/workers/loc.api/di/factories/helpers/get-migration-file-metadata.js @@ -1,6 +1,6 @@ 'use strict' -const { orderBy } = require('lodash') +const { orderBy } = require('../../../helpers') module.exports = (migrationFileDirents) => { const metadata = migrationFileDirents.reduce((accum, dirent) => { diff --git a/workers/loc.api/generate-report-file/pdf-writer/index.js b/workers/loc.api/generate-report-file/pdf-writer/index.js index 46c1f37cb..fb8eccc56 100644 --- a/workers/loc.api/generate-report-file/pdf-writer/index.js +++ b/workers/loc.api/generate-report-file/pdf-writer/index.js @@ -16,7 +16,8 @@ const { const TEMPLATE_FILE_NAMES = require('./template-file-names') const { - PDFBufferUnderElectronCreationError + PDFBufferUnderElectronCreationError, + PDFBufferUnderFrameworkCreationError } = require('@bitfinex/bfx-report/workers/loc.api/errors') const { decorateInjectable } = require('../../di/utils') @@ -59,43 +60,51 @@ class PdfWriter extends MainPdfWriter { * @override */ async createPDFBuffer (args) { - const { - template = 'No data', - format = 'portrait', - orientation = 'Letter' - } = args ?? {} + const _args = { + template: 'No data', + format: 'portrait', + orientation: 'Letter', + timeout: 10 * 60 * 1000, + ...args + } if (this.isElectronjsEnv) { - return await this.createPDFBufferUnderElectron({ - template, - format, - orientation - }) + return await this.createPDFBufferUnderElectron(_args) } - const browser = await puppeteer.launch({ - args: ['--no-sandbox', '--disable-gpu'] - }) - const page = await browser.newPage() - await page.setContent(template, { - waitUntil: 'domcontentloaded' - }) - await page.emulateMediaType('print') - const u8ArrayPdf = await page.pdf({ - landscape: format !== 'portrait', - format: orientation, - margins: { - top: 0, - bottom: 0, - left: 0, - right: 0 - }, - displayHeaderFooter: true, - footerTemplate: this.#getFooterTemplate(args) - }) - await browser.close() + return await this.createPDFBufferUnderFramework(_args) + } + + async createPDFBufferUnderFramework (args) { + try { + const browser = await puppeteer.launch({ + args: ['--no-sandbox', '--disable-gpu'], + protocolTimeout: args?.timeout, + timeout: 60_000 + }) + const page = await browser.newPage() + await page.setContent(args?.template, { + waitUntil: 'domcontentloaded' + }) + await page.emulateMediaType('print') + const u8ArrayPdf = await page.pdf({ + landscape: args?.format !== 'portrait', + format: args?.orientation, + margins: { + top: 0, + bottom: 0, + left: 0, + right: 0 + }, + displayHeaderFooter: true, + footerTemplate: this.#getFooterTemplate(args) + }) + await browser.close() - return Buffer.from(u8ArrayPdf) + return Buffer.from(u8ArrayPdf) + } catch (err) { + throw new PDFBufferUnderFrameworkCreationError(err) + } } async createPDFBufferUnderElectron (args) { @@ -117,7 +126,8 @@ class PdfWriter extends MainPdfWriter { templateFilePath: uniqueFileName, format: args?.format, orientation: args?.orientation, - uid + uid, + timeout: args?.timeout } ) diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index 0c4751655..88e55a20b 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -11,7 +11,8 @@ const { sumAllObjectsNumbers, pickAllLowerObjectsNumbers, sumArrayVolumes, - pushLargeArr + pushLargeArr, + orderBy } = require('./utils') const { isSubAccountApiKeys, @@ -35,5 +36,6 @@ module.exports = { pickAllLowerObjectsNumbers, sumArrayVolumes, pushLargeArr, + orderBy, isBfxApiStaging } diff --git a/workers/loc.api/helpers/utils.js b/workers/loc.api/helpers/utils.js deleted file mode 100644 index 10ac2aa0c..000000000 --- a/workers/loc.api/helpers/utils.js +++ /dev/null @@ -1,287 +0,0 @@ -'use strict' - -const { pick } = require('@bitfinex/lib-js-util-base') - -const { - AuthError -} = require('@bitfinex/bfx-report/workers/loc.api/errors') - -const { - isUpdatableArr, - isUpdatableArrObj -} = require('../sync/schema/utils') - -const checkParamsAuth = (args) => { - const { - apiKey, - apiSecret, - authToken, - isSubAccount - } = args?.auth ?? {} - - if ( - !isSubAccount && - authToken - ) { - return - } - if ( - !apiKey || - typeof apiKey !== 'string' || - !apiSecret || - typeof apiSecret !== 'string' - ) { - throw new AuthError() - } -} - -const tryParseJSON = ( - jsonString, - isNotObject -) => { - try { - if (typeof jsonString !== 'string') { - return false - } - - const obj = JSON.parse(jsonString) - - if ( - isNotObject || - ( - obj && - typeof obj === 'object' - ) - ) { - return obj - } - } catch (e) { } - - return false -} - -const collObjToArr = (coll = [], opts) => { - const { - projection, - type - } = opts ?? {} - const _projection = Array.isArray(projection) - ? projection - : [projection] - const isProjectionExisted = ( - _projection[0] && - typeof _projection[0] === 'string' - ) - - const res = [] - - if (isUpdatableArr(type)) { - const fieldName = isProjectionExisted - ? _projection[0] - : null - - for (const obj of coll) { - if ( - !obj || - typeof obj !== 'object' - ) { - continue - } - - const _fieldName = fieldName ?? Object.keys(obj) - .filter((key) => key !== '_id')[0] - - if (typeof obj?.[_fieldName] === 'undefined') { - continue - } - - res.push(obj?.[_fieldName]) - } - } - if (isUpdatableArrObj(type)) { - if (!isProjectionExisted) { - return coll - } - - for (const obj of coll) { - if ( - !obj || - typeof obj !== 'object' - ) { - continue - } - - res.push(pick(obj, projection)) - } - } - - return res -} - -const getDateString = (mc) => { - return new Date(mc).toDateString().split(' ').join('-') -} - -const isNotSyncRequired = (args) => { - return ( - args && - typeof args === 'object' && - args.params && - typeof args.params === 'object' && - args.params.isNotSyncRequired - ) -} - -const sumObjectsNumbers = (propName, objects = []) => { - return objects.reduce((accum, curr) => { - return Number.isFinite(curr?.[propName]) - ? accum + curr[propName] - : accum - }, 0) -} - -const pickLowerObjectsNumbers = (propName, objects = []) => { - return objects.reduce((accum, curr) => { - if (!Number.isFinite(curr?.[propName])) { - return accum - } - if (!Number.isFinite(accum)) { - return curr[propName] - } - - return curr[propName] < accum - ? curr[propName] - : accum - }, null) -} - -const pickAllLowerObjectsNumbers = (propName, objects = []) => { - return objects.reduce((accum, curr) => { - if ( - !curr?.[propName] || - typeof curr?.[propName] !== 'object' - ) { - return accum - } - - const entries = Object.entries(curr[propName]) - - return entries.reduce((accum, [key, val]) => { - if (!Number.isFinite(val)) { - return accum - } - if (!Number.isFinite(accum?.[key])) { - accum[key] = val - - return accum - } - - accum[key] = val < accum[key] - ? val - : accum[key] - - return accum - }, accum) - }, {}) -} - -const sumAllObjectsNumbers = (propName, objects = []) => { - return objects.reduce((accum, curr) => { - if ( - !curr?.[propName] || - typeof curr?.[propName] !== 'object' - ) { - return accum - } - - const entries = Object.entries(curr[propName]) - - return entries.reduce((accum, [key, val]) => { - const prevVal = Number.isFinite(accum?.[key]) - ? accum[key] - : 0 - const currVal = Number.isFinite(val) - ? val - : 0 - - accum[key] = prevVal + currVal - - return accum - }, accum) - }, {}) -} - -const sumArrayVolumes = (propName, objects = []) => { - return objects.reduce((accum, curr) => { - if (!Array.isArray(curr?.[propName])) { - return accum - } - - for (const obj of curr[propName]) { - if (typeof obj?.curr !== 'string') { - continue - } - - const entries = Object.entries(obj) - .filter(([key]) => key !== 'curr') - - if (entries.length === 0) { - continue - } - - if (accum.length === 0) { - accum.push({ ...obj }) - - continue - } - - const accumObjIndex = accum - .findIndex((item) => item?.curr === obj.curr) - - if (accumObjIndex === -1) { - accum.push({ ...obj }) - - continue - } - - const resObj = entries.reduce((accum, [key, vol]) => { - const accumVol = Number.isFinite(accum?.[key]) - ? accum[key] - : 0 - const currVol = Number.isFinite(vol) - ? vol - : 0 - - accum[key] = accumVol + currVol - - return accum - }, accum[accumObjIndex]) - - // For right order in resulting array - accum.splice(accumObjIndex, 1) - accum.push(resObj) - } - - return accum - }, []) -} - -const pushLargeArr = (dest, src) => { - for (const item of src) { - dest.push(item) - } -} - -module.exports = { - checkParamsAuth, - tryParseJSON, - collObjToArr, - getDateString, - isNotSyncRequired, - sumObjectsNumbers, - pickLowerObjectsNumbers, - sumAllObjectsNumbers, - pickAllLowerObjectsNumbers, - sumArrayVolumes, - pushLargeArr -} diff --git a/workers/loc.api/helpers/utils/__test__/order-by.spec.js b/workers/loc.api/helpers/utils/__test__/order-by.spec.js new file mode 100644 index 000000000..8dfa2ac37 --- /dev/null +++ b/workers/loc.api/helpers/utils/__test__/order-by.spec.js @@ -0,0 +1,350 @@ +'use strict' + +const { assert } = require('chai') + +const orderBy = require('../order-by') + +describe('orderBy util', () => { + it('Order array of objects by name prop in desc', () => { + const mockedArr = [ + { name: 'aaa' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['desc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'aaa' } + ]) + }) + + it('Order array of objects by name prop in asc', () => { + const mockedArr = [ + { name: 'aaa' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['asc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'aaa' }, + { name: 'bbb' }, + { name: 'www' }, + { name: 'yyy' } + ]) + }) + + it('Order array of objects by name and value props in desc and asc', () => { + const mockedArr = [ + { name: 'aaa', value: 't' }, + { name: 'www', value: 'b' }, + { name: 'bbb', value: 'f' }, + { name: 'www', value: 'a' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy( + mockedArr, + ['name', 'value'], + ['desc', 'asc'] + ) + + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www', value: 'a' }, + { name: 'www', value: 'b' }, + { name: 'bbb', value: 'f' }, + { name: 'aaa', value: 't' } + ]) + }) + + it('Order array of objects by name and value props in asc by default', () => { + const mockedArr = [ + { name: 'aaa', value: 't' }, + { name: 'www', value: 'b' }, + { name: 'bbb', value: 'f' }, + { name: 'www', value: 'a' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy( + mockedArr, + ['name', 'value'] + ) + + assert.deepStrictEqual(orderedArr, [ + { name: 'aaa', value: 't' }, + { name: 'bbb', value: 'f' }, + { name: 'www', value: 'a' }, + { name: 'www', value: 'b' }, + { name: 'yyy' } + ]) + }) + + it('Order array of objects by name prop in desc using dot syntax', () => { + const mockedArr = [ + { nestedObj: { name: 'aaa' } }, + { nestedObj: { name: 'www' } }, + { nestedObj: { name: 'bbb' } }, + { nestedObj: { name: 'yyy' } } + ] + + const orderedArr = orderBy( + mockedArr, + ['nestedObj.name'], + ['desc'] + ) + + assert.deepStrictEqual(orderedArr, [ + { nestedObj: { name: 'yyy' } }, + { nestedObj: { name: 'www' } }, + { nestedObj: { name: 'bbb' } }, + { nestedObj: { name: 'aaa' } } + ]) + }) + + it('Order array of objects by second item of array in desc using fn', () => { + const mockedArr = [ + { nestedObj: { arr: [0, 'aaa'] } }, + { nestedObj: { arr: [0, 'www'] } }, + { nestedObj: { arr: [0, 'bbb'] } }, + { nestedObj: { arr: [0, 'yyy'] } } + ] + + const orderedArr = orderBy( + mockedArr, + [(item) => item?.nestedObj?.arr?.[1]], + ['desc'] + ) + + assert.deepStrictEqual(orderedArr, [ + { nestedObj: { arr: [0, 'yyy'] } }, + { nestedObj: { arr: [0, 'www'] } }, + { nestedObj: { arr: [0, 'bbb'] } }, + { nestedObj: { arr: [0, 'aaa'] } } + ]) + }) + + it('Order array of array by index in desc', () => { + const mockedArr = [ + ['name', 'aaa'], + ['name', 'www'], + ['name', 'bbb'], + ['name', 'yyy'] + ] + + const orderedArr = orderBy(mockedArr, [1], ['desc']) + + assert.deepStrictEqual(orderedArr, [ + ['name', 'yyy'], + ['name', 'www'], + ['name', 'bbb'], + ['name', 'aaa'] + ]) + }) + + it('Order array by index in desc', () => { + const mockedArr = ['aaa', 'www', 'bbb', 'yyy'] + + const orderedArr = orderBy(mockedArr, [(item) => item], ['desc']) + + assert.deepStrictEqual(orderedArr, ['yyy', 'www', 'bbb', 'aaa']) + }) + + it('Order iterable object (Set) of objects by name prop in desc', () => { + const mockedSet = new Set([ + { name: 'aaa' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'yyy' } + ]) + + const orderedArr = orderBy(mockedSet, ['name'], ['desc']) + + // If iterable object is required, just create it after ordering + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'aaa' } + ]) + }) + + it('Order iterable object (Map) of objects by name prop and key in desc', () => { + const mockedMap = new Map([ + ['key1', { name: 'aaa' }], + ['key2', { name: 'www' }], + ['key3', { name: 'bbb' }], + ['key4', { name: 'yyy' }] + ]) + + const orderedArrByVal = orderBy( + mockedMap, + [([key, val]) => val?.name], + ['desc'] + ) + const orderedArrByKey = orderBy( + mockedMap, + [([key, val]) => key], + ['desc'] + ) + + // If iterable object is required, just create it after ordering + assert.deepStrictEqual(orderedArrByVal, [ + ['key4', { name: 'yyy' }], + ['key2', { name: 'www' }], + ['key3', { name: 'bbb' }], + ['key1', { name: 'aaa' }] + ]) + assert.deepStrictEqual(orderedArrByKey, [ + ['key4', { name: 'yyy' }], + ['key3', { name: 'bbb' }], + ['key2', { name: 'www' }], + ['key1', { name: 'aaa' }] + ]) + }) + + it('Order array of objects by name prop in desc with equal values', () => { + const mockedArr = [ + { name: 'aaa' }, + { name: 'www' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'yyy' }, + { name: 'aaa' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['desc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'aaa' }, + { name: 'aaa' } + ]) + }) + + it('Order array of objects by name prop in desc with null and undefined', () => { + const mockedArr = [ + { name: undefined }, + { name: 'aaa' }, + { name: null }, + { name: 'www' }, + { name: null }, + { name: undefined }, + { name: null }, + { name: 'bbb' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['desc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'aaa' }, + { name: undefined }, + { name: null }, + { name: null }, + { name: undefined }, + { name: null } + ]) + }) + + it('Order array of objects by name prop in asc with null and undefined', () => { + const mockedArr = [ + { name: undefined }, + { name: 'aaa' }, + { name: null }, + { name: 'www' }, + { name: null }, + { name: undefined }, + { name: null }, + { name: 'bbb' }, + { name: 'yyy' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['asc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'aaa' }, + { name: 'bbb' }, + { name: 'www' }, + { name: 'yyy' }, + { name: undefined }, + { name: null }, + { name: null }, + { name: undefined }, + { name: null } + ]) + }) + + it('Order array of objects by name prop in desc with null and undefined items', () => { + const mockedArr = [ + { name: undefined }, + { name: 'aaa' }, + { name: null }, + { name: 'www' }, + null, + undefined, + { name: null }, + { name: undefined }, + { name: null }, + { name: 'bbb' }, + null, + { name: 'yyy' } + ] + + const orderedArr = orderBy(mockedArr, ['name'], ['desc']) + + assert.deepStrictEqual(orderedArr, [ + { name: 'yyy' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'aaa' }, + { name: undefined }, + { name: null }, + null, + { name: null }, + { name: undefined }, + { name: null }, + null, + /** + * Array.prototype.sort() behaviour: all undefined elements + * are sorted to the end of the array, with no call to compareFn + */ + undefined + ]) + }) + + it('Throw TypeError if non-iterable object is ordered', () => { + assert.throws( + () => orderBy(null, ['name'], ['desc']), + TypeError + ) + }) + + it('Throw TypeError if non-iterable object is prop name list', () => { + const mockedArr = [ + { name: 'aaa' }, + { name: 'www' }, + { name: 'bbb' }, + { name: 'yyy' } + ] + + assert.throws( + () => orderBy(mockedArr, null, ['desc']), + TypeError + ) + }) +}) diff --git a/workers/loc.api/helpers/utils/check-params-auth.js b/workers/loc.api/helpers/utils/check-params-auth.js new file mode 100644 index 000000000..fe1c3dd86 --- /dev/null +++ b/workers/loc.api/helpers/utils/check-params-auth.js @@ -0,0 +1,29 @@ +'use strict' + +const { + AuthError +} = require('@bitfinex/bfx-report/workers/loc.api/errors') + +module.exports = (args) => { + const { + apiKey, + apiSecret, + authToken, + isSubAccount + } = args?.auth ?? {} + + if ( + !isSubAccount && + authToken + ) { + return + } + if ( + !apiKey || + typeof apiKey !== 'string' || + !apiSecret || + typeof apiSecret !== 'string' + ) { + throw new AuthError() + } +} diff --git a/workers/loc.api/helpers/utils/coll-obj-to-arr.js b/workers/loc.api/helpers/utils/coll-obj-to-arr.js new file mode 100644 index 000000000..dafc35c42 --- /dev/null +++ b/workers/loc.api/helpers/utils/coll-obj-to-arr.js @@ -0,0 +1,66 @@ +'use strict' + +const { pick } = require('@bitfinex/lib-js-util-base') + +const { + isUpdatableArr, + isUpdatableArrObj +} = require('../../sync/schema/utils') + +module.exports = (coll = [], opts) => { + const { + projection, + type + } = opts ?? {} + const _projection = Array.isArray(projection) + ? projection + : [projection] + const isProjectionExisted = ( + _projection[0] && + typeof _projection[0] === 'string' + ) + + const res = [] + + if (isUpdatableArr(type)) { + const fieldName = isProjectionExisted + ? _projection[0] + : null + + for (const obj of coll) { + if ( + !obj || + typeof obj !== 'object' + ) { + continue + } + + const _fieldName = fieldName ?? Object.keys(obj) + .filter((key) => key !== '_id')[0] + + if (typeof obj?.[_fieldName] === 'undefined') { + continue + } + + res.push(obj?.[_fieldName]) + } + } + if (isUpdatableArrObj(type)) { + if (!isProjectionExisted) { + return coll + } + + for (const obj of coll) { + if ( + !obj || + typeof obj !== 'object' + ) { + continue + } + + res.push(pick(obj, projection)) + } + } + + return res +} diff --git a/workers/loc.api/helpers/utils/get-date-string.js b/workers/loc.api/helpers/utils/get-date-string.js new file mode 100644 index 000000000..5bb2ac6ea --- /dev/null +++ b/workers/loc.api/helpers/utils/get-date-string.js @@ -0,0 +1,5 @@ +'use strict' + +module.exports = (mc) => { + return new Date(mc).toDateString().split(' ').join('-') +} diff --git a/workers/loc.api/helpers/utils/index.js b/workers/loc.api/helpers/utils/index.js new file mode 100644 index 000000000..621783361 --- /dev/null +++ b/workers/loc.api/helpers/utils/index.js @@ -0,0 +1,29 @@ +'use strict' + +const checkParamsAuth = require('./check-params-auth') +const tryParseJSON = require('./try-parse-json') +const collObjToArr = require('./coll-obj-to-arr') +const getDateString = require('./get-date-string') +const isNotSyncRequired = require('./is-not-sync-required') +const pickLowerObjectsNumbers = require('./pick-lower-objects-numbers') +const pickAllLowerObjectsNumbers = require('./pick-all-lower-objects-numbers') +const sumObjectsNumbers = require('./sum-objects-numbers') +const sumAllObjectsNumbers = require('./sum-all-objects-numbers') +const sumArrayVolumes = require('./sum-array-volumes') +const pushLargeArr = require('./push-large-arr') +const orderBy = require('./order-by') + +module.exports = { + checkParamsAuth, + tryParseJSON, + collObjToArr, + getDateString, + isNotSyncRequired, + pickLowerObjectsNumbers, + pickAllLowerObjectsNumbers, + sumObjectsNumbers, + sumAllObjectsNumbers, + sumArrayVolumes, + pushLargeArr, + orderBy +} diff --git a/workers/loc.api/helpers/utils/is-not-sync-required.js b/workers/loc.api/helpers/utils/is-not-sync-required.js new file mode 100644 index 000000000..a38aad37b --- /dev/null +++ b/workers/loc.api/helpers/utils/is-not-sync-required.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = (args) => !!args?.params?.isNotSyncRequired diff --git a/workers/loc.api/helpers/utils/order-by.js b/workers/loc.api/helpers/utils/order-by.js new file mode 100644 index 000000000..38f46ae22 --- /dev/null +++ b/workers/loc.api/helpers/utils/order-by.js @@ -0,0 +1,61 @@ +'use strict' + +const _getValue = (obj, path) => { + if ( + obj === undefined || + obj === null + ) { + return obj + } + if (typeof path !== 'string') { + return obj?.[path] + } + + return path.split('.').reduce((accum, key) => accum?.[key], obj) +} + +module.exports = (collection, iteratees = [], orders = []) => { + // It's able to consider iterable objects as well + // If iterable object is required, just create it after ordering, not here + const copiedColl = [...collection] + + return copiedColl.sort((a, b) => { + for (const [i, iteratee] of iteratees.entries()) { + const direction = orders[i] === 'desc' ? -1 : 1 + const isIterateeFn = typeof iteratee === 'function' + + const valA = isIterateeFn + ? iteratee(a) + : _getValue(a, iteratee) + const valB = isIterateeFn + ? iteratee(b) + : _getValue(b, iteratee) + + if (valA === valB) { + continue + } + + if ( + valA === undefined || + valA === null + ) { + return 1 + } + if ( + valB === undefined || + valB === null + ) { + return -1 + } + + if (valA > valB) { + return direction + } + if (valA < valB) { + return -direction + } + } + + return 0 + }) +} diff --git a/workers/loc.api/helpers/utils/pick-all-lower-objects-numbers.js b/workers/loc.api/helpers/utils/pick-all-lower-objects-numbers.js new file mode 100644 index 000000000..cdc80155c --- /dev/null +++ b/workers/loc.api/helpers/utils/pick-all-lower-objects-numbers.js @@ -0,0 +1,31 @@ +'use strict' + +module.exports = (propName, objects = []) => { + return objects.reduce((accum, curr) => { + if ( + !curr?.[propName] || + typeof curr?.[propName] !== 'object' + ) { + return accum + } + + const entries = Object.entries(curr[propName]) + + return entries.reduce((accum, [key, val]) => { + if (!Number.isFinite(val)) { + return accum + } + if (!Number.isFinite(accum?.[key])) { + accum[key] = val + + return accum + } + + accum[key] = val < accum[key] + ? val + : accum[key] + + return accum + }, accum) + }, {}) +} diff --git a/workers/loc.api/helpers/utils/pick-lower-objects-numbers.js b/workers/loc.api/helpers/utils/pick-lower-objects-numbers.js new file mode 100644 index 000000000..7b562cd2e --- /dev/null +++ b/workers/loc.api/helpers/utils/pick-lower-objects-numbers.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = (propName, objects = []) => { + return objects.reduce((accum, curr) => { + if (!Number.isFinite(curr?.[propName])) { + return accum + } + if (!Number.isFinite(accum)) { + return curr[propName] + } + + return curr[propName] < accum + ? curr[propName] + : accum + }, null) +} diff --git a/workers/loc.api/helpers/utils/push-large-arr.js b/workers/loc.api/helpers/utils/push-large-arr.js new file mode 100644 index 000000000..6056b3c2e --- /dev/null +++ b/workers/loc.api/helpers/utils/push-large-arr.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = (dest, src) => { + for (const item of src) { + dest.push(item) + } +} diff --git a/workers/loc.api/helpers/utils/sum-all-objects-numbers.js b/workers/loc.api/helpers/utils/sum-all-objects-numbers.js new file mode 100644 index 000000000..d0733c7b5 --- /dev/null +++ b/workers/loc.api/helpers/utils/sum-all-objects-numbers.js @@ -0,0 +1,27 @@ +'use stricts' + +module.exports = (propName, objects = []) => { + return objects.reduce((accum, curr) => { + if ( + !curr?.[propName] || + typeof curr?.[propName] !== 'object' + ) { + return accum + } + + const entries = Object.entries(curr[propName]) + + return entries.reduce((accum, [key, val]) => { + const prevVal = Number.isFinite(accum?.[key]) + ? accum[key] + : 0 + const currVal = Number.isFinite(val) + ? val + : 0 + + accum[key] = prevVal + currVal + + return accum + }, accum) + }, {}) +} diff --git a/workers/loc.api/helpers/utils/sum-array-volumes.js b/workers/loc.api/helpers/utils/sum-array-volumes.js new file mode 100644 index 000000000..fedb8635a --- /dev/null +++ b/workers/loc.api/helpers/utils/sum-array-volumes.js @@ -0,0 +1,56 @@ +'use strict' + +module.exports = (propName, objects = []) => { + return objects.reduce((accum, curr) => { + if (!Array.isArray(curr?.[propName])) { + return accum + } + + for (const obj of curr[propName]) { + if (typeof obj?.curr !== 'string') { + continue + } + + const entries = Object.entries(obj) + .filter(([key]) => key !== 'curr') + + if (entries.length === 0) { + continue + } + + if (accum.length === 0) { + accum.push({ ...obj }) + + continue + } + + const accumObjIndex = accum + .findIndex((item) => item?.curr === obj.curr) + + if (accumObjIndex === -1) { + accum.push({ ...obj }) + + continue + } + + const resObj = entries.reduce((accum, [key, vol]) => { + const accumVol = Number.isFinite(accum?.[key]) + ? accum[key] + : 0 + const currVol = Number.isFinite(vol) + ? vol + : 0 + + accum[key] = accumVol + currVol + + return accum + }, accum[accumObjIndex]) + + // For right order in resulting array + accum.splice(accumObjIndex, 1) + accum.push(resObj) + } + + return accum + }, []) +} diff --git a/workers/loc.api/helpers/utils/sum-objects-numbers.js b/workers/loc.api/helpers/utils/sum-objects-numbers.js new file mode 100644 index 000000000..d2fdaf8f5 --- /dev/null +++ b/workers/loc.api/helpers/utils/sum-objects-numbers.js @@ -0,0 +1,9 @@ +'use strict' + +module.exports = (propName, objects = []) => { + return objects.reduce((accum, curr) => { + return Number.isFinite(curr?.[propName]) + ? accum + curr[propName] + : accum + }, 0) +} diff --git a/workers/loc.api/helpers/utils/try-parse-json.js b/workers/loc.api/helpers/utils/try-parse-json.js new file mode 100644 index 000000000..0a5bfaf20 --- /dev/null +++ b/workers/loc.api/helpers/utils/try-parse-json.js @@ -0,0 +1,26 @@ +'use strict' + +module.exports = ( + jsonString, + isNotObject +) => { + try { + if (typeof jsonString !== 'string') { + return false + } + + const obj = JSON.parse(jsonString) + + if ( + isNotObject || + ( + obj && + typeof obj === 'object' + ) + ) { + return obj + } + } catch (e) { } + + return false +} diff --git a/workers/loc.api/sync/colls.accessors/public.colls.conf.accessors.js b/workers/loc.api/sync/colls.accessors/public.colls.conf.accessors.js index c9bc51cd0..40f36b039 100644 --- a/workers/loc.api/sync/colls.accessors/public.colls.conf.accessors.js +++ b/workers/loc.api/sync/colls.accessors/public.colls.conf.accessors.js @@ -1,8 +1,5 @@ 'use strict' -const { - orderBy -} = require('lodash') const { omit, pick, @@ -15,6 +12,9 @@ const { FindMethodError } = require('@bitfinex/bfx-report/workers/loc.api/errors') +const { + orderBy +} = require('../../helpers') const { GetPublicDataError } = require('../../errors') diff --git a/workers/loc.api/sync/currency.converter/index.js b/workers/loc.api/sync/currency.converter/index.js index d78f9d45e..0ae436165 100644 --- a/workers/loc.api/sync/currency.converter/index.js +++ b/workers/loc.api/sync/currency.converter/index.js @@ -1,7 +1,6 @@ 'use strict' const moment = require('moment') -const { orderBy } = require('lodash') const { FindMethodError @@ -16,7 +15,10 @@ const { const { isForexSymb } = require('../helpers') -const { tryParseJSON } = require('../../helpers') +const { + tryParseJSON, + orderBy +} = require('../../helpers') const SyncTempTablesManager = require( '../data.inserter/sync.temp.tables.manager' ) diff --git a/workers/loc.api/sync/dao/db.backup.manager/index.js b/workers/loc.api/sync/dao/db.backup.manager/index.js index 666e9f081..c0e1e601f 100644 --- a/workers/loc.api/sync/dao/db.backup.manager/index.js +++ b/workers/loc.api/sync/dao/db.backup.manager/index.js @@ -4,9 +4,11 @@ const { mkdirSync } = require('fs') const { readdir, rm, copyFile } = require('fs/promises') const path = require('path') const moment = require('moment') -const { orderBy } = require('lodash') -const { isBfxApiStaging } = require('../../../helpers') +const { + isBfxApiStaging, + orderBy +} = require('../../../helpers') const { decorateInjectable } = require('../../../di/utils') diff --git a/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js b/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js index c3c440888..fb4ada0ba 100644 --- a/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js +++ b/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js @@ -3,14 +3,12 @@ const { promisify } = require('util') const setImmediatePromise = promisify(setImmediate) -const { - orderBy -} = require('lodash') const { merge } = require('@bitfinex/lib-js-util-base') const SyncTempTablesManager = require('../sync.temp.tables.manager') const DataInserterHook = require('./data.inserter.hook') const { getAuthFromDb } = require('../helpers/utils') +const { orderBy } = require('../../../helpers') const { SubAccountLedgersBalancesRecalcError } = require('../../../errors') diff --git a/workers/loc.api/sync/movements/index.js b/workers/loc.api/sync/movements/index.js index 62c4db9bc..362c4a270 100644 --- a/workers/loc.api/sync/movements/index.js +++ b/workers/loc.api/sync/movements/index.js @@ -1,9 +1,11 @@ 'use strict' -const { orderBy } = require('lodash') const { merge } = require('@bitfinex/lib-js-util-base') -const { pushLargeArr } = require('../../helpers/utils') +const { + pushLargeArr, + orderBy +} = require('../../helpers/utils') const { decorateInjectable } = require('../../di/utils') @@ -119,7 +121,7 @@ class Movements { const { propNames, orders - } = this._getLodashOrder(sort) + } = this._getOrder(sort) const orderedRes = orderBy( movements, propNames, @@ -158,7 +160,7 @@ class Movements { const { propNames, orders - } = this._getLodashOrder(sort) + } = this._getOrder(sort) const orderedRes = orderBy( movements, propNames, @@ -338,7 +340,7 @@ class Movements { return [['mts', orderSign], ['id', orderSign]] } - _getLodashOrder (sort) { + _getOrder (sort) { const propNames = [] const orders = [] diff --git a/workers/loc.api/sync/positions.snapshot/index.js b/workers/loc.api/sync/positions.snapshot/index.js index da9a547ff..16ebf6326 100644 --- a/workers/loc.api/sync/positions.snapshot/index.js +++ b/workers/loc.api/sync/positions.snapshot/index.js @@ -1,10 +1,10 @@ 'use strict' -const { orderBy } = require('lodash') - const { splitSymbolPairs } = require('@bitfinex/bfx-report/workers/loc.api/helpers') + +const { orderBy } = require('../../helpers') const { groupByTimeframe, getMtsGroupedByTimeframe, diff --git a/workers/loc.api/sync/sub.account.api.data/index.js b/workers/loc.api/sync/sub.account.api.data/index.js index aa79fdcf3..b7eb12402 100644 --- a/workers/loc.api/sync/sub.account.api.data/index.js +++ b/workers/loc.api/sync/sub.account.api.data/index.js @@ -1,8 +1,5 @@ 'use strict' -const { - orderBy -} = require('lodash') const { isEmpty } = require('@bitfinex/lib-js-util-base') @@ -14,6 +11,9 @@ const { FindMethodError } = require('@bitfinex/bfx-report/workers/loc.api/errors') +const { + orderBy +} = require('../../helpers') const { DatePropNameError } = require('../../errors') diff --git a/workers/loc.api/sync/sub.account/index.js b/workers/loc.api/sync/sub.account/index.js index fc17d73ec..a6fbaf5e6 100644 --- a/workers/loc.api/sync/sub.account/index.js +++ b/workers/loc.api/sync/sub.account/index.js @@ -1,14 +1,13 @@ 'use strict' -const { orderBy } = require('lodash') - const { AuthError } = require('@bitfinex/bfx-report/workers/loc.api/errors') const { isSubAccountApiKeys, - getSubAccountAuthFromAuth + getSubAccountAuthFromAuth, + orderBy } = require('../../helpers') const { SubAccountCreatingError,