diff --git a/.gitignore b/.gitignore index 5cabf296d..b74d49773 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ lerna-debug.log !**/.env.schema **/package-lock.json -*.tsbuildinfo \ No newline at end of file +*.tsbuildinfo +# TODO: Remove after testing +/modules/server/testConfigs* \ No newline at end of file diff --git a/integration-tests/admin/test/index.test.js b/integration-tests/admin/test/index.test.js index 4ea0c5a84..b15aa2a91 100644 --- a/integration-tests/admin/test/index.test.js +++ b/integration-tests/admin/test/index.test.js @@ -1,9 +1,12 @@ +import { after, before, describe } from 'node:test'; + import { Client } from '@elastic/elasticsearch'; import express from 'express'; -import Arranger, { adminGraphql } from '../../../modules/server/dist'; -import ajax from '../../../modules/server/dist/utils/ajax'; -import addProject from './addProject'; +import Arranger, { adminGraphql } from '../../../modules/server/dist/index.js'; +import ajax from '../../../modules/server/dist/utils/ajax.js'; + +import addProject from './addProject.js'; const file_centric_mapppings = require('./assets/file_centric.mappings.json'); @@ -19,58 +22,58 @@ const app = express(); const api = ajax(`http://localhost:${port}`); const esClient = new Client({ - ...(useAuth && { - auth: { - username: esUser, - password: esPwd, - }, - }), - node: esHost, + ...(useAuth && { + auth: { + username: esUser, + password: esPwd, + }, + }), + node: esHost, }); const cleanup = () => - Promise.all([ - esClient.indices.delete({ - index: esIndex, - }), - esClient.indices.delete({ - index: 'arranger-projects*', - }), - ]); + Promise.all([ + esClient.indices.delete({ + index: esIndex, + }), + esClient.indices.delete({ + index: 'arranger-projects*', + }), + ]); describe('@arranger/admin', () => { - let server; - const adminPath = '/admin/graphql'; - before(async () => { - console.log('===== Initializing Elasticsearch data ====='); - try { - await cleanup(); - } catch (err) {} - await esClient.indices.create({ - index: esIndex, - body: file_centric_mapppings, - }); + let server; + const adminPath = '/admin/graphql'; + before(async () => { + console.log('===== Initializing Elasticsearch data ====='); + try { + await cleanup(); + } catch (err) {} + await esClient.indices.create({ + index: esIndex, + body: file_centric_mapppings, + }); - console.log('===== Starting arranger app for test ====='); - const router = await Arranger({ esHost, enableAdmin: false }); - const adminApp = await adminGraphql({ esHost }); - adminApp.applyMiddleware({ app, path: adminPath }); - app.use(router); - await new Promise((resolve) => { - server = app.listen(port, () => { - resolve(); - }); - }); - }); - after(async () => { - server?.close(); - await cleanup(); - }); + console.log('===== Starting arranger app for test ====='); + const router = await Arranger({ esHost, enableAdmin: false }); + const adminApp = await adminGraphql({ esHost }); + adminApp.applyMiddleware({ app, path: adminPath }); + app.use(router); + await new Promise((resolve) => { + server = app.listen(port, () => { + resolve(); + }); + }); + }); + after(async () => { + server?.close(); + await cleanup(); + }); - const env = { - api, - esIndex, - adminPath, - }; - addProject(env); + const env = { + api, + esIndex, + adminPath, + }; + addProject(env); }); diff --git a/integration-tests/server/test/index.test.js b/integration-tests/server/test/index.test.js index 51c56639c..572d65c9f 100644 --- a/integration-tests/server/test/index.test.js +++ b/integration-tests/server/test/index.test.js @@ -1,13 +1,10 @@ import { after, before, suite } from 'node:test'; -import { Client } from '@elastic/elasticsearch'; import Arranger from '@overture-stack/arranger-server'; import ajax from '@overture-stack/arranger-server/dist/utils/ajax.js'; import express from 'express'; -// import { print } from 'graphql'; -// import gql from 'graphql-tag'; -// import Arranger, { adminGraphql } from '../../../modules/server/dist'; +import getSearchClient from '../../../modules/server/src/searchClient/index.js'; // test modules import data from './assets/model_centric.data.json'; @@ -26,6 +23,7 @@ const setsIndex = process.env.ES_ARRANGER_SET_INDEX || 'arranger-sets-testing'; const esPwd = process.env.ES_PASS; const esUser = process.env.ES_USER; const port = process.env.PORT || 5678; +const clientType = process.env.SEARCH_CLIENT; const useAuth = !!esPwd && !!esUser; @@ -36,7 +34,7 @@ const api = ajax(`http://localhost:${port}`, { endpoint: '/graphql', }); -const esClient = new Client({ +const esClient = await getSearchClient({ ...(useAuth && { auth: { username: esUser, @@ -44,6 +42,7 @@ const esClient = new Client({ }, }), node: esHost, + clientType, }); const cleanup = () => { @@ -61,110 +60,70 @@ suite('integration-tests/server', () => { let server; const documentType = 'model'; - before(async () => { - console.log('\n(Initializing Elasticsearch and Arranger)'); + before( + async () => { + console.log('\n(Initializing Elasticsearch and Arranger)'); - try { - await cleanup(); - } catch (err) { - // - } + try { + await cleanup(); + } catch (err) { + // + } - await esClient.indices.create({ - index: esIndex, - body: mappings, - }); - - for (const datum of data) { - await esClient.index({ + await esClient.indices.create({ index: esIndex, - id: datum._id, - body: datum._source, - refresh: 'wait_for', + body: mappings, }); - } - try { - const router = await Arranger({ - // needed to see the mapping - enableAdmin, - // This may be useful when troubleshooting tests - enableLogs: true, - esHost, - getServerSideFilter: () => ({ - op: 'not', - content: [ - { - op: 'in', - content: { - fieldName: 'access_denied', - value: ['true'], + for (const datum of data) { + await esClient.index({ + index: esIndex, + id: datum._id, + body: datum._source, + refresh: 'wait_for', + }); + } + + try { + const router = await Arranger({ + // needed to see the mapping + enableAdmin, + // This may be useful when troubleshooting tests + enableLogs: true, + esHost, + getServerSideFilter: () => ({ + op: 'not', + content: [ + { + op: 'in', + content: { + fieldName: 'access_denied', + value: ['true'], + }, }, - }, - ], - }), - setsIndex, - }); - - app.use(router); + ], + }), + setsIndex, + }); - // TODO: reenable once Admin is back online - // const adminApp = await adminGraphql({ esHost }); - // adminApp.applyMiddleware({ app, path: adminPath }); + app.use(router); - await new Promise((resolve) => { - server = app.listen(port, () => { - resolve(null); + await new Promise((resolve) => { + server = app.listen(port, () => { + resolve(null); + }); }); - }); - // TODO: reenable once Admin is back online - // /** - // * uses the admin API to adds some metadata - // */ - // await api.post({ - // endpoint: adminPath, - // body: { - // query: print(gql` - // mutation($projectId: String!) { - // newProject(id: $projectId) { - // id - // __typename - // } - // } - // `), - // variables: { - // projectId, - // }, - // }, - // }); - - // await api.post({ - // endpoint: adminPath, - // body: { - // query: print(gql` - // mutation($projectId: String!, $documentType: String!, $esIndex: String!) { - // newIndex(projectId: $projectId, documentType: $documentType, esIndex: $esIndex) { - // id - // } - // } - // `), - // variables: { - // projectId, - // documentType, - // esIndex, - // }, - // }, - // }); - - console.log('******* Starting tests *******'); - } catch (err) { - console.error('error:', err); - throw err; - } - }, { - timeout: 10000, - }); + console.log('******* Starting tests *******'); + } catch (err) { + console.error('error:', err); + throw err; + } + }, + { + timeout: 10000, + }, + ); after(async () => { try { diff --git a/integration-tests/server/test/manageSets.js b/integration-tests/server/test/manageSets.js index 0ef4465e9..72235a7e0 100644 --- a/integration-tests/server/test/manageSets.js +++ b/integration-tests/server/test/manageSets.js @@ -27,7 +27,7 @@ export default ({ api, documentType }) => { assert.equal(data.errors, undefined); - setId = data.data.newSet.setId; + setId = data?.data?.newSet?.setId; }); test('2.retrieves newly created set successfully', async () => { @@ -55,11 +55,8 @@ export default ({ api, documentType }) => { }); assert.equal(data.errors, undefined); - const allSetIds = data.data.sets.hits.edges.map(({ node }) => node.setId); + const allSetIds = data?.data?.sets?.hits?.edges.map(({ node }) => node.setId) || []; - assert.ok( - allSetIds.includes(setId), - `Expected [${allSetIds.join(', ')}] to include ${setId}` - ); + assert.ok(allSetIds.includes(setId), `Expected [${allSetIds.join(', ')}] to include ${setId}`); }); }; diff --git a/integration-tests/server/test/readAggregation.js b/integration-tests/server/test/readAggregation.js index 847a5062e..c68383c1a 100644 --- a/integration-tests/server/test/readAggregation.js +++ b/integration-tests/server/test/readAggregation.js @@ -6,10 +6,12 @@ import gql from 'graphql-tag'; import { orderBy } from 'lodash-es'; export default async ({ api, documentType }) => { - const expectedBuckets = [{ - doc_count: 2, - key: 'Stage I', - }]; + const expectedBuckets = [ + { + doc_count: 2, + key: 'Stage I', + }, + ]; test('1.reads aggregations properly', async () => { const { data } = await api @@ -84,10 +86,9 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual(orderBy( - data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, - 'key', - ), expectedBuckets, + assert.deepEqual( + orderBy(data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, 'key'), + expectedBuckets, ); }); @@ -135,10 +136,10 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual(orderBy( - data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, - 'key', - ), expectedBuckets); + assert.deepEqual( + orderBy(data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, 'key'), + expectedBuckets, + ); }); test('4.should work with postfix filter sqon', async () => { @@ -185,10 +186,10 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual(orderBy( - data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, - 'key', - ), expectedBuckets); + assert.deepEqual( + orderBy(data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, 'key'), + expectedBuckets, + ); }); test('5.should work with pre and post-fix filter sqon', async () => { @@ -235,10 +236,10 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual(orderBy( - data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, - 'key', - ), expectedBuckets); + assert.deepEqual( + orderBy(data.data[documentType].aggregations.clinical_diagnosis__clinical_stage_grouping.buckets, 'key'), + expectedBuckets, + ); }); test('6.should count the correct number of buckets', async () => { @@ -294,10 +295,7 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual( - data.data[documentType].aggregations.clinical_diagnosis__histological_type.bucket_count, - 0, - ); + assert.deepEqual(data.data[documentType].aggregations.clinical_diagnosis__histological_type.bucket_count, 0); }); test('8.should count buckets with key "MISSING" when include_missing is defaulted to true', async () => { @@ -323,10 +321,7 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual( - data.data[documentType].aggregations.clinical_diagnosis__histological_type.bucket_count, - 1, - ); + assert.deepEqual(data.data[documentType].aggregations.clinical_diagnosis__histological_type.bucket_count, 1); }); test('9.should not include access_denied documents', async () => { @@ -355,9 +350,6 @@ export default async ({ api, documentType }) => { console.log('readAggregation error', err); }); - assert.deepEqual( - data.data[documentType].aggregations.access_denied.buckets, - [{ key_as_string: 'false' }], - ); + assert.deepEqual(data.data[documentType].aggregations.access_denied.buckets, [{ key_as_string: 'false' }]); }); }; diff --git a/modules/server/.env.schema b/modules/server/.env.schema index a4bd01f09..8b63dd6ae 100644 --- a/modules/server/.env.schema +++ b/modules/server/.env.schema @@ -17,4 +17,5 @@ MAX_RESULTS_WINDOW=10000 PING_MS=2200 PING_PATH=/ping PORT=5050 -ROW_ID_FIELD_NAME=id \ No newline at end of file +ROW_ID_FIELD_NAME=id +SEARCH_CLIENT='' \ No newline at end of file diff --git a/modules/server/package.json b/modules/server/package.json index 0b1ca95fd..d2b7c7430 100644 --- a/modules/server/package.json +++ b/modules/server/package.json @@ -28,10 +28,11 @@ "watch": "npm run clear:dist && npm run build -- --watch" }, "dependencies": { - "@graphql-tools/merge": "^9.0.4", "@elastic/elasticsearch": "^7.17.14", + "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "apollo-server": "^3.10.3", "apollo-server-core": "^3.10.3", diff --git a/modules/server/src/admin/index.ts b/modules/server/src/admin/index.ts index 956bb66cb..e57de4d12 100644 --- a/modules/server/src/admin/index.ts +++ b/modules/server/src/admin/index.ts @@ -2,100 +2,99 @@ import { addMocksToSchema } from '@graphql-tools/mock'; import { mergeSchemas } from '@graphql-tools/schema'; import { ApolloServer } from 'apollo-server-express'; -import { Client } from '@elastic/elasticsearch'; +import { type GraphQLSchema } from 'graphql'; import { print } from 'graphql/language/printer'; -import { GraphQLSchema } from 'graphql'; + +import { type SearchClient } from '#searchClient/types.js'; import { - createAggsStateByIndexResolver, - createColumnsStateByIndexResolver, - createExtendedMappingsByIndexResolver, - createIndexByProjectResolver, - createIndicesByProjectResolver, - createMatchBoxStateByIndexResolver, -} from './resolvers'; -import { createSchema as createProjectSchema } from './schemas/ProjectSchema'; -import { createSchema as createIndexSchema } from './schemas/IndexSchema'; -import { createSchema as createAggsStateSchema } from './schemas/AggsState'; -import { createSchema as createMatchboxStateSchema } from './schemas/MatchboxState'; -import { createSchema as createColumnsStateSchema } from './schemas/ColumnsState'; -import { createSchema as createExtendedMappingSchema } from './schemas/ExtendedMapping'; -import mergedTypeDefs from './schemaTypeDefs'; -import { constants } from './services/constants'; -import { createClient as createElasticsearchClient } from './services/elasticsearch'; -import { AdminApiConfig, IQueryContext } from './types'; + createAggsStateByIndexResolver, + createColumnsStateByIndexResolver, + createExtendedMappingsByIndexResolver, + createIndexByProjectResolver, + createIndicesByProjectResolver, + createMatchBoxStateByIndexResolver, +} from './resolvers.js'; +import { createSchema as createAggsStateSchema } from './schemas/AggsState/index.js'; +import { createSchema as createColumnsStateSchema } from './schemas/ColumnsState/index.js'; +import { createSchema as createExtendedMappingSchema } from './schemas/ExtendedMapping/index.js'; +import { createSchema as createIndexSchema } from './schemas/IndexSchema/index.js'; +import { createSchema as createMatchboxStateSchema } from './schemas/MatchboxState/index.js'; +import { createSchema as createProjectSchema } from './schemas/ProjectSchema/index.js'; +import mergedTypeDefs from './schemaTypeDefs.js'; +import { constants } from './services/constants.js'; +import { createClient as createElasticsearchClient } from './services/elasticsearch/index.js'; +import { type AdminApiConfig, type IQueryContext } from './types.js'; const createSchema = async () => { - const typeDefs = mergedTypeDefs; + const typeDefs = mergedTypeDefs; - const projectSchema = await createProjectSchema(); - const aggsStateSchema = await createAggsStateSchema(); - const collumnsStateSchema = await createColumnsStateSchema(); - const extendedMappingShema = await createExtendedMappingSchema(); - const matchBoxStateSchema = await createMatchboxStateSchema(); - const indexSchema = await createIndexSchema(); + const projectSchema = await createProjectSchema(); + const aggsStateSchema = await createAggsStateSchema(); + const collumnsStateSchema = await createColumnsStateSchema(); + const extendedMappingShema = await createExtendedMappingSchema(); + const matchBoxStateSchema = await createMatchboxStateSchema(); + const indexSchema = await createIndexSchema(); - const mergedSchema = mergeSchemas({ - schemas: [ - projectSchema, - indexSchema, - aggsStateSchema, - collumnsStateSchema, - extendedMappingShema, - matchBoxStateSchema, - print(typeDefs) as unknown as GraphQLSchema, // TODO: this type coercion is smelly - ], - resolvers: { - Project: { - index: createIndexByProjectResolver(indexSchema), - indices: createIndicesByProjectResolver(indexSchema), - }, - Index: { - extended: createExtendedMappingsByIndexResolver(extendedMappingShema), - columnsState: createColumnsStateByIndexResolver(collumnsStateSchema), - aggsState: createAggsStateByIndexResolver(aggsStateSchema), - matchBoxState: createMatchBoxStateByIndexResolver(matchBoxStateSchema), - }, - }, - }); - addMocksToSchema({ schema: mergedSchema, preserveResolvers: true }); - return mergedSchema; + const mergedSchema = mergeSchemas({ + schemas: [ + projectSchema, + indexSchema, + aggsStateSchema, + collumnsStateSchema, + extendedMappingShema, + matchBoxStateSchema, + print(typeDefs) as unknown as GraphQLSchema, // TODO: this type coercion is smelly + ], + resolvers: { + Project: { + index: createIndexByProjectResolver(indexSchema), + indices: createIndicesByProjectResolver(indexSchema), + }, + Index: { + extended: createExtendedMappingsByIndexResolver(extendedMappingShema), + columnsState: createColumnsStateByIndexResolver(collumnsStateSchema), + aggsState: createAggsStateByIndexResolver(aggsStateSchema), + matchBoxState: createMatchBoxStateByIndexResolver(matchBoxStateSchema), + }, + }, + }); + addMocksToSchema({ schema: mergedSchema, preserveResolvers: true }); + return mergedSchema; }; function buildElasticsearchClient(config: AdminApiConfig) { - return createElasticsearchClient(config.esHost, config.esUser, config.esPass); + return createElasticsearchClient(config.esHost, config.esUser, config.esPass); } -const initialize = (config: AdminApiConfig): Promise => - new Promise(async (resolve, reject) => { - console.info('Initializing Elasticsearch Client for host: ' + config.esHost); - const esClient = buildElasticsearchClient(config); - try { - console.info( - 'Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch', - ); - const exists = await esClient.indices.exists({ - index: constants.ARRANGER_PROJECT_INDEX, - }); - if (!exists) { - esClient.indices.create({ - index: constants.ARRANGER_PROJECT_INDEX, - }); - } - resolve(esClient); - } catch (err) { - setTimeout(() => { - initialize(config).then(() => resolve(esClient)); - }, 1000); - } - }); +const initialize = async (config: AdminApiConfig): Promise => { + console.info('Initializing Elasticsearch Client for host: ' + config.esHost); + const esClient = await buildElasticsearchClient(config); + try { + console.info('Checking if index ' + constants.ARRANGER_PROJECT_INDEX + ' exists in Elasticsearch'); + const exists = await esClient.indices.exists({ + index: constants.ARRANGER_PROJECT_INDEX, + }); + if (!exists) { + esClient.indices.create({ + index: constants.ARRANGER_PROJECT_INDEX, + }); + } + return esClient; + } catch (err) { + setTimeout(async () => { + return await initialize(config); + }, 1000); + } +}; export default async (config: AdminApiConfig) => { - const esClient = await initialize(config); - return new ApolloServer({ - schema: await createSchema(), - context: (): IQueryContext => ({ - es: esClient, - }), - }); + const esClient = await initialize(config); + if (!esClient) throw new Error('Could not initialize esClient'); + return new ApolloServer({ + schema: await createSchema(), + context: (): IQueryContext => ({ + es: esClient, + }), + }); }; diff --git a/modules/server/src/admin/schemas/AggsState/index.ts b/modules/server/src/admin/schemas/AggsState/index.ts index c82969520..5679f87bc 100644 --- a/modules/server/src/admin/schemas/AggsState/index.ts +++ b/modules/server/src/admin/schemas/AggsState/index.ts @@ -1,11 +1,12 @@ -import resolvers from './resolvers'; -import typeDefs from './schemaTypeDefs'; import { makeExecutableSchema } from 'graphql-tools'; +import resolvers from './resolvers.js'; +import typeDefs from './schemaTypeDefs.js'; + export const createSchema = async () => { - const schema = makeExecutableSchema({ - typeDefs: await typeDefs(), - resolvers, - }); - return schema; + const schema = makeExecutableSchema({ + typeDefs: await typeDefs(), + resolvers, + }); + return schema; }; diff --git a/modules/server/src/admin/schemas/AggsState/resolvers.ts b/modules/server/src/admin/schemas/AggsState/resolvers.ts index 397d19c6c..21138942e 100644 --- a/modules/server/src/admin/schemas/AggsState/resolvers.ts +++ b/modules/server/src/admin/schemas/AggsState/resolvers.ts @@ -1,33 +1,34 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { IQueryContext } from '../../types'; -import { I_AggsSetState, I_AggsStateQueryInput, I_SaveAggsStateMutationInput } from './types'; -import { getAggsSetState, saveAggsSetState } from './utils'; -import { Resolver } from '../types'; +import { type GraphQLResolveInfo } from 'graphql'; -const saveAggsStateMutationResolver: Resolver = - async ( - obj: {}, - args, - { es }: IQueryContext, - info: GraphQLResolveInfo, - ): Promise => { - return await saveAggsSetState(es)(args); - }; +import { type IQueryContext } from '../../types.js'; +import { type Resolver } from '../types.js'; + +import { type I_AggsSetState, type I_AggsStateQueryInput, type I_SaveAggsStateMutationInput } from './types.js'; +import { getAggsSetState, saveAggsSetState } from './utils.js'; + +const saveAggsStateMutationResolver: Resolver = async ( + obj: object, + args, + { es }: IQueryContext, + info: GraphQLResolveInfo, +): Promise => { + return await saveAggsSetState(es)(args); +}; const aggsStateQueryResolver: Resolver = ( - obj: {}, - args, - { es }: IQueryContext, - info: GraphQLResolveInfo, + obj: object, + args, + { es }: IQueryContext, + info: GraphQLResolveInfo, ): Promise => { - return getAggsSetState(es)(args); + return getAggsSetState(es)(args); }; export default { - Query: { - aggsState: aggsStateQueryResolver, - }, - Mutation: { - saveAggsState: saveAggsStateMutationResolver, - }, + Query: { + aggsState: aggsStateQueryResolver, + }, + Mutation: { + saveAggsState: saveAggsStateMutationResolver, + }, }; diff --git a/modules/server/src/admin/schemas/AggsState/utils.ts b/modules/server/src/admin/schemas/AggsState/utils.ts index 63689eea7..f49a78aa4 100644 --- a/modules/server/src/admin/schemas/AggsState/utils.ts +++ b/modules/server/src/admin/schemas/AggsState/utils.ts @@ -1,67 +1,64 @@ -import { Client } from '@elastic/elasticsearch'; -import { mappingToAggsState } from '../../../mapping'; import { sortBy } from 'ramda'; + +import { type SearchClient } from '#searchClient/types.js'; + +import { mappingToAggsState } from '../../../mapping/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { timestamp } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_AggsSetState, - I_AggsState, - I_AggsStateQueryInput, - I_SaveAggsStateMutationInput, -} from './types'; -import { timestamp } from '../../services'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; -import { EsIndexLocation } from '../types'; -import { getEsMapping } from '../../services/elasticsearch'; + type I_AggsSetState, + type I_AggsState, + type I_AggsStateQueryInput, + type I_SaveAggsStateMutationInput, +} from './types.js'; export const createAggsSetState = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - const rawEsmapping = await getEsMapping(es)({ esIndex }); - const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; - const aggsState: I_AggsState[] = mappingToAggsState(mapping); - return { timestamp: timestamp(), state: aggsState }; - }; + (es: SearchClient) => + async ({ esIndex }: EsIndexLocation): Promise => { + const rawEsmapping = await getEsMapping(es)({ esIndex }); + const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings.properties; + const aggsState: I_AggsState[] = mappingToAggsState(mapping); + return { timestamp: timestamp(), state: aggsState }; + }; export const getAggsSetState = - (es: Client) => - async (args: I_AggsStateQueryInput): Promise => { - const { projectId, graphqlField } = args; - const metaData = (await getProjectStorageMetadata(es)(projectId)).find( - (entry) => entry.name === graphqlField, - ); - return metaData.config['aggs-state']; - }; + (es: SearchClient) => + async (args: I_AggsStateQueryInput): Promise => { + const { projectId, graphqlField } = args; + const metaData = (await getProjectStorageMetadata(es)(projectId)).find((entry) => entry.name === graphqlField); + return metaData?.config['aggs-state']; + }; export const saveAggsSetState = - (es: Client) => - async (args: I_SaveAggsStateMutationInput): Promise => { - const { graphqlField, projectId, state } = args; - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - const currentAggsState = currentMetadata.config['aggs-state']; - const sortByNewOrder = sortBy((i: I_AggsState) => - state.findIndex((_i) => _i.field === i.field), - ); - const newAggsSetState: typeof currentAggsState = { - timestamp: timestamp(), - state: sortByNewOrder( - currentAggsState.state.map((item) => ({ - ...(state.find((_item) => _item.field === item.field) || item), - type: item.type, - })), - ), - }; + (es: SearchClient) => + async (args: I_SaveAggsStateMutationInput): Promise => { + const { graphqlField, projectId, state } = args; + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + const currentAggsState = currentMetadata?.config['aggs-state']; + const sortByNewOrder = sortBy((i: I_AggsState) => state.findIndex((_i) => _i.field === i.field)); + const newAggsSetState: typeof currentAggsState = { + timestamp: timestamp(), + state: sortByNewOrder( + currentAggsState?.state.map((item) => ({ + ...(state.find((_item) => _item.field === item.field) || item), + type: item.type, + })), + ), + }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentMetadata.index, - name: currentMetadata.name, - config: { - 'aggs-state': newAggsSetState, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentMetadata?.index, + name: currentMetadata?.name, + config: { + 'aggs-state': newAggsSetState, + }, + }, + }); - return newAggsSetState; - }; + return newAggsSetState; + }; diff --git a/modules/server/src/admin/schemas/ColumnsState/utils.ts b/modules/server/src/admin/schemas/ColumnsState/utils.ts index ae366e06b..9d07c33f7 100644 --- a/modules/server/src/admin/schemas/ColumnsState/utils.ts +++ b/modules/server/src/admin/schemas/ColumnsState/utils.ts @@ -1,80 +1,75 @@ -import { Client } from '@elastic/elasticsearch'; -import { - I_Column, - I_ColumnSetState, - I_ColumnStateQueryInput, - I_SaveColumnsStateMutationInput, -} from './types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; -import { EsIndexLocation } from '../types'; -import { mappingToColumnsState } from '../../../mapping'; -import { replaceBy, timestamp } from '../../services'; -import { getEsMapping } from '../../services/elasticsearch'; import { sortBy } from 'ramda'; +import { type SearchClient } from '#searchClient/types.js'; + +import { mappingToColumnsState } from '../../../mapping/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { replaceBy, timestamp } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + +import { + type I_Column, + type I_ColumnSetState, + type I_ColumnStateQueryInput, + type I_SaveColumnsStateMutationInput, +} from './types.js'; + export const getColumnSetState = - (es: Client) => - async (args: I_ColumnStateQueryInput): Promise => { - const { graphqlField, projectId } = args; - const metaData = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - return metaData.config['columns-state']; - }; + (es: SearchClient) => + async (args: I_ColumnStateQueryInput): Promise => { + const { graphqlField, projectId } = args; + const metaData = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + return metaData?.config['columns-state']; + }; export const createColumnSetState = - (es: Client) => - async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { - const rawEsmapping = await getEsMapping(es)({ - esIndex, - }); - const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings; - const columns: I_Column[] = mappingToColumnsState(mapping.properties); - return { - state: { - type: graphqlField, - keyField: 'id', - defaultSorted: [{ id: columns[0].id || columns[0].accessor, desc: false }], - columns, - }, - timestamp: timestamp(), - }; - }; + (es: SearchClient) => + async ({ esIndex }: EsIndexLocation, graphqlField: string): Promise => { + const rawEsmapping = await getEsMapping(es)({ + esIndex, + }); + const mapping = rawEsmapping[Object.keys(rawEsmapping)[0]].mappings; + const columns: I_Column[] = mappingToColumnsState(mapping.properties); + return { + state: { + type: graphqlField, + keyField: 'id', + defaultSorted: [{ id: columns[0].id || columns[0].accessor, desc: false }], + columns, + }, + timestamp: timestamp(), + }; + }; export const saveColumnState = - (es: Client) => - async ({ - graphqlField, - projectId, - state, - }: I_SaveColumnsStateMutationInput): Promise => { - const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); - const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); - const sortByNewOrder = sortBy((i: I_Column) => - state.columns.findIndex((c) => c.field === i.field), - ); - const mergedState: typeof state = { - ...state, - columns: sortByNewOrder( - replaceBy( - currentIndexMetadata.config['columns-state'].state.columns, - state.columns, - (oldCol, newCol) => oldCol.field === newCol.field, - ), - ), - }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - 'columns-state': { - timestamp: timestamp(), - state: mergedState, - }, - }, - }, - }); - return getColumnSetState(es)({ projectId, graphqlField }); - }; + (es: SearchClient) => + async ({ graphqlField, projectId, state }: I_SaveColumnsStateMutationInput): Promise => { + const currentProjectMetadata = await getProjectStorageMetadata(es)(projectId); + const currentIndexMetadata = currentProjectMetadata.find((i) => i.name === graphqlField); + const sortByNewOrder = sortBy((i: I_Column) => state.columns.findIndex((c) => c.field === i.field)); + const mergedState: typeof state = { + ...state, + columns: sortByNewOrder( + replaceBy( + currentIndexMetadata?.config['columns-state']?.state?.columns, + state.columns, + (oldCol, newCol) => oldCol.field === newCol.field, + ), + ), + }; + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata?.index, + name: currentIndexMetadata?.name, + config: { + 'columns-state': { + timestamp: timestamp(), + state: mergedState, + }, + }, + }, + }); + return getColumnSetState(es)({ projectId, graphqlField }); + }; diff --git a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts index 6773ee9c4..d64ba530b 100644 --- a/modules/server/src/admin/schemas/ExtendedMapping/utils.ts +++ b/modules/server/src/admin/schemas/ExtendedMapping/utils.ts @@ -1,145 +1,137 @@ -import { Client } from '@elastic/elasticsearch'; import { UserInputError } from 'apollo-server'; -import { extendMapping } from '../../../mapping'; -import { getEsMapping } from '../../services/elasticsearch'; -import { replaceBy } from '../../services'; -import { EsIndexLocation } from '../types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; +import { type SearchClient } from '#searchClient/types.js'; + +import { extendMapping } from '../../../mapping/index.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { replaceBy } from '../../services/index.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_ExtendedFieldsMappingsQueryArgs, - I_GqlExtendedFieldMapping, - I_SaveExtendedMappingMutationArgs, - I_UpdateExtendedMappingMutationArgs, -} from './types'; + type I_ExtendedFieldsMappingsQueryArgs, + type I_GqlExtendedFieldMapping, + type I_SaveExtendedMappingMutationArgs, + type I_UpdateExtendedMappingMutationArgs, +} from './types.js'; export const createExtendedMapping = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - let extendedMappings: I_GqlExtendedFieldMapping[] = []; - try { - const esMapping = await getEsMapping(es)({ esIndex }); - const indexName = Object.keys(esMapping)[0]; //assumes all mappings returned are the same - const esMappingProperties = esMapping[indexName].mappings.properties; - extendedMappings = extendMapping(esMappingProperties) as I_GqlExtendedFieldMapping[]; - } catch (err) { - console.log('error: ', err); - throw err; - } - return extendedMappings; - }; + (es: SearchClient) => + async ({ esIndex }: EsIndexLocation): Promise => { + let extendedMappings: I_GqlExtendedFieldMapping[] = []; + try { + const esMapping = await getEsMapping(es)({ esIndex }); + const indexName = Object.keys(esMapping)[0]; //assumes all mappings returned are the same + const esMappingProperties = esMapping[indexName].mappings.properties; + extendedMappings = extendMapping(esMappingProperties) as I_GqlExtendedFieldMapping[]; + } catch (err) { + console.log('error: ', err); + throw err; + } + return extendedMappings; + }; export const getExtendedMapping = - (es: Client) => - async ({ - projectId, - graphqlField, - field, - }: I_ExtendedFieldsMappingsQueryArgs): Promise => { - const assertOutputType = (i: any): I_GqlExtendedFieldMapping => ({ - gqlId: `${projectId}::${graphqlField}::extended::${i.field}`, - field: i.field, - type: i.type, - displayName: i.displayName, - active: i.active, - isArray: i.isArray, - primaryKey: i.primaryKey, - quickSearchEnabled: i.quickSearchEnabled, - unit: i.unit, - displayValues: i.displayValues, - rangeStep: i.rangeStep, - }); - const indexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (metaData) => metaData.name === graphqlField, - ); - if (indexMetadata) { - if (field) { - return indexMetadata.config.extended - .filter((ex) => field === ex.field) - .map(assertOutputType); - } else { - return indexMetadata.config.extended.map(assertOutputType); - } - } else { - throw new UserInputError( - `no index found under name ${graphqlField} for project ${projectId}`, - ); - } - }; + (es: SearchClient) => + async ({ + projectId, + graphqlField, + field, + }: I_ExtendedFieldsMappingsQueryArgs): Promise => { + const assertOutputType = (i: any): I_GqlExtendedFieldMapping => ({ + gqlId: `${projectId}::${graphqlField}::extended::${i.field}`, + field: i.field, + type: i.type, + displayName: i.displayName, + active: i.active, + isArray: i.isArray, + primaryKey: i.primaryKey, + quickSearchEnabled: i.quickSearchEnabled, + unit: i.unit, + displayValues: i.displayValues, + rangeStep: i.rangeStep, + }); + const indexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( + (metaData) => metaData.name === graphqlField, + ); + if (indexMetadata) { + if (field) { + return indexMetadata.config.extended.filter((ex) => field === ex.field).map(assertOutputType); + } else { + return indexMetadata.config.extended.map(assertOutputType); + } + } else { + throw new UserInputError(`no index found under name ${graphqlField} for project ${projectId}`); + } + }; export const updateFieldExtendedMapping = - (es: Client) => - async ({ - field: mutatedField, - graphqlField, - projectId, - extendedFieldMappingInput, - }: I_UpdateExtendedMappingMutationArgs): Promise => { - const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (metaData) => { - return metaData.name === graphqlField; - }, - ); + (es: SearchClient) => + async ({ + field: mutatedField, + graphqlField, + projectId, + extendedFieldMappingInput, + }: I_UpdateExtendedMappingMutationArgs): Promise => { + const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find((metaData) => { + return metaData.name === graphqlField; + }); - if (currentIndexMetadata) { - const indexExtendedMappingFields = await getExtendedMapping(es)({ - projectId, - graphqlField, - }); + if (currentIndexMetadata) { + const indexExtendedMappingFields = await getExtendedMapping(es)({ + projectId, + graphqlField, + }); - const newIndexExtendedMappingFields: I_GqlExtendedFieldMapping[] = - indexExtendedMappingFields.map((field) => - (field.field as string) === mutatedField - ? { ...field, ...extendedFieldMappingInput } - : field, - ); + const newIndexExtendedMappingFields: I_GqlExtendedFieldMapping[] = indexExtendedMappingFields.map( + (field) => + (field.field as string) === mutatedField ? { ...field, ...extendedFieldMappingInput } : field, + ); - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - extended: newIndexExtendedMappingFields, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata.index, + name: currentIndexMetadata.name, + config: { + extended: newIndexExtendedMappingFields, + }, + }, + }); - return newIndexExtendedMappingFields.find((field) => field.field === mutatedField); - } else { - throw new UserInputError( - `no index found under name ${graphqlField} for project ${projectId}`, - ); - } - }; + return newIndexExtendedMappingFields.find((field) => field.field === mutatedField); + } else { + throw new UserInputError(`no index found under name ${graphqlField} for project ${projectId}`); + } + }; export const saveExtendedMapping = - (es: Client) => - async (args: I_SaveExtendedMappingMutationArgs): Promise => { - const { projectId, graphqlField, input } = args; - const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (entry) => entry.name === graphqlField, - ); - const { - config: { extended: currentStoredExtendedMapping }, - } = currentIndexMetadata; + (es: SearchClient) => + async (args: I_SaveExtendedMappingMutationArgs): Promise => { + const { projectId, graphqlField, input } = args; + const currentIndexMetadata = (await getProjectStorageMetadata(es)(projectId)).find( + (entry) => entry.name === graphqlField, + ); + const { + config: { extended: currentStoredExtendedMapping }, + } = currentIndexMetadata; - const newExtendedMapping: I_GqlExtendedFieldMapping[] = replaceBy( - currentStoredExtendedMapping, - input, - (el1, el2) => el1.field === el2.field, - ); + const newExtendedMapping: I_GqlExtendedFieldMapping[] = replaceBy( + currentStoredExtendedMapping, + input, + (el1, el2) => el1.field === el2.field, + ); - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentIndexMetadata.index, - name: currentIndexMetadata.name, - config: { - extended: newExtendedMapping, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentIndexMetadata?.index, + name: currentIndexMetadata?.name, + config: { + extended: newExtendedMapping, + }, + }, + }); - return newExtendedMapping; - }; + return newExtendedMapping; + }; diff --git a/modules/server/src/admin/schemas/IndexSchema/utils.ts b/modules/server/src/admin/schemas/IndexSchema/utils.ts index e310a92b6..29af66d3c 100644 --- a/modules/server/src/admin/schemas/IndexSchema/utils.ts +++ b/modules/server/src/admin/schemas/IndexSchema/utils.ts @@ -1,203 +1,199 @@ -import { Client } from '@elastic/elasticsearch'; import { UserInputError } from 'apollo-server'; import Qew from 'qew'; // TODO: using 0.9.13 because later versions break the async -import { getEsMapping } from '../../services/elasticsearch'; -import { constants } from '../../services/constants'; -import { serializeToGqlField, timestamp } from '../../services'; -import { createExtendedMapping } from '../ExtendedMapping/utils'; -import { getArrangerProjects } from '../ProjectSchema/utils'; -import { EsIndexLocation } from '../types'; +import { type SearchClient } from '#searchClient/types.js'; + +import { constants } from '../../services/constants.js'; +import { getEsMapping } from '../../services/elasticsearch/index.js'; +import { serializeToGqlField, timestamp } from '../../services/index.js'; +import { createAggsSetState } from '../AggsState/utils.js'; +import { createColumnSetState } from '../ColumnsState/utils.js'; +import { createExtendedMapping } from '../ExtendedMapping/utils.js'; +import { createMatchboxState } from '../MatchboxState/utils.js'; +import { getArrangerProjects } from '../ProjectSchema/utils.js'; +import { type EsIndexLocation } from '../types.js'; + import { - I_ProjectIndexMetadataUpdateDoc, - IIndexGqlModel, - IIndexQueryInput, - IIndexRemovalMutationInput, - INewIndexInput, - IProjectIndexMetadata, -} from './types'; -import { createColumnSetState } from '../ColumnsState/utils'; -import { createAggsSetState } from '../AggsState/utils'; -import { createMatchboxState } from '../MatchboxState/utils'; + type I_ProjectIndexMetadataUpdateDoc, + type IIndexGqlModel, + type IIndexQueryInput, + type IIndexRemovalMutationInput, + type INewIndexInput, + type IProjectIndexMetadata, +} from './types.js'; const { ARRANGER_PROJECT_INDEX } = constants; export const getProjectMetadataEsLocation = ( - projectId: string, + projectId: string, ): { - index: string; + index: string; } => ({ - index: `${ARRANGER_PROJECT_INDEX}-${projectId}`, + index: `${ARRANGER_PROJECT_INDEX}-${projectId}`, }); const mappingExistsOn = - (es: Client) => - async ({ esIndex }: EsIndexLocation): Promise => { - try { - await getEsMapping(es)({ esIndex }); - return true; - } catch (err) { - return false; - } - }; + (es: SearchClient) => + async ({ esIndex }: EsIndexLocation): Promise => { + try { + await getEsMapping(es)({ esIndex }); + return true; + } catch (err) { + return false; + } + }; export const getProjectStorageMetadata = - (es: Client) => - async (projectId: string): Promise => { - try { - const { - body: { - hits: { hits }, - }, - } = await es.search({ - ...getProjectMetadataEsLocation(projectId), - }); - return hits.map(({ _source }) => _source as IProjectIndexMetadata); - } catch (err) { - throw new UserInputError(`cannot find project of id ${projectId}`, err); - } - }; + (es: SearchClient) => + async (projectId: string): Promise => { + try { + const { + body: { + hits: { hits }, + }, + } = await es.search({ + ...getProjectMetadataEsLocation(projectId), + }); + return hits.map(({ _source }) => _source as IProjectIndexMetadata); + } catch (err) { + throw new UserInputError(`cannot find project of id ${projectId}`, err); + } + }; export const getProjectMetadata = - (es: Client) => - async (projectId: string): Promise => - Promise.all( - (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ - id: `${projectId}::${metadata.name}`, - hasMapping: mappingExistsOn(es)({ - esIndex: metadata.index, - }), - graphqlField: metadata.name, - projectId: projectId, - esIndex: metadata.index, - })), - ); + (es: SearchClient) => + async (projectId: string): Promise => + Promise.all( + (await getProjectStorageMetadata(es)(projectId)).map(async (metadata) => ({ + id: `${projectId}::${metadata.name}`, + hasMapping: mappingExistsOn(es)({ + esIndex: metadata.index, + }), + graphqlField: metadata.name, + projectId: projectId, + esIndex: metadata.index, + })), + ); + +export const getProjectIndex = + (es: SearchClient) => + async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { + try { + const output = (await getProjectMetadata(es)(projectId)).find( + ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, + ); + return output; + } catch { + throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); + } + }; export const createNewIndex = - (es: Client) => - async (args: INewIndexInput): Promise => { - const { projectId, graphqlField, esIndex } = args; - const arrangerProject: {} = (await getArrangerProjects(es)).find( - (project) => project.id === projectId, - ); - if (arrangerProject) { - const serializedGqlField = serializeToGqlField(graphqlField); - - const extendedMapping = await createExtendedMapping(es)({ - esIndex, - }); - - const metadataContent: IProjectIndexMetadata = { - index: esIndex, - name: serializedGqlField, - timestamp: timestamp(), - active: true, - config: { - 'aggs-state': await createAggsSetState(es)({ esIndex }), - 'columns-state': await createColumnSetState(es)( - { - esIndex, - }, - graphqlField, - ), - 'matchbox-state': createMatchboxState({ - graphqlField, - extendedFields: extendedMapping, - }), - extended: extendedMapping, - }, - }; - - await es.create({ - ...getProjectMetadataEsLocation(projectId), - id: esIndex, - body: metadataContent, - refresh: 'true', - }); - - return getProjectIndex(es)({ - projectId, - graphqlField: serializedGqlField, - }); - } else { - throw new UserInputError(`no project with ID ${projectId} was found`); - } - }; + (es: SearchClient) => + async (args: INewIndexInput): Promise => { + const { projectId, graphqlField, esIndex } = args; + const arrangerProject = (await getArrangerProjects(es)).find((project) => project.id === projectId); + if (arrangerProject) { + const serializedGqlField = serializeToGqlField(graphqlField); + + const extendedMapping = await createExtendedMapping(es)({ + esIndex, + }); + + const metadataContent: IProjectIndexMetadata = { + index: esIndex, + name: serializedGqlField, + timestamp: timestamp(), + active: true, + config: { + 'aggs-state': await createAggsSetState(es)({ esIndex }), + 'columns-state': await createColumnSetState(es)( + { + esIndex, + }, + graphqlField, + ), + 'matchbox-state': createMatchboxState({ + graphqlField, + extendedFields: extendedMapping, + }), + extended: extendedMapping, + }, + }; + + await es.create({ + ...getProjectMetadataEsLocation(projectId), + id: esIndex, + body: metadataContent, + refresh: 'true', + }); + + return getProjectIndex(es)({ + projectId, + graphqlField: serializedGqlField, + }); + } else { + throw new UserInputError(`no project with ID ${projectId} was found`); + } + }; // because different metadata entities write to the same ES document, update operations need to be queued up to a single concurrency controlled task queue for each project. This factory creates a task queue manager for this purpose. const createProjectQueueManager = () => { - const queues: { - [projectId: string]: any; - } = {}; - return { - getQueue: (projectId: string) => { - if (!queues[projectId]) { - queues[projectId] = new Qew(); - } - return queues[projectId]; - }, - }; + const queues: Record = {}; + return { + getQueue: (projectId: string) => { + if (!queues[projectId]) { + queues[projectId] = new Qew(); + } + return queues[projectId]; + }, + }; }; // pretty bad, since we're just taking anything right now in run time, but at least graphQl will ensure `metaData` is typed in runtime const projectQueueManager = createProjectQueueManager(); export const updateProjectIndexMetadata = - (es: Client) => - async ({ - projectId, - metaData, - }: { - projectId: string; - metaData: I_ProjectIndexMetadataUpdateDoc; - }): Promise => { - const queue = projectQueueManager.getQueue(projectId); - - return queue.pushProm(async () => { - await es.update({ - ...getProjectMetadataEsLocation(projectId), - id: metaData.index, - body: { - doc: metaData, - }, - refresh: 'true', - }); - - const output = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === metaData.name, - ); - - return output; - }); - }; - -export const getProjectIndex = - (es: Client) => - async ({ projectId, graphqlField }: IIndexQueryInput): Promise => { - try { - const output = (await getProjectMetadata(es)(projectId)).find( - ({ graphqlField: _graphqlField }) => graphqlField === _graphqlField, - ); - return output; - } catch { - throw new UserInputError(`could not find index ${graphqlField} of project ${projectId}`); - } - }; + (es: SearchClient) => + async ({ + projectId, + metaData, + }: { + projectId: string; + metaData: I_ProjectIndexMetadataUpdateDoc; + }): Promise => { + const queue = projectQueueManager.getQueue(projectId); + + return queue.pushProm(async () => { + await es.update({ + ...getProjectMetadataEsLocation(projectId), + id: metaData.index, + body: { + doc: metaData, + }, + refresh: 'true', + }); + + const output = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === metaData.name); + + return output; + }); + }; export const removeProjectIndex = - (es: Client) => - async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { - try { - const removedIndexMetadata = await getProjectIndex(es)({ - projectId, - graphqlField, - }); - await es.delete({ - ...getProjectMetadataEsLocation(projectId), - id: removedIndexMetadata.esIndex as string, - refresh: 'true', - }); - return removedIndexMetadata; - } catch (err) { - throw new UserInputError(`could not remove index ${graphqlField} of project ${projectId}`); - } - }; + (es: SearchClient) => + async ({ projectId, graphqlField }: IIndexRemovalMutationInput): Promise => { + try { + const removedIndexMetadata = await getProjectIndex(es)({ + projectId, + graphqlField, + }); + await es.delete({ + ...getProjectMetadataEsLocation(projectId), + id: removedIndexMetadata.esIndex as string, + refresh: 'true', + }); + return removedIndexMetadata; + } catch (err) { + throw new UserInputError(`could not remove index ${graphqlField} of project ${projectId}`); + } + }; diff --git a/modules/server/src/admin/schemas/MatchboxState/utils.ts b/modules/server/src/admin/schemas/MatchboxState/utils.ts index 1e1264ee4..71b179eb4 100644 --- a/modules/server/src/admin/schemas/MatchboxState/utils.ts +++ b/modules/server/src/admin/schemas/MatchboxState/utils.ts @@ -1,69 +1,66 @@ -import { Client } from '@elastic/elasticsearch'; +import { type SearchClient } from '#searchClient/types.js'; + +import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping/index.js'; +import { replaceBy, timestamp } from '../../services/index.js'; +import { type I_GqlExtendedFieldMapping } from '../ExtendedMapping/types.js'; +import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils.js'; -import { mappingToMatchBoxState as extendedFieldsToMatchBoxState } from '../../../mapping'; -import { replaceBy, timestamp } from '../../services'; -import { I_GqlExtendedFieldMapping } from '../ExtendedMapping/types'; -import { getProjectStorageMetadata, updateProjectIndexMetadata } from '../IndexSchema/utils'; import { - I_MatchBoxField, - I_MatchBoxState, - I_MatchBoxStateQueryInput, - I_SaveMatchBoxStateMutationInput, -} from './types'; + type I_MatchBoxField, + type I_MatchBoxState, + type I_MatchBoxStateQueryInput, + type I_SaveMatchBoxStateMutationInput, +} from './types.js'; export const createMatchboxState = ({ - extendedFields, - graphqlField, + extendedFields, + graphqlField, }: { - extendedFields: Array; - graphqlField: string; + extendedFields: I_GqlExtendedFieldMapping[]; + graphqlField: string; }): I_MatchBoxState => { - const fields: I_MatchBoxField[] = extendedFieldsToMatchBoxState({ - extendedFields, - name: graphqlField, - }); - return { state: fields, timestamp: timestamp() }; + const fields: I_MatchBoxField[] = extendedFieldsToMatchBoxState({ + extendedFields, + name: graphqlField, + }); + return { state: fields, timestamp: timestamp() }; }; export const getMatchBoxState = - (es: Client) => - async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - return currentMetadata.config['matchbox-state']; - }; + (es: SearchClient) => + async ({ graphqlField, projectId }: I_MatchBoxStateQueryInput): Promise => { + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + return currentMetadata?.config['matchbox-state']; + }; export const saveMatchBoxState = - (es: Client) => - async ({ - graphqlField, - projectId, - state: updatedMatchboxFields, - }: I_SaveMatchBoxStateMutationInput): Promise => { - const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find( - (i) => i.name === graphqlField, - ); - const currentMatchboxFields = currentMetadata.config['matchbox-state'].state; - const newMatchboxState: I_MatchBoxState = { - timestamp: timestamp(), - state: replaceBy( - currentMatchboxFields, - updatedMatchboxFields, - ({ field: field1 }, { field: field2 }) => field1 === field2, - ), - }; + (es: SearchClient) => + async ({ + graphqlField, + projectId, + state: updatedMatchboxFields, + }: I_SaveMatchBoxStateMutationInput): Promise => { + const currentMetadata = (await getProjectStorageMetadata(es)(projectId)).find((i) => i.name === graphqlField); + const currentMatchboxFields = currentMetadata?.config['matchbox-state'].state || []; + const newMatchboxState: I_MatchBoxState = { + timestamp: timestamp(), + state: replaceBy( + currentMatchboxFields, + updatedMatchboxFields, + ({ field: field1 }, { field: field2 }) => field1 === field2, + ), + }; - await updateProjectIndexMetadata(es)({ - projectId, - metaData: { - index: currentMetadata.index, - name: currentMetadata.name, - config: { - 'matchbox-state': newMatchboxState, - }, - }, - }); + await updateProjectIndexMetadata(es)({ + projectId, + metaData: { + index: currentMetadata?.index, + name: currentMetadata?.name, + config: { + 'matchbox-state': newMatchboxState, + }, + }, + }); - return newMatchboxState; - }; + return newMatchboxState; + }; diff --git a/modules/server/src/admin/schemas/ProjectSchema/index.ts b/modules/server/src/admin/schemas/ProjectSchema/index.ts index 489cbcc60..71b3654f3 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/index.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/index.ts @@ -1,14 +1,14 @@ import { addMocksToSchema } from '@graphql-tools/mock'; import { makeExecutableSchema } from 'graphql-tools'; -import { createResolvers } from './resolvers'; -import typeDefs from './schemaTypeDefs'; +import { createResolvers } from './resolvers.js'; +import typeDefs from './schemaTypeDefs.js'; export const createSchema = async () => { - const schema = makeExecutableSchema({ - typeDefs: await typeDefs(), - resolvers: await createResolvers(), - }); - addMocksToSchema({ schema, preserveResolvers: true }); - return schema; + const schema = makeExecutableSchema({ + typeDefs: await typeDefs(), + resolvers: await createResolvers(), + }); + addMocksToSchema({ schema, preserveResolvers: true }); + return schema; }; diff --git a/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts b/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts index 711ccccfe..769ed13eb 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/resolvers.ts @@ -1,45 +1,36 @@ -import { addArrangerProject, getArrangerProjects, removeArrangerProject } from './utils'; -import { IArrangerProject, IProjectQueryInput } from './types'; -import { Resolver } from '../types'; +import { type Resolver } from '../types.js'; -const projectsQueryResolver: Resolver> = async (_, args, { es }, info) => - getArrangerProjects(es); +import { type IArrangerProject, type IProjectQueryInput } from './types.js'; +import { addArrangerProject, getArrangerProjects, removeArrangerProject } from './utils.js'; -const singleProjectQueryResolver: Resolver = async ( - _, - { id }, - { es }, - info, -) => { - const projects = await getArrangerProjects(es); - return projects.find(({ id: _id }: { id: String }) => id === _id); +const projectsQueryResolver: Resolver = async (_, args, { es }, info) => + await getArrangerProjects(es); + +const singleProjectQueryResolver: Resolver = async (_, { id }, { es }, info) => { + const projects = await getArrangerProjects(es); + return projects.find(({ id: _id }: { id: string }) => id === _id); }; -const newProjectMutationResolver: Resolver = async ( - _, - { id }, - { es }, - info, -) => - addArrangerProject(es)(id).catch((err: Error) => { - err.message = 'potential project ID conflict'; - return Promise.reject(err); - }); +const newProjectMutationResolver: Resolver = async (_, { id }, { es }, info) => + addArrangerProject(es)(id).catch((err: Error) => { + err.message = 'potential project ID conflict'; + return Promise.reject(err); + }); const deleteProjectMutationResolver: Resolver = async ( - _, - { id }, - { es }, - info, + _, + { id }, + { es }, + info, ) => removeArrangerProject(es)(id); export const createResolvers = async () => ({ - Query: { - projects: projectsQueryResolver, - project: singleProjectQueryResolver, - }, - Mutation: { - newProject: newProjectMutationResolver, - deleteProject: deleteProjectMutationResolver, - }, + Query: { + projects: projectsQueryResolver, + project: singleProjectQueryResolver, + }, + Mutation: { + newProject: newProjectMutationResolver, + deleteProject: deleteProjectMutationResolver, + }, }); diff --git a/modules/server/src/admin/schemas/ProjectSchema/utils.ts b/modules/server/src/admin/schemas/ProjectSchema/utils.ts index d5a06bdcf..f51a88b9e 100644 --- a/modules/server/src/admin/schemas/ProjectSchema/utils.ts +++ b/modules/server/src/admin/schemas/ProjectSchema/utils.ts @@ -1,82 +1,84 @@ -import { Client } from '@elastic/elasticsearch'; -import { constants } from '../../services/constants'; -import { serializeToEsId } from '../../services'; -import { IArrangerProject } from './types'; -import { getProjectMetadataEsLocation } from '../IndexSchema/utils'; +import { type SearchClient } from '#searchClient/types.js'; + +import { constants } from '../../services/constants.js'; +import { serializeToEsId } from '../../services/index.js'; +import { getProjectMetadataEsLocation } from '../IndexSchema/utils.js'; + +import { type IArrangerProject } from './types.js'; const { ARRANGER_PROJECT_INDEX } = constants; export const newArrangerProject = (id: string): IArrangerProject => ({ - id: serializeToEsId(id), - active: true, - timestamp: new Date().toISOString(), + id: serializeToEsId(id), + active: true, + timestamp: new Date().toISOString(), }); -export const getArrangerProjects = async (es: Client): Promise> => { - const { - body: { - hits: { hits }, - }, - }: { - body: { - hits: { - hits: Array<{ - _source: any; - }>; - }; - }; - } = await es - .search({ - index: ARRANGER_PROJECT_INDEX, - }) - .catch(() => ({ - body: { - hits: { - hits: [], - }, - }, - })); - return hits.map(({ _source }) => _source as IArrangerProject); +export const getArrangerProjects = async (es: SearchClient): Promise => { + const { + body: { + hits: { hits }, + }, + }: { + body: { + hits: { + hits: { + _source: any; + }[]; + }; + }; + } = await es + .search({ + index: ARRANGER_PROJECT_INDEX, + }) + .catch(() => ({ + body: { + hits: { + hits: [], + }, + }, + })); + return hits.map(({ _source }) => _source as IArrangerProject); }; export const addArrangerProject = - (es: Client) => - async (id: string): Promise => { - //id must be lower case - const _id = serializeToEsId(id); - const newProject = newArrangerProject(_id); - await Promise.all([ - await es.indices.create({ index: getProjectMetadataEsLocation(id).index }), - await es - .create({ - index: ARRANGER_PROJECT_INDEX, - id: _id, - body: newProject, - refresh: 'true', - }) - .then(() => newProject) - .catch(Promise.reject), - ]); - return getArrangerProjects(es); - }; + (es: SearchClient) => + async (id: string): Promise => { + //id must be lower case + const _id = serializeToEsId(id); + const newProject = newArrangerProject(_id); + await Promise.all([ + await es.indices.create({ index: getProjectMetadataEsLocation(id).index }), + await es + .create({ + index: ARRANGER_PROJECT_INDEX, + id: _id, + body: newProject, + refresh: 'true', + }) + .then(() => newProject) + .catch(Promise.reject), + ]); + return getArrangerProjects(es); + }; export const removeArrangerProject = - (es: Client) => - async (id: string): Promise => { - const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); - if (existingProject) { - await Promise.all([ - es.indices.delete({ - index: getProjectMetadataEsLocation(id).index, - }), - es.delete({ - index: ARRANGER_PROJECT_INDEX, - id: id, - refresh: 'true', - }), - ]); - return getArrangerProjects(es); - } else { - return Promise.reject(`No project with id ${id} was found`); - } - }; + (es: SearchClient) => + async (id: string): Promise => { + const existingProject = (await getArrangerProjects(es)).find(({ id: _id }) => id === _id); + if (existingProject) { + await Promise.all([ + es.indices.delete({ + index: getProjectMetadataEsLocation(id).index, + }), + es.delete({ + index: ARRANGER_PROJECT_INDEX, + id: id, + refresh: 'true', + }), + ]); + return getArrangerProjects(es); + } else { + return Promise.reject(`No project with id ${id} was found`); + } + }; diff --git a/modules/server/src/admin/schemas/types.ts b/modules/server/src/admin/schemas/types.ts index 31cb178aa..5366fea92 100644 --- a/modules/server/src/admin/schemas/types.ts +++ b/modules/server/src/admin/schemas/types.ts @@ -1,18 +1,19 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { IQueryContext } from '../types'; -import { MergeInfo } from 'graphql-tools'; +import { type GraphQLResolveInfo } from 'graphql'; +import { type MergeInfo } from 'graphql-tools'; + +import { type IQueryContext } from '../types.js'; export type ResolverOutput = T | Promise; export interface EsIndexLocation { - esIndex: string; + esIndex: string; } -export type Resolver = - | (( - a: any, - args: Args, - c: IQueryContext, - d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, - ) => ResolverOutput) - | ResolverOutput; +export type Resolver = + | (( + a: any, + args: Args, + c: IQueryContext, + d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, + ) => ResolverOutput) + | ResolverOutput; diff --git a/modules/server/src/admin/services/elasticsearch/index.ts b/modules/server/src/admin/services/elasticsearch/index.ts index 176faa353..1f19239ad 100644 --- a/modules/server/src/admin/services/elasticsearch/index.ts +++ b/modules/server/src/admin/services/elasticsearch/index.ts @@ -1,27 +1,29 @@ -import { Client } from '@elastic/elasticsearch'; -import { EsMapping } from './types'; +import getSearchClient from '#searchClient/index.js'; +import { type SearchClient } from '#searchClient/types.js'; -export const createClient = (esHost: string, esUser: string, esPass: string) => { - const esConf = { node: esHost }; - if (esUser && esPass) { - esConf['auth'] = { - username: esUser, - password: esPass, - }; - } - return new Client(esConf); +import { type EsMapping } from './types.js'; + +export const createClient = async (esHost: string, esUser: string, esPass: string) => { + const esConf = { node: esHost }; + if (esUser && esPass) { + esConf['auth'] = { + username: esUser, + password: esPass, + }; + } + return await getSearchClient(esConf); }; export const getEsMapping = - (es: Client) => - async ({ - esIndex, - }: { - esIndex: string; - esType?: string; //deprecated - }): Promise => { - const response = await es.indices.getMapping({ - index: esIndex, - }); - return response.body as EsMapping; - }; + (es: SearchClient) => + async ({ + esIndex, + }: { + esIndex: string; + esType?: string; //deprecated + }): Promise => { + const response = await es.indices.getMapping({ + index: esIndex, + }); + return response.body as EsMapping; + }; diff --git a/modules/server/src/admin/types.ts b/modules/server/src/admin/types.ts index c697b1a7a..1ce511792 100644 --- a/modules/server/src/admin/types.ts +++ b/modules/server/src/admin/types.ts @@ -1,28 +1,29 @@ -import { GraphQLResolveInfo } from 'graphql'; -import { MergeInfo } from 'graphql-tools'; -import { Client } from '@elastic/elasticsearch'; +import { type GraphQLResolveInfo } from 'graphql'; +import { type MergeInfo } from 'graphql-tools'; -export interface AdminApiConfig { - esHost: string; - esUser: string; - esPass: string; -} -export interface IQueryContext { - es: Client; -} +import { type SearchClient } from '#searchClient/types.js'; + +export type AdminApiConfig = { + esHost: string; + esUser: string; + esPass: string; +}; +export type IQueryContext = { + es: SearchClient; +}; export type ResolverOutput = T | Promise; -export type MergeResolver = - | (( - a: any, - args: Args, - c: IQueryContext, - d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, - ) => ResolverOutput) - | ResolverOutput; +export type MergeResolver = + | (( + a: any, + args: Args, + c: IQueryContext, + d: GraphQLResolveInfo & { mergeInfo: MergeInfo }, + ) => ResolverOutput) + | ResolverOutput; -export interface I_MergeSchema { - fragment: string; - resolve: MergeResolver; -} +export type I_MergeSchema = { + fragment: string; + resolve: MergeResolver; +}; diff --git a/modules/server/src/config/constants.ts b/modules/server/src/config/constants.ts index f0a855fa6..dcf6bd735 100644 --- a/modules/server/src/config/constants.ts +++ b/modules/server/src/config/constants.ts @@ -16,6 +16,7 @@ export const ES_INDEX = process.env.ES_INDEX || ''; export const ES_LOG = process.env.ES_LOG?.split?.(',') || 'error'; export const ES_PASS = process.env.ES_PASS || ''; export const ES_USER = process.env.ES_USER || ''; +export const SEARCH_CLIENT = process.env.SEARCH_CLIENT || ''; export const MAX_DOWNLOAD_ROWS = stringToNumber(process.env.MAX_DOWNLOAD_ROWS) || 100; export const MAX_LIVE_VERSIONS = process.env.MAX_LIVE_VERSIONS || 3; export const MAX_RESULTS_WINDOW = stringToNumber(process.env.MAX_RESULTS_WINDOW) || 10000; diff --git a/modules/server/src/config/utils/index.ts b/modules/server/src/config/utils/index.ts index 66d38186e..5ccdaabee 100644 --- a/modules/server/src/config/utils/index.ts +++ b/modules/server/src/config/utils/index.ts @@ -1,14 +1,13 @@ -import type { Client } from '@elastic/elasticsearch'; - import { ENV_CONFIG } from '#config/index.js'; import { type ConfigObject, configProperties } from '#config/types.js'; import { setsMapping } from '#schema/index.js'; +import { type SearchClient } from '#searchClient/types.js'; export const initializeSets = async ({ esClient, setsIndex: setsIndexParam, }: { - esClient: Client; + esClient: SearchClient; setsIndex: string; }): Promise => { ENV_CONFIG.DEBUG_MODE && console.log(`Attempting to create Sets index "${setsIndexParam}"...`); diff --git a/modules/server/src/download/index.js b/modules/server/src/download/index.js index e92b659ae..42b8ea591 100644 --- a/modules/server/src/download/index.js +++ b/modules/server/src/download/index.js @@ -91,8 +91,6 @@ export const dataStream = async ({ ctx, params }) => { }; const download = ({ enableAdmin }) => { - console.log('enableAdmin', enableAdmin); - const router = Router(); router.use(urlencoded({ extended: true })); diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts index 65afa7e70..62d44d985 100644 --- a/modules/server/src/gqlServer.ts +++ b/modules/server/src/gqlServer.ts @@ -1,8 +1,9 @@ -import { Client } from '@elastic/elasticsearch'; import { type GraphQLResolveInfo } from 'graphql'; +import { type SearchClient } from './searchClient/types.js'; + export type Context = { - esClient: Client; + esClient: SearchClient; }; export type Root = Record; @@ -19,7 +20,7 @@ export type ResolverOutput = T | Promise; * @return Returns resolved value; */ type DefaultRoot = Root; -export type Resolver = ( +export type Resolver = ( root: Root, args: QueryArgs, context: Context, diff --git a/modules/server/src/index.js b/modules/server/src/index.js index 5fef2fe01..18ec737b9 100644 --- a/modules/server/src/index.js +++ b/modules/server/src/index.js @@ -1,3 +1,4 @@ export { default as App } from './app.js'; export { createSchemasFromConfigs, default as getGraphQLRoutes } from './graphqlRoutes.js'; export { default } from './server.js'; +export { default as getSearchClient } from './searchClient/index.js'; diff --git a/modules/server/src/mapping/resolveAggregations.ts b/modules/server/src/mapping/resolveAggregations.ts index 8b8b349a1..34b9c5feb 100644 --- a/modules/server/src/mapping/resolveAggregations.ts +++ b/modules/server/src/mapping/resolveAggregations.ts @@ -102,12 +102,13 @@ const getAggregationsResolver = ({ const response = await esSearch(esClient)({ index: type.index, size: 0, - // @ts-expect-error - valid search query parameter in ES 7.17, not in types _source: false, + // @ts-expect-error - valid search query parameter in ES 7.17, not in types body, }); + const aggregations = flattenAggregations({ - aggregations: response.aggregations, + aggregations: response?.body?.aggregations, includeMissing: include_missing, }); diff --git a/modules/server/src/mapping/resolveHits.js b/modules/server/src/mapping/resolveHits.js index 0b821bbe4..7d1142039 100644 --- a/modules/server/src/mapping/resolveHits.js +++ b/modules/server/src/mapping/resolveHits.js @@ -1,10 +1,6 @@ import getFields from 'graphql-fields'; import { JSONPath } from 'jsonpath-plus'; -import { - chunk, - isObject, - flattenDeep -} from 'lodash-es'; +import { chunk, isObject, flattenDeep } from 'lodash-es'; // import { ENV_CONFIG } from '#config/index.js'; import { buildQuery, isESValueSafeJSInt } from '#middleware/index.js'; @@ -38,13 +34,9 @@ const processChunk = ({ copyToSourceFields, extendedFieldsObj, hits, nestedField json: node, path: path .split('.') - .reduce( - (acc, part, index) => - index === 0 ? `$.${part}` : `${acc}..${part}`, - '', - ), - }) - ) + .reduce((acc, part, index) => (index === 0 ? `$.${part}` : `${acc}..${part}`), ''), + }), + ), ); return found; }, {}); @@ -52,36 +44,25 @@ const processChunk = ({ copyToSourceFields, extendedFieldsObj, hits, nestedField }; return hits.map((hit) => { - const joinParent = (parent, fieldName) => ( - parent ? `${parent}.${fieldName}` : fieldName - ); + const joinParent = (parent, fieldName) => (parent ? `${parent}.${fieldName}` : fieldName); - const resolveNested = ({ - node, - nestedFieldNames, - parent = '' - }) => { + const resolveNested = ({ node, nestedFieldNames, parent = '' }) => { if (!isObject(node) || !node) { // Backwards compatibility for Array fields when data has not been migrated - return extendedFieldsObj?.[parent]?.isArray && !Array.isArray(node) - ? [node] - : node; + return extendedFieldsObj?.[parent]?.isArray && !Array.isArray(node) ? [node] : node; } - return Object - .entries(node) - .reduce((acc, entry) => { - const fieldName = entry[0]; - const hits = entry[1]; + return Object.entries(node).reduce((acc, entry) => { + const fieldName = entry[0]; + const hits = entry[1]; - // TODO: inner hits query if necessary - const fullPath = joinParent(parent, fieldName); - const areHitsNested = nestedFieldNames?.includes(fullPath); - const hitsAreActuallyNested = areHitsNested && - Array.isArray(hits); + // TODO: inner hits query if necessary + const fullPath = joinParent(parent, fieldName); + const areHitsNested = nestedFieldNames?.includes(fullPath); + const hitsAreActuallyNested = areHitsNested && Array.isArray(hits); - acc[fieldName] = hitsAreActuallyNested - ? { + acc[fieldName] = hitsAreActuallyNested + ? { hits: { edges: hits.map((node) => ({ node: Object.assign( @@ -97,8 +78,8 @@ const processChunk = ({ copyToSourceFields, extendedFieldsObj, hits, nestedField total: hits.length, }, } - : isObject(hits) && hits - ? Object.assign( + : isObject(hits) && hits + ? Object.assign( hits.constructor(), resolveNested({ node: hits, @@ -106,14 +87,14 @@ const processChunk = ({ copyToSourceFields, extendedFieldsObj, hits, nestedField parent: fullPath, }), ) - : resolveNested({ + : resolveNested({ node: hits, nestedFieldNames, parent: fullPath, }); - return acc; - }, {}); + return acc; + }, {}); }; const source = hit._source; @@ -126,9 +107,7 @@ const processChunk = ({ copyToSourceFields, extendedFieldsObj, hits, nestedField const copied_to_nodes = resolveCopiedTo({ node: source }); return { - searchAfter: hit.sort - ? hit.sort.map(isESValueSafeJSInt) - : [], + searchAfter: hit.sort ? hit.sort.map(isESValueSafeJSInt) : [], node: Object.assign( source, // we're not afraid of mutating source here! { id: hit._id }, @@ -160,55 +139,53 @@ export const hitsToEdges = ({ map-reduce based on # of cores available. Otherwise, only one child-process is spawn for compute */ - const dataSize = hits.hits.length; - const chunkSize = dataSize > 1000 - ? dataSize / systemCores + (dataSize % systemCores) - : dataSize; + const dataSize = hits?.hits?.length || 0; + const chunkSize = dataSize > 1000 ? dataSize / systemCores + (dataSize % systemCores) : dataSize; const chunks = chunk(hits.hits, chunkSize); - const chunkPromises = chunks.map( - (chunk) => { - const params = { - copyToSourceFields, - extendedFieldsObj, - hits: chunk, - nestedFieldNames, - }; - - //Parallel.spawn output has a .then but it's not returning an actual promise - return new Promise((resolve, reject) => { - if (chunkSize >= dataSize) { - try { - const results = processChunk(params); - return resolve(results); - } catch (err) { - return reject(err); - } + const chunkPromises = chunks.map((chunk) => { + const params = { + copyToSourceFields, + extendedFieldsObj, + hits: chunk, + nestedFieldNames, + }; + + //Parallel.spawn output has a .then but it's not returning an actual promise + return new Promise((resolve, reject) => { + if (chunkSize >= dataSize) { + try { + const results = processChunk(params); + return resolve(results); + } catch (err) { + return reject(err); } + } - new Parallel(params) - .require(flattenDeep, isObject, JSONPath) - .spawn(processChunk) - .then(resolve, (err) => { - reject(err); - }); - }); + new Parallel(params) + .require(flattenDeep, isObject, JSONPath) + .spawn(processChunk) + .then(resolve, (err) => { + reject(err); + }); }); + }); return Promise.all(chunkPromises) .then((chunks) => { - return chunks.reduce((acc, chunk) => - acc.concat(chunk), [] - ); + return chunks.reduce((acc, chunk) => acc.concat(chunk), []); }) - .catch((err) => - console.log("err", err) - ); + .catch((err) => console.log('err', err)); }; export default ({ type, Parallel, getServerSideFilter }) => - async (obj, { first = 10, offset = 0, filters, score, sort, searchAfter, trackTotalHits = true }, context, info) => { + async ( + obj, + { first = 10, offset = 0, filters, score, sort, searchAfter, trackTotalHits = true }, + context, + info, + ) => { const fields = getFields(info); const nestedFieldNames = type.nested_fieldNames; @@ -239,7 +216,13 @@ export default ({ type, Parallel, getServerSideFilter }) => return { [fieldName]: { - missing: missing ? (missing === 'first' ? '_first' : '_last') : order === 'asc' ? '_first' : '_last', + missing: missing + ? missing === 'first' + ? '_first' + : '_last' + : order === 'asc' + ? '_first' + : '_last', order, ...rest, ...(nested_path?.length ? { nested: { path: nested_path } } : {}), @@ -254,7 +237,7 @@ export default ({ type, Parallel, getServerSideFilter }) => const copyToSourceFields = findCopyToSourceFields(type.mapping); - const { hits } = await esSearch(esClient)({ + const searchResult = await esSearch(esClient)({ index: type.index, size: first, from: offset, @@ -267,6 +250,8 @@ export default ({ type, Parallel, getServerSideFilter }) => body, }); + const hits = searchResult?.body?.hits || { hits: [], total: { value: 0 } }; + return { edges: () => hitsToEdges({ diff --git a/modules/server/src/mapping/resolveSets.js b/modules/server/src/mapping/resolveSets.js index 74d67cb4f..d01302c53 100644 --- a/modules/server/src/mapping/resolveSets.js +++ b/modules/server/src/mapping/resolveSets.js @@ -6,15 +6,7 @@ import { buildQuery } from '#middleware/index.js'; import compileFilter from './utils/compileFilter.js'; import esSearch from './utils/esSearch.js'; -const retrieveSetIds = async ({ - esClient, - index, - query, - path, - sort, - BULK_SIZE = 1000, - trackTotalHits = true -}) => { +const retrieveSetIds = async ({ esClient, index, query, path, sort, BULK_SIZE = 1000, trackTotalHits = true }) => { const search = async ({ searchAfter } = {}) => { const body = { ...(!isEmpty(query) && { query }), @@ -30,16 +22,18 @@ const retrieveSetIds = async ({ track_total_hits: trackTotalHits, body, }); - const ids = response.hits.hits.map((x) => get(x, `_source.${path.split('__').join('.')}`, x._id || '')); + + const hits = response?.body?.hits; + const ids = hits?.hits.map((x) => get(x, `_source.${path.split('__').join('.')}`, x._id || '')) || []; const nextSearchAfter = sort - .map(({ fieldName }) => response.hits.hits.map((x) => x._source[fieldName] || x[fieldName])) + .map(({ fieldName }) => hits?.hits.map((x) => x._source[fieldName] || x[fieldName])) .reduce((acc, vals) => [...acc, ...vals.slice(-1)], []); return { ids, searchAfter: nextSearchAfter, - total: response.hits.total.value, + total: hits?.total.value, }; }; const handleResult = async ({ searchAfter, total, ids = [] }) => { @@ -55,52 +49,52 @@ const retrieveSetIds = async ({ export const saveSet = ({ getServerSideFilter, setsIndex, types }) => - async (obj, { type, userId, sqon, path, sort, refresh = 'WAIT_FOR' }, context) => { - const { nested_fieldNames: nestedFieldNames, index } = types.find(([, x]) => x.name === type)[1]; - const { esClient } = context; + async (obj, { type, userId, sqon, path, sort, refresh = 'WAIT_FOR' }, context) => { + const { nested_fieldNames: nestedFieldNames, index } = types.find(([, x]) => x.name === type)[1]; + const { esClient } = context; - const query = buildQuery({ - caller: 'resolveSets', - nestedFieldNames, - filters: compileFilter({ - clientSideFilter: sqon, - serverSideFilter: getServerSideFilter(context), - }), - }); + const query = buildQuery({ + caller: 'resolveSets', + nestedFieldNames, + filters: compileFilter({ + clientSideFilter: sqon, + serverSideFilter: getServerSideFilter(context), + }), + }); - const ids = await retrieveSetIds({ - esClient, - index, - query, - path, - sort: - sort && sort.length - ? sort - : [ + const ids = await retrieveSetIds({ + esClient, + index, + query, + path, + sort: + sort && sort.length + ? sort + : [ { fieldName: '_id', order: 'asc', }, ], - }); + }); - const body = { - setId: uuid(), - createdAt: Date.now(), - ids, - type, - path, - sqon, - userId, - size: ids.length, - }; + const body = { + setId: uuid(), + createdAt: Date.now(), + ids, + type, + path, + sqon, + userId, + size: ids.length, + }; - await esClient.index({ - index: setsIndex, - id: body.setId, - refresh: refresh.toLowerCase(), - body, - }); + await esClient.index({ + index: setsIndex, + id: body.setId, + refresh: refresh.toLowerCase(), + body, + }); - return body; - }; + return body; + }; diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts index 659a70b51..eacc39c18 100644 --- a/modules/server/src/mapping/utils/esSearch.ts +++ b/modules/server/src/mapping/utils/esSearch.ts @@ -1,5 +1,5 @@ -import { Client, type RequestParams } from '@elastic/elasticsearch'; +import { type SearchClient, type SearchResponse } from '#searchClient/types.js'; -export default (esClient: Client) => async (params: RequestParams.Search) => { - return (await esClient?.search(params))?.body; +export default (esClient: SearchClient) => async (params: SearchResponse) => { + return await esClient.search(params); }; diff --git a/modules/server/src/mapping/utils/fetchMapping.ts b/modules/server/src/mapping/utils/fetchMapping.ts index 3cddb929b..21b7bd2b7 100644 --- a/modules/server/src/mapping/utils/fetchMapping.ts +++ b/modules/server/src/mapping/utils/fetchMapping.ts @@ -1,7 +1,8 @@ -import type { Client } from '@elastic/elasticsearch/api/new'; import type { CatAliasesAliasesRecord } from '@elastic/elasticsearch/api/types'; -export const getESAliases = async (esClient: Client) => { +import { type SearchClient } from '#searchClient/types.js'; + +export const getESAliases = async (esClient: SearchClient) => { const { body } = await esClient.cat.aliases({ format: 'json' }); return body; @@ -10,7 +11,7 @@ export const getESAliases = async (esClient: Client) => { export const checkESAlias = (aliases: CatAliasesAliasesRecord[], possibleAlias: string) => aliases?.find((foundIndex = { alias: undefined }) => foundIndex.alias === possibleAlias)?.index; -export const fetchMapping = async ({ esClient, index }: { esClient: Client; index: string }) => { +export const fetchMapping = async ({ esClient, index }: { esClient: SearchClient; index: string }) => { if (esClient) { console.log(`Fetching ES mapping for "${index}"...`); const aliases = await getESAliases(esClient); @@ -19,7 +20,7 @@ export const fetchMapping = async ({ esClient, index }: { esClient: Client; inde const accessor = alias || index; - const mapping = await esClient?.indices + const mapping = await esClient.indices .getMapping({ index: accessor, }) diff --git a/modules/server/src/schema/Root.ts b/modules/server/src/schema/Root.ts index 9e93626c4..2c52080b6 100644 --- a/modules/server/src/schema/Root.ts +++ b/modules/server/src/schema/Root.ts @@ -1,4 +1,3 @@ -import type { Client } from '@elastic/elasticsearch/api/new'; import { GraphQLDate } from 'graphql-scalars'; import { GraphQLJSON } from 'graphql-type-json'; import { startCase } from 'lodash-es'; @@ -7,6 +6,7 @@ import Parallel from 'paralleljs'; import { ENV_CONFIG } from '#config/index.js'; import { createConnectionResolvers, saveSet, mappingToFields } from '#mapping/index.js'; import { checkESAlias, getESAliases } from '#mapping/utils/fetchMapping.js'; +import { type SearchClient } from '#searchClient/types.js'; import { typeDefs as AggregationsTypeDefs } from './Aggregations.js'; import ConfigsTypeDefs from './configQuery.js'; @@ -82,7 +82,7 @@ export const resolvers = ({ enableAdmin, types, rootTypes, scalarTypes, getServe Date: GraphQLDate, Root: { viewer: resolveObject, - hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: Client }) => { + hasValidConfig: async (obj, { documentType, index }, { esClient }: { esClient: SearchClient }) => { if (documentType) { if (index) { const [_, type] = types.find(([name]) => name === documentType) || []; diff --git a/modules/server/src/searchClient/createElasticSearchClient.ts b/modules/server/src/searchClient/createElasticSearchClient.ts new file mode 100644 index 000000000..b27b86235 --- /dev/null +++ b/modules/server/src/searchClient/createElasticSearchClient.ts @@ -0,0 +1,88 @@ +import { Client, type ClientOptions } from '@elastic/elasticsearch'; + +import type { SearchClient } from './types.js'; + +export type ESClientOptions = ClientOptions & { + clientType: 'elasticsearch'; +}; + +export function createElasticSearchClient(options: ESClientOptions) { + const elasticSearchClient = new Client(options); + + const searchClient: SearchClient = { + indices: { + close: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.close(input, options); + return output; + }, + create: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.create(input, options); + return output; + }, + delete: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.delete(input, options); + return output; + }, + exists: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.exists(input, options); + return output; + }, + getMapping: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.getMapping(input, options); + return output; + }, + putSettings: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.putSettings(input, options); + return output; + }, + putMapping: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.putMapping(input, options); + return output; + }, + open: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.open(input, options); + return output; + }, + refresh: async (input: any, options?: any) => { + const output = await elasticSearchClient.indices.refresh(input, options); + return output; + }, + }, + cat: { + aliases: async (input: any, options?: any) => { + const output = await elasticSearchClient.cat.aliases(input, options); + return output; + }, + }, + bulk: async (input: any, options?: any) => { + const output = await elasticSearchClient.bulk(input, options); + return output; + }, + index: async (input: any, options?: any) => { + const output = await elasticSearchClient.index(input, options); + return output; + }, + search: async (input: any, options?: any) => { + const output = await elasticSearchClient.search(input, options); + return output; + }, + update: async (input: any, options?: any) => { + const output = await elasticSearchClient.update(input, options); + return output; + }, + create: async (input: any, options?: any) => { + const output = await elasticSearchClient.create(input, options); + return output; + }, + delete: async (input: any, options?: any) => { + const output = await elasticSearchClient.delete(input, options); + return output; + }, + deleteByQuery: async (input: any, options?: any) => { + const output = await elasticSearchClient.deleteByQuery(input, options); + return output; + }, + }; + + return searchClient; +} diff --git a/modules/server/src/searchClient/createOpenSearchClient.ts b/modules/server/src/searchClient/createOpenSearchClient.ts new file mode 100644 index 000000000..647d8420a --- /dev/null +++ b/modules/server/src/searchClient/createOpenSearchClient.ts @@ -0,0 +1,88 @@ +import { Client, type ClientOptions } from '@opensearch-project/opensearch'; + +import type { SearchClient } from './types.js'; + +export type OSClientOptions = ClientOptions & { + clientType: 'opensearch'; +}; + +export function createOpenSearchClient(options: OSClientOptions): SearchClient { + const openSearchClient = new Client(options); + + const searchClient: SearchClient = { + indices: { + close: async (input: any, options?: any) => { + const output = await openSearchClient.indices.close(input, options); + return output; + }, + create: async (input: any, options?: any) => { + const output = await openSearchClient.indices.create(input, options); + return output; + }, + delete: async (input: any, options?: any) => { + const output = await openSearchClient.indices.delete(input, options); + return output; + }, + exists: async (input: any, options?: any) => { + const output = await openSearchClient.indices.exists(input, options); + return output; + }, + getMapping: async (input: any, options?: any) => { + const output = await openSearchClient.indices.getMapping(input, options); + return output; + }, + putSettings: async (input: any, options?: any) => { + const output = await openSearchClient.indices.putSettings(input, options); + return output; + }, + putMapping: async (input: any, options?: any) => { + const output = await openSearchClient.indices.putMapping(input, options); + return output; + }, + open: async (input: any, options?: any) => { + const output = await openSearchClient.indices.open(input, options); + return output; + }, + refresh: async (input: any, options?: any) => { + const output = await openSearchClient.indices.refresh(input, options); + return output; + }, + }, + cat: { + aliases: async (input: any, options?: any) => { + const output = await openSearchClient.cat.aliases(input, options); + return output; + }, + }, + bulk: async (input: any, options?: any) => { + const output = await openSearchClient.bulk(input, options); + return output; + }, + create: async (input: any, options?: any) => { + const output = await openSearchClient.create(input, options); + return output; + }, + delete: async (input: any, options?: any) => { + const output = await openSearchClient.delete(input, options); + return output; + }, + deleteByQuery: async (input: any, options?: any) => { + const output = await openSearchClient.deleteByQuery(input, options); + return output; + }, + index: async (input: any, options?: any) => { + const output = await openSearchClient.index(input, options); + return output; + }, + search: async (input: any, options?: any) => { + const output = await openSearchClient.search(input, options); + return output; + }, + update: async (input: any, options?: any) => { + const output = await openSearchClient.update(input, options); + return output; + }, + }; + + return searchClient; +} diff --git a/modules/server/src/searchClient/index.ts b/modules/server/src/searchClient/index.ts new file mode 100644 index 000000000..226a8820a --- /dev/null +++ b/modules/server/src/searchClient/index.ts @@ -0,0 +1,57 @@ +import { createElasticSearchClient, type ESClientOptions } from './createElasticSearchClient.js'; +import { createOpenSearchClient, type OSClientOptions } from './createOpenSearchClient.js'; +import type { SearchClient, SupportedClientTypes, SearchConfig, SearchConfigWithClient } from './types.js'; + +export const supportedClientValues: SupportedClientTypes[] = ['opensearch', 'elasticsearch'] as const; + +export const createSearchClient = (clientConfig: SearchConfigWithClient): SearchClient => { + const { clientType } = clientConfig; + if (clientType === 'opensearch') { + const options: OSClientOptions = { ...clientConfig, clientType }; + return createOpenSearchClient(options); + } else { + const options: ESClientOptions = { ...clientConfig, clientType }; + return createElasticSearchClient(options); + } +}; + +/** + * Uses Cluster Info to determine Search Client version information + */ +const getClientVersion = async (config: SearchConfig) => { + try { + const response = await (await fetch(config.node)).json(); + if (!response?.version) { + throw new Error('Could not retrieve version information'); + } + + // Determine which search client is being used + // Distribution field is specific to OpenSearch + // Else, if number field is a valid string, default to 'elasticSearch' as client type + const { distribution, number } = response.version; + const version = + typeof distribution === 'string' ? distribution : typeof number === 'string' ? 'elasticsearch' : undefined; + if (typeof version === 'string') { + return version; + } + return undefined; + } catch (error) { + console.error(error); + throw new Error('Error identifying Search Client version from server response'); + } +}; + +export default async function getSearchClient(config: SearchConfig) { + try { + const configClientType = !config.clientType ? await getClientVersion(config) : config.clientType; + const clientType = supportedClientValues.find((key) => typeof key === 'string' && key === configClientType); + if (!clientType) { + throw new Error('Error with Search Client configuration clientType value'); + } + const options = { ...config, clientType }; + return createSearchClient(options); + } catch (error) { + console.error(error); + throw new Error(`Error configuring Search Client`); + } +} diff --git a/modules/server/src/searchClient/types.ts b/modules/server/src/searchClient/types.ts new file mode 100644 index 000000000..f0777b09d --- /dev/null +++ b/modules/server/src/searchClient/types.ts @@ -0,0 +1,61 @@ +import type { Client as ElasticClient, ApiResponse } from '@elastic/elasticsearch'; +import type { Client as OpenSearchClient } from '@opensearch-project/opensearch'; + +import type { ESClientOptions } from './createElasticSearchClient.js'; +import type { OSClientOptions } from './createOpenSearchClient.js'; + +export type AllSupportedClients = ElasticClient | OpenSearchClient; +export type SupportedClients = { elasticsearch: ElasticClient; opensearch: OpenSearchClient }; +export type SupportedClientTypes = keyof SupportedClients; +export type SupportedClientOptions = ESClientOptions | OSClientOptions; + +export type SearchConfig = { + node: string; + clientType?: SupportedClientTypes | string; + auth?: { + password: string; + username: string; + }; +}; + +export type SearchConfigWithClient = Omit & { + clientType: SupportedClientTypes; +}; + +// Approximates > +type SearchClientResponseHandler = Promise>; + +export type SearchClient = { + indices: { + close: (input: any, options?: any) => SearchClientResponseHandler>; + create: (input: any, options?: any) => SearchClientResponseHandler>; + delete: (input: any, options?: any) => SearchClientResponseHandler>; + exists: (input: any, options?: any) => SearchClientResponseHandler; + getMapping: (input: any, options?: any) => SearchClientResponseHandler>; + putSettings: (input: any, options?: any) => SearchClientResponseHandler>; + putMapping: (input: any, options?: any) => SearchClientResponseHandler>; + open: (input: any, options?: any) => SearchClientResponseHandler>; + refresh: (input: any, options?: any) => SearchClientResponseHandler>; + }; + cat: { + aliases: (input: any, options?: any) => SearchClientResponseHandler>; + }; + bulk: (input: any, options?: any) => SearchClientResponseHandler>; + deleteByQuery: (input: any, options?: any) => SearchClientResponseHandler>; + index: (input: any, options?: any) => SearchClientResponseHandler>; + search: (input: any, options?: any) => SearchClientResponseHandler>; + update: (input: any, options?: any) => SearchClientResponseHandler>; + create: (input: any, options?: any) => SearchClientResponseHandler>; + delete: (input: any, options?: any) => SearchClientResponseHandler>; +}; + +// Todo: Expected return Type for .search +export interface SearchResponse extends Record { + body: { + hits: { + hits: { + _source: any; + }[]; + }; + }; +} diff --git a/modules/server/src/server.ts b/modules/server/src/server.ts index 131e49150..5801c79a8 100644 --- a/modules/server/src/server.ts +++ b/modules/server/src/server.ts @@ -1,4 +1,3 @@ -import { Client, type ClientOptions } from '@elastic/elasticsearch'; import { Router } from 'express'; import morgan from 'morgan'; @@ -6,43 +5,29 @@ import { ENABLE_LOGS, ES_ARRANGER_SET_INDEX, ENABLE_NETWORK_AGGREGATION } from ' import { ENV_CONFIG } from './config/index.js'; import downloadRoutes from './download/index.js'; import getGraphQLRoutes from './graphqlRoutes.js'; +import getSearchClient from './searchClient/index.js'; +import { type SearchConfig } from './searchClient/types.js'; import getDefaultServerSideFilter from './utils/getDefaultServerSideFilter.js'; -const { - CONFIG_FILES_PATH, - DEBUG_MODE, - ENABLE_ADMIN, - ES_HOST, - ES_USER, - ES_PASS, - ES_LOG, //TODO: ES doesn't include a logger anymore - PING_PATH, -} = ENV_CONFIG; +const { CONFIG_FILES_PATH, DEBUG_MODE, ENABLE_ADMIN, ES_HOST, ES_USER, ES_PASS, PING_PATH, SEARCH_CLIENT } = ENV_CONFIG; -export const buildEsClient = (esHost = '', esUser = '', esPass = '') => { - if (!esHost) { - console.error('no elasticsearch host was provided'); +export const createSearchConfig = (host = '', username = '', password = '', clientType = '') => { + if (!host) { + throw new Error('Search Client host URL was not provided'); } - - const esConfig: ClientOptions = { - node: esHost, + const auth = (username && password) ? { username, password } : undefined; + const searchConfig: SearchConfig = { + node: host, + auth, + clientType, }; - if (esUser) { - if (!esPass) { - console.error('ES user was defined, but password was not'); - } - esConfig['auth'] = { - username: esUser, - password: esPass, - }; - } - - return new Client(esConfig); + return searchConfig; }; -export const buildEsClientViaEnv = () => { - return buildEsClient(ES_HOST, ES_USER, ES_PASS); +export const buildSearchClient = async (host: string, user: string, password: string, clientType: string) => { + const config = createSearchConfig(host, user, password, clientType); + return await getSearchClient(config); }; const arrangerServer = async ({ @@ -58,8 +43,9 @@ const arrangerServer = async ({ graphqlOptions = {}, pingPath = PING_PATH, setsIndex = ES_ARRANGER_SET_INDEX, + searchClient = SEARCH_CLIENT, } = {}): Promise => { - const esClient = customEsClient || buildEsClient(esHost, esUser, esPass); + const esClient = customEsClient || (await buildSearchClient(esHost, esUser, esPass, searchClient)); const router = Router(); console.log('------------------------------------'); diff --git a/package-lock.json b/package-lock.json index 5d7515a49..796448ee9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -188,6 +188,7 @@ "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "apollo-server": "^3.10.3", "apollo-server-core": "^3.10.3", @@ -4423,6 +4424,50 @@ "node": ">=12.4.0" } }, + "node_modules/@opensearch-project/opensearch": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-3.5.1.tgz", + "integrity": "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA==", + "license": "Apache-2.0", + "dependencies": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "json11": "^2.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "engines": { + "node": ">=14", + "yarn": "^1.22.10" + } + }, + "node_modules/@opensearch-project/opensearch/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@opensearch-project/opensearch/node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@overture-stack/arranger-components": { "resolved": "modules/components", "link": true @@ -6876,6 +6921,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", + "license": "MIT" + }, "node_modules/axe-core": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz", @@ -17553,6 +17604,15 @@ "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", "dev": true }, + "node_modules/json11": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/json11/-/json11-2.0.2.tgz", + "integrity": "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==", + "license": "MIT", + "bin": { + "json11": "dist/cli.mjs" + } + }, "node_modules/json3": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz", @@ -32248,6 +32308,34 @@ "integrity": "sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==", "dev": true }, + "@opensearch-project/opensearch": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-3.5.1.tgz", + "integrity": "sha512-6bf+HcuERzAtHZxrm6phjref54ABse39BpkDie/YO3AUFMCBrb3SK5okKSdT5n3+nDRuEEQLhQCl0RQV3s1qpA==", + "requires": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "json11": "^2.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "dependencies": { + "debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "requires": { + "ms": "^2.1.3" + } + }, + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + } + } + }, "@overture-stack/arranger-components": { "version": "file:modules/components", "requires": { @@ -32360,6 +32448,7 @@ "@graphql-tools/merge": "^9.0.4", "@graphql-tools/schema": "^9.0.17", "@graphql-tools/utils": "^10.2.2", + "@opensearch-project/opensearch": "^3.5.1", "@overture-stack/sqon-builder": "^1.1.0", "@tsconfig/node22": "^22.0.0", "@types/convert-units": "^2.3.5", @@ -34254,6 +34343,11 @@ "possible-typed-array-names": "^1.0.0" } }, + "aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==" + }, "axe-core": { "version": "4.10.2", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.2.tgz", @@ -42119,6 +42213,11 @@ "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", "dev": true }, + "json11": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/json11/-/json11-2.0.2.tgz", + "integrity": "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==" + }, "json3": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz",