Traders endpoints improvements (#426)

* Update trader stats endpoint to use trader_fills index

* Update trader metrics endpoint to use trader_fills endpoint

* Update active trader metrics endpoint to use trader_fills index

* Migrate trader lookup endpoint to trader_fills endpoint

* Update relayer endpoint to pull trader count from new index

* Update relayer metrics to use trader_fills

* Update relayers endpoint to use trader_fills

* Update fill endpoints to return correct taker

* Migrate traders endpoint to trader_fills index

Author: cbovis

src/app/routes/v1/fills.js

@@ -152,6 +152,8 @@ const createRouter = () => {
             { path: 'assets.token', select: 'decimals name symbol type' },
             { path: 'fees.token', select: 'decimals name symbol type' },
             { path: 'affiliate', select: 'name imageUrl' },
+            { path: 'takerMetadata', select: 'isContract' },
+            { path: 'transaction', select: 'from' },
           ],
         })
       : null;

src/app/routes/v1/traders.js

@@ -4,29 +4,10 @@ const Router = require('koa-router');
 const { TIME_PERIOD } = require('../../../constants');
 const AddressMetadata = require('../../../model/address-metadata');
 const getDatesForTimePeriod = require('../../../util/get-dates-for-time-period');
-const getTradersWith24HourStats = require('../../../traders/get-traders-with-24-hour-stats');
 const getTradersWithStatsForDates = require('../../../traders/get-traders-with-stats-for-dates');
 const InvalidParameterError = require('../../errors/invalid-parameter-error');
 const middleware = require('../../middleware');
 
-const parseBooleanString = value => {
-  if (value === 'true') {
-    return true;
-  }
-
-  if (value === 'false') {
-    return false;
-  }
-
-  return undefined;
-};
-
-const VALID_SORT_VALUES = [
-  'fillVolume.maker',
-  'fillVolume.taker',
-  'fillVolume.total',
-];
-
 const createRouter = () => {
   const router = new Router();
 
@@ -39,7 +20,7 @@ const createRouter = () => {
     }),
     middleware.timePeriod('statsPeriod', TIME_PERIOD.DAY),
     async ({ pagination, params, request, response }, next) => {
-      const { excludeRelayers, sortBy, type } = request.query;
+      const { excludeRelayers, type } = request.query;
 
       if (type !== undefined && type !== 'maker' && type !== 'taker') {
         throw new InvalidParameterError(
@@ -59,32 +40,19 @@ const createRouter = () => {
         );
       }
 
-      if (sortBy !== undefined && !VALID_SORT_VALUES.includes(sortBy)) {
-        throw new InvalidParameterError(
-          `Must be one of: ${VALID_SORT_VALUES.join(', ')}`,
-          'Invalid query parameter: sortBy',
-        );
-      }
-
       const { limit, page } = pagination;
       const { statsPeriod } = params;
       const { dateFrom, dateTo } = getDatesForTimePeriod(statsPeriod);
-      const { traders, resultCount } =
-        statsPeriod === TIME_PERIOD.DAY
-          ? await getTradersWith24HourStats({
-              excludeRelayers: parseBooleanString(excludeRelayers),
-              page,
-              limit,
-              sortBy,
-              type,
-            })
-          : await getTradersWithStatsForDates(dateFrom, dateTo, {
-              excludeRelayers,
-              page,
-              limit,
-              sortBy,
-              type,
-            });
+      const { traders, resultCount } = await getTradersWithStatsForDates(
+        dateFrom,
+        dateTo,
+        {
+          excludeRelayers,
+          page,
+          limit,
+          type,
+        },
+      );
 
       const addresses = traders.map(trader => trader.address);
 

src/app/routes/v1/util/transform-fill.js

@@ -13,6 +13,9 @@ const transformFill = fill => {
   const assets = getAssetsForFill(fill);
   const fees = getFeesForFill(fill);
   const conversions = _.get(fill, `conversions.USD`);
+  const taker = _.get(fill, 'takerMetadata.isContract', false)
+    ? _.get(fill, 'transaction.from', fill.taker)
+    : fill.taker;
 
   const protocolFee =
     fill.protocolFee !== undefined
@@ -42,7 +45,7 @@ const transformFill = fill => {
     relayer: formatRelayer(fill.relayer),
     senderAddress: fill.senderAddress,
     status: formatFillStatus(fill.status),
-    takerAddress: fill.taker,
+    takerAddress: taker,
     transactionHash: fill.transactionHash,
     value: _.has(conversions, 'amount')
       ? {

src/app/routes/v1/util/transform-fills.js

@@ -11,6 +11,9 @@ const transformRelayer = relayer =>
 const transformFill = fill => {
   const assets = getAssetsForFill(fill);
   const conversions = _.get(fill, `conversions.USD`);
+  const taker = _.get(fill, 'takerMetadata.isContract', false)
+    ? _.get(fill, 'transaction.from', fill.taker)
+    : fill.taker;
 
   const protocolFee =
     fill.protocolFee !== undefined
@@ -30,7 +33,7 @@ const transformFill = fill => {
     protocolVersion: fill.protocolVersion,
     relayer: transformRelayer(fill.relayer),
     status: formatFillStatus(fill.status),
-    takerAddress: fill.taker,
+    takerAddress: taker,
     value: _.has(conversions, 'amount')
       ? {
           USD: _.get(conversions, 'amount'),

src/fills/search-fills.js

@@ -21,7 +21,8 @@ const searchFills = async (params, options) => {
     .populate([
       { path: 'relayer', select: 'imageUrl name slug' },
       { path: 'assets.token', select: 'decimals name symbol type imageUrl' },
-      { path: 'fees.token', select: 'decimals name symbol type' },
+      { path: 'transaction', select: 'from' },
+      { path: 'takerMetadata', select: 'isContract' },
     ])
     .sort({ date: -1 });
 

src/metrics/get-relayer-metrics.js

@@ -1,9 +1,72 @@
 const elasticsearch = require('../util/elasticsearch');
 const getDatesForMetrics = require('../util/get-dates-for-metrics');
 
-const getRelayerMetrics = async (relayerId, period, granularity) => {
-  const { dateFrom, dateTo } = getDatesForMetrics(period, granularity);
+const getQuery = (relayerId, dateFrom, dateTo) => {
+  return {
+    bool: {
+      filter: [
+        {
+          range: {
+            date: {
+              gte: dateFrom,
+              lte: dateTo,
+            },
+          },
+        },
+        relayerId !== null
+          ? {
+              term: {
+                relayerId,
+              },
+            }
+          : undefined,
+      ].filter(x => x !== undefined),
+      must_not: [
+        relayerId === null
+          ? {
+              exists: {
+                field: 'relayerId',
+              },
+            }
+          : undefined,
+      ].filter(x => x !== undefined),
+    },
+  };
+};
 
+const getTraderMetrics = async (relayerId, dateFrom, dateTo, granularity) => {
+  const results = await elasticsearch.getClient().search({
+    index: 'trader_fills',
+    body: {
+      aggs: {
+        metrics_by_date: {
+          date_histogram: {
+            field: 'date',
+            calendar_interval: granularity,
+            extended_bounds: {
+              min: dateFrom,
+              max: dateTo,
+            },
+          },
+          aggs: {
+            traderCount: {
+              cardinality: { field: 'address' },
+            },
+          },
+        },
+      },
+      size: 0,
+      query: getQuery(relayerId, dateFrom, dateTo),
+    },
+  });
+
+  return results.body.aggregations.metrics_by_date.buckets.map(x => ({
+    date: new Date(x.key_as_string),
+    traderCount: x.traderCount.value,
+  }));
+};
+
+const getBasicMetrics = async (relayerId, dateFrom, dateTo, granularity) => {
   const results = await elasticsearch.getClient().search({
     index: 'fills',
     body: {
@@ -27,43 +90,11 @@ const getRelayerMetrics = async (relayerId, period, granularity) => {
             tradeVolume: {
               sum: { field: 'tradeVolume' },
             },
-            traderCount: {
-              cardinality: { field: 'traders' },
-            },
           },
         },
       },
       size: 0,
-      query: {
-        bool: {
-          filter: [
-            {
-              range: {
-                date: {
-                  gte: dateFrom,
-                  lte: dateTo,
-                },
-              },
-            },
-            relayerId !== null
-              ? {
-                  term: {
-                    relayerId,
-                  },
-                }
-              : undefined,
-          ].filter(x => x !== undefined),
-          must_not: [
-            relayerId === null
-              ? {
-                  exists: {
-                    field: 'relayerId',
-                  },
-                }
-              : undefined,
-          ].filter(x => x !== undefined),
-        },
-      },
+      query: getQuery(relayerId, dateFrom, dateTo),
     },
   });
 
@@ -73,8 +104,29 @@ const getRelayerMetrics = async (relayerId, period, granularity) => {
     fillVolume: x.fillVolume.value,
     tradeCount: relayerId === null ? x.doc_count : x.tradeCount.value,
     tradeVolume: relayerId === null ? x.fillVolume.value : x.tradeVolume.value,
-    traderCount: x.traderCount.value,
   }));
 };
 
+const getRelayerMetrics = async (relayerId, period, granularity) => {
+  const { dateFrom, dateTo } = getDatesForMetrics(period, granularity);
+
+  const [basicMetrics, traderMetrics] = await Promise.all([
+    getBasicMetrics(relayerId, dateFrom, dateTo, granularity),
+    getTraderMetrics(relayerId, dateFrom, dateTo, granularity),
+  ]);
+
+  const combinedMetrics = basicMetrics.map(basicMetric => {
+    const traderMetric = traderMetrics.find(
+      tm => tm.date.toISOString() === basicMetric.date.toISOString(),
+    );
+
+    return {
+      ...basicMetric,
+      traderCount: traderMetric.traderCount,
+    };
+  });
+
+  return combinedMetrics;
+};
+
 module.exports = getRelayerMetrics;

src/model/address-metadata.js

@@ -3,6 +3,7 @@ const mongoose = require('mongoose');
 const schema = mongoose.Schema({
   address: String,
   imageUrl: String,
+  isContract: Boolean,
   name: String,
 });
 

src/model/address-metric.js

@@ -4,6 +4,8 @@ const { FILL_STATUS } = require('../constants');
 
 const { Schema } = mongoose;
 
+require('./transaction');
+
 const schema = Schema({
   affiliateAddress: String,
   assets: [
@@ -77,6 +79,20 @@ schema.virtual('affiliate', {
   justOne: true,
 });
 
+schema.virtual('takerMetadata', {
+  ref: 'AddressMetadata',
+  localField: 'taker',
+  foreignField: 'address',
+  justOne: true,
+});
+
+schema.virtual('transaction', {
+  ref: 'Transaction',
+  localField: 'transactionHash',
+  foreignField: 'hash',
+  justOne: true,
+});
+
 const Model = mongoose.model('Fill', schema);
 
 module.exports = Model;

src/model/fill.js

@@ -0,0 +1,25 @@
+const mongoose = require('mongoose');
+
+const { Schema } = mongoose;
+
+const schema = Schema({
+  affiliateAddress: String,
+  blockHash: String,
+  blockNumber: Number,
+  data: String,
+  date: Date,
+  from: String,
+  gasLimit: Number,
+  gasPrice: String,
+  gasUsed: Number,
+  hash: String,
+  index: Number,
+  nonce: String,
+  quoteDate: Date,
+  to: String,
+  value: String,
+});
+
+const Transaction = mongoose.model('Transaction', schema);
+
+module.exports = Transaction;