[SER-2755] Rate limiter for API

CHANGELOG.md

@@ -3,6 +3,7 @@ Fixes:
 - [core] fixes for changeOwner script
 - [core] Add null checking for user permission when opening the dashboard
 - [core] Preserve URL hash during oauth
+- [core] Rate limiting for api endpoints
 
 Enterprise Fixes:
 - [retention_segments] Adding null check for breakdown filtering
@@ -29,7 +30,7 @@ Dependencies:
 - Bump sass from 1.93.3 to 1.96.0
 - Bump sass-embedded from 1.93.3 to 1.96.0
 - Bump sharp from 0.34.4 to 0.34.5
-- Bump sharp from 0.34.4 to 0.34.5 
+- Bump sharp from 0.34.4 to 0.34.5
 - Bump swiper from 11.2.10 to 12.0.3
 - Bump terser from 5.44.0 to 5.44.1
 - Bump vite from 7.1.12 to 7.2.7

api/api.js

@@ -17,6 +17,7 @@ const pack = require('../package.json');
 const versionInfo = require('../frontend/express/version.info.js');
 const moment = require("moment");
 const tracker = require('./parts/mgmt/tracker.js');
+const { RateLimiterMemory } = require("rate-limiter-flexible");
 
 var t = ["countly:", "api"];
 common.processRequest = processRequest;
@@ -119,6 +120,8 @@ plugins.connectToAllDatabases().then(function() {
         api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000; includeSubDomains; preload\nAccess-Control-Allow-Origin:*",
         dashboard_rate_limit_window: 60,
         dashboard_rate_limit_requests: 500,
+        api_rate_limit_window: 0,
+        api_rate_limit_requests: 0,
         proxy_hostname: "",
         proxy_port: "",
         proxy_username: "",
@@ -375,6 +378,35 @@ plugins.connectToAllDatabases().then(function() {
         console.log("Starting worker", process.pid, "parent:", process.ppid);
         const taskManager = require('./utils/taskmanager.js');
 
+        const rateLimitWindow = parseInt(plugins.getConfig("security").api_rate_limit_window, 10) || 0;
+        const rateLimitRequests = parseInt(plugins.getConfig("security").api_rate_limit_requests, 10) || 0;
+        const rateLimiterInstance = new RateLimiterMemory({ points: rateLimitRequests, duration: rateLimitWindow });
+        const requiresRateLimiting = rateLimitWindow > 0 && rateLimitRequests > 0;
+        const omit = /^\/i(\/bulk)?(\?|$)/; // omit /i endpoint from rate limiting
+        /**
+         * Rate Limiting Middleware
+         * @param {Function} next - The next middleware function
+         * @returns {Function} - The wrapped middleware function with rate limiting
+         */
+        const rateLimit = (next) => {
+            if (!requiresRateLimiting) {
+                return next;
+            }
+            return (req, res) => {
+                if (omit.test(req.url)) {
+                    return next(req, res);
+                }
+                const ip = common.getIpAddress(req);
+                rateLimiterInstance
+                    .consume(ip)
+                    .then(() => next(req, res))
+                    .catch(() => {
+                        log.w(`Rate limit exceeded for IP: ${ip}`);
+                        common.returnMessage({ req, res, qstring: {} }, 429, "Too Many Requests");
+                    });
+            };
+        };
+
         common.cache = new CacheWorker();
         common.cache.start();
 
@@ -412,10 +444,10 @@ plugins.connectToAllDatabases().then(function() {
             if (common.config.api.ssl.ca) {
                 sslOptions.ca = fs.readFileSync(common.config.api.ssl.ca);
             }
-            server = https.createServer(sslOptions, handleRequest);
+            server = https.createServer(sslOptions, rateLimit(handleRequest));
         }
         else {
-            server = http.createServer(handleRequest);
+            server = http.createServer(rateLimit(handleRequest));
         }
 
         server.listen(serverOptions.port, serverOptions.host).timeout = common.config.api.timeout || 120000;

api/utils/common.js

@@ -1486,7 +1486,9 @@ var ipLogger = common.log('ip:api');
 common.getIpAddress = function(req) {
     var ipAddress = "";
     if (req) {
-        if (req.headers) {
+        // TODO: add config option to trust x-forwarded-for header
+        // or add a configuration option to set trusted proxies
+        if (req.headers && ("x-forwarded-for" in req.headers || "x-real-ip" in req.headers)) {
             ipAddress = req.headers['x-forwarded-for'] || req.headers['x-real-ip'] || "";
         }
         else if (req.connection && req.connection.remoteAddress) {

package.json

@@ -92,6 +92,7 @@
     "offline-geocoder": "git+https://github.com/Countly/offline-geocoder.git",
     "properties-parser": "0.6.0",
     "puppeteer": "^24.6.1",
+    "rate-limiter-flexible": "^9.0.1",
     "sass": "1.96.0",
     "semver": "^7.7.1",
     "sharp": "^0.34.2",

plugins/plugins/frontend/public/localization/plugins.properties

@@ -127,6 +127,8 @@ configs.user-level-configuration = User Level Configuration
 configs.table-description = Settings in this section will override global settings for the user
 configs.security-dashboard_rate_limit_window = Dashboard Rate Limit Time (seconds)
 configs.security-dashboard_rate_limit_requests = Dashboard Request Rate Limit
+configs.security-api_rate_limit_window = API Rate Limit Time (seconds)
+configs.security-api_rate_limit_requests = API Request Rate Limit
 configs.danger-zone = Danger Zone
 configs.password = Password
 configs.fill-required-fields = Please fill all required fields
@@ -234,6 +236,8 @@ configs.help.security-password_expiration = Number of days after which user must
 configs.help.user-level-configuration = Allow separate dashboard users to change these configs for their account only.
 configs.help.security-dashboard_rate_limit_window = Will start blocking if request amount is reached in this time window
 configs.help.security-dashboard_rate_limit_requests = How many requests to allow per time window?
+configs.help.security-api_rate_limit_window = Will start blocking if request amount is reached in this time window. Requires a server restart.
+configs.help.security-api_rate_limit_requests = How many requests to allow per time window? Requires a server restart.
 configs.help.push-proxyhost = Hostname or IP address of HTTP CONNECT proxy server to use for communication with APN & FCM when sending push notifications.
 configs.help.push-proxyport = Port number of the proxy server
 configs.help.push-proxyuser = (if needed) Username for proxy server HTTP Basic authentication