diff --git a/.env.example b/.env.example index 1972614..0ba3fb1 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,12 @@ JWT_ACCESS_TOKEN_TTL_SECONDS=900 REDIS_URL=redis://localhost:6379 ENABLE_REDIS_CACHE=true +# Query cost governor (#755): rolling per-wallet/per-IP database query budget +QUERY_COST_BUDGET=200 +QUERY_COST_WINDOW_MS=60000 +# QUERY_COST_MAP_JSON={"GET /search": 8} +# QUERY_COST_ADMIN_WALLETS= + # Key trade lockup (seconds between a buy and when the keys unlock for sale) LOCKUP_DURATION_SECONDS=0 diff --git a/prisma/schema/migrations/20260828000000_add_key_ownership_last_buy_at/migration.sql b/prisma/schema/migrations/20260828000000_add_key_ownership_last_buy_at/migration.sql new file mode 100644 index 0000000..87a2d89 --- /dev/null +++ b/prisma/schema/migrations/20260828000000_add_key_ownership_last_buy_at/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "KeyOwnership" ADD COLUMN "lastBuyAt" TIMESTAMP(3); diff --git a/src/config.schema.ts b/src/config.schema.ts index d474eb3..6a0f507 100644 --- a/src/config.schema.ts +++ b/src/config.schema.ts @@ -228,6 +228,24 @@ export const envSchema = z // Left unset by default, so no caller is trusted unless configured. TRACE_ID_TRUSTED_TOKEN: optionalNonEmptyString, INTERNAL_SERVICE_KEY: optionalNonEmptyString, + + // Query cost governor (#755): rolling per-wallet (or per-IP, when + // unauthenticated) database query budget. See + // src/middlewares/query-cost-governor.middleware.ts. + QUERY_COST_BUDGET: z.coerce.number().int().positive().default(200), + QUERY_COST_WINDOW_MS: z.coerce + .number() + .int() + .positive() + .default(60_000), + // JSON object overriding/extending the default route->cost map in + // src/constants/query-cost.constants.ts, e.g. + // '{"GET /search": 8, "GET /custom-route": 2}'. Merged over the + // defaults, not a full replacement, so operators only need to + // specify what differs. + QUERY_COST_MAP_JSON: optionalNonEmptyString, + // Comma-separated wallet addresses that bypass the governor entirely. + QUERY_COST_ADMIN_WALLETS: optionalNonEmptyString, HORIZON_WEBHOOK_SECRET: optionalNonEmptyString, WEBHOOK_RETRY_BASE_DELAY_MS: z.coerce .number() @@ -247,7 +265,9 @@ export const envSchema = z .default(5000), SSE_REPLAY_MAX_EVENTS: z.coerce.number().int().positive().default(100), - // Server-Sent Events (SSE) subscription limits + // SSE subscription management (src/modules/subscriptions) — a wallet's + // subscription set, persisted in Redis, distinct from the per-connection + // heartbeat/queue/replay tuning above. SSE_MAX_CONNECTIONS_PER_WALLET: z.coerce .number() .int() @@ -268,9 +288,6 @@ export const envSchema = z .int() .positive() .default(1000), - - // Stellar auth challenge signing secret - STELLAR_AUTH_SECRET: optionalNonEmptyString, }) .superRefine((data, ctx) => { if (data.MODE === 'production' && data.STELLAR_NETWORK === 'testnet') { diff --git a/src/constants/error.constants.ts b/src/constants/error.constants.ts index c207198..d285028 100644 --- a/src/constants/error.constants.ts +++ b/src/constants/error.constants.ts @@ -9,6 +9,7 @@ export const ErrorCode = { FORBIDDEN: 'FORBIDDEN', CONFLICT: 'CONFLICT', BAD_REQUEST: 'BAD_REQUEST', + UNPROCESSABLE_ENTITY: 'UNPROCESSABLE_ENTITY', INTERNAL_ERROR: 'INTERNAL_ERROR', RATE_LIMIT: 'RATE_LIMIT', PRISMA_ERROR: 'DATABASE_ERROR', diff --git a/src/constants/query-cost.constants.ts b/src/constants/query-cost.constants.ts new file mode 100644 index 0000000..19deec2 --- /dev/null +++ b/src/constants/query-cost.constants.ts @@ -0,0 +1,29 @@ +// src/constants/query-cost.constants.ts +// Default route->cost map for the query cost governor (#755). +// +// Keys are `${METHOD} ${pattern}`, where `pattern` uses Express-style +// `:param` segments (matched via src/utils/query-cost.utils.ts, not +// req.route — the governor runs as router-level middleware, before Express +// resolves the specific route, so req.route isn't populated yet). +// +// The issue's own example routes (`GET /search`, `GET /creators/:id/history`) +// don't exist verbatim in this codebase; substituted for the closest real +// equivalents (`GET /keys/search`, `GET /creators/:id/stats`) — see the PR +// description for the full mapping rationale. +export const DEFAULT_QUERY_COST_MAP: Record = { + 'GET /creators': 1, + 'GET /creators/:id/holders': 3, + 'GET /creators/:id/stats': 2, + 'GET /keys/search': 5, +}; + +/** Cost applied to any authenticated route with no explicit entry in the map. */ +export const DEFAULT_QUERY_COST = 1; + +/** + * Route patterns exempt from query cost governance entirely: health checks + * (must stay reachable regardless of load) and the governor's own internal + * management routes (resetting a wallet's budget must never itself be + * throttled by that same budget). + */ +export const QUERY_COST_EXEMPT_PATH_PREFIXES = ['/health', '/internal/qcost']; diff --git a/src/middlewares/jwt.middleware.ts b/src/middlewares/jwt.middleware.ts index d0308f7..e7a70b9 100644 --- a/src/middlewares/jwt.middleware.ts +++ b/src/middlewares/jwt.middleware.ts @@ -1,5 +1,5 @@ import { Request, Response, NextFunction } from 'express'; -import jwt, { SignOptions } from 'jsonwebtoken'; +import jwt from 'jsonwebtoken'; import { envConfig } from '../config'; import { sendUnauthorized } from '../utils/api-response.utils'; import { logger } from '../utils/logger.utils'; @@ -69,10 +69,3 @@ export function jwtAuth(req: Request, res: Response, next: NextFunction): void { sendUnauthorized(res, 'Invalid or expired token'); } } - -export function signJwt(payload: JwtPayload): string { - const options: SignOptions = { - expiresIn: envConfig.JWT_EXPIRES_IN as any, - }; - return jwt.sign(payload, envConfig.JWT_SECRET, options); -} diff --git a/src/middlewares/query-cost-governor.middleware.test.ts b/src/middlewares/query-cost-governor.middleware.test.ts new file mode 100644 index 0000000..3baf113 --- /dev/null +++ b/src/middlewares/query-cost-governor.middleware.test.ts @@ -0,0 +1,285 @@ +// Unit tests for the adaptive query cost governor (#755). + +const mockEnvConfig: { + INTERNAL_SERVICE_KEY?: string; + QUERY_COST_BUDGET: number; + QUERY_COST_WINDOW_MS: number; + QUERY_COST_MAP_JSON?: string; + QUERY_COST_ADMIN_WALLETS?: string; +} = { + INTERNAL_SERVICE_KEY: undefined, + QUERY_COST_BUDGET: 200, + QUERY_COST_WINDOW_MS: 60_000, + QUERY_COST_MAP_JSON: undefined, + QUERY_COST_ADMIN_WALLETS: undefined, +}; + +jest.mock('../config', () => ({ + envConfig: mockEnvConfig, +})); + +jest.mock('../utils/logger.utils', () => ({ + logger: { warn: jest.fn(), error: jest.fn(), debug: jest.fn(), info: jest.fn() }, +})); + +function buildFakeRedis() { + const store = new Map>(); + + return { + zremrangebyscore: jest.fn(async (key: string, _min: number, max: number) => { + const entries = store.get(key) ?? []; + store.set( + key, + entries.filter(entry => entry.score > max) + ); + }), + zrange: jest.fn(async (key: string, _start: number, _stop: number, withScores?: string) => { + const entries = (store.get(key) ?? []).sort((a, b) => a.score - b.score); + if (withScores === 'WITHSCORES') { + const first = entries[0]; + return first ? [first.member, String(first.score)] : []; + } + return entries.map(entry => entry.member); + }), + zadd: jest.fn(async (key: string, score: number, member: string) => { + const entries = store.get(key) ?? []; + entries.push({ score, member }); + store.set(key, entries); + }), + pexpire: jest.fn(async () => 1), + del: jest.fn(async (key: string) => { + store.delete(key); + }), + __store: store, + }; +} + +jest.mock('../utils/redis.utils', () => ({ + getRedis: jest.fn(), +})); + +const mockVerifyWalletAccessToken = jest.fn(); +jest.mock('../utils/jwt.utils', () => ({ + extractBearerToken: (header: unknown) => + typeof header === 'string' && header.startsWith('Bearer ') + ? header.slice(7) + : undefined, + verifyWalletAccessToken: (token: string) => mockVerifyWalletAccessToken(token), +})); + +import { getRedis } from '../utils/redis.utils'; +import { queryCostGovernor } from './query-cost-governor.middleware'; + +const mockGetRedis = getRedis as jest.Mock; + +function makeReq(overrides: Partial> = {}): any { + return { + method: 'GET', + path: '/creators', + query: {}, + headers: {}, + ip: '203.0.113.5', + ...overrides, + }; +} + +function makeRes(): any { + const res: any = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + res.set = jest.fn().mockReturnValue(res); + return res; +} + +describe('queryCostGovernor', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockEnvConfig.INTERNAL_SERVICE_KEY = undefined; + mockEnvConfig.QUERY_COST_BUDGET = 200; + mockEnvConfig.QUERY_COST_WINDOW_MS = 60_000; + mockEnvConfig.QUERY_COST_MAP_JSON = undefined; + mockEnvConfig.QUERY_COST_ADMIN_WALLETS = undefined; + mockVerifyWalletAccessToken.mockReset(); + }); + + it('admits requests summing to exactly the budget, keyed per IP when unauthenticated', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 10; + const governor = queryCostGovernor(); + + // 10 requests at cost 1 (GET /creators) = exactly the budget. + for (let i = 0; i < 10; i++) { + const req = makeReq(); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + } + }); + + it('throttles the request that pushes total cost over budget, with Retry-After and reset headers', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 5; + const governor = queryCostGovernor(); + + // GET /creators/:id/holders costs 3 by default. + for (let i = 0; i < 1; i++) { + const req = makeReq({ path: '/creators/abc/holders' }); + const res = makeRes(); + await governor(req, res, jest.fn()); + } + + const req = makeReq({ path: '/creators/abc/holders' }); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(429); + expect(res.set).toHaveBeenCalledWith('Retry-After', expect.any(String)); + expect(res.set).toHaveBeenCalledWith( + 'X-Query-Budget-Reset', + expect.any(String) + ); + const body = res.json.mock.calls[0][0]; + expect(body.type).toBe('query_budget_exceeded'); + }); + + it('sets X-Query-Cost and X-Query-Budget-Remaining on successful responses', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + const governor = queryCostGovernor(); + + const req = makeReq({ path: '/creators/abc/holders' }); + const res = makeRes(); + await governor(req, res, jest.fn()); + + expect(res.set).toHaveBeenCalledWith('X-Query-Cost', '3'); + expect(res.set).toHaveBeenCalledWith( + 'X-Query-Budget-Remaining', + String(mockEnvConfig.QUERY_COST_BUDGET - 3) + ); + }); + + it('multiplies cost by the limit query param', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + const governor = queryCostGovernor(); + + const req = makeReq({ path: '/creators/abc/holders', query: { limit: '10' } }); + const res = makeRes(); + await governor(req, res, jest.fn()); + + expect(res.set).toHaveBeenCalledWith('X-Query-Cost', '30'); + }); + + it('allows requests again after the rolling window expires', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 1; + mockEnvConfig.QUERY_COST_WINDOW_MS = 50; + const governor = queryCostGovernor(); + + const first = makeReq(); + await governor(first, makeRes(), jest.fn()); + + const blocked = makeReq(); + const blockedRes = makeRes(); + const blockedNext = jest.fn(); + await governor(blocked, blockedRes, blockedNext); + expect(blockedNext).not.toHaveBeenCalled(); + + await new Promise(resolve => setTimeout(resolve, 60)); + + const afterWindow = makeReq(); + const afterRes = makeRes(); + const afterNext = jest.fn(); + await governor(afterWindow, afterRes, afterNext); + expect(afterNext).toHaveBeenCalledTimes(1); + }); + + it('bypasses the governor entirely for admin wallets', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 1; + mockEnvConfig.QUERY_COST_ADMIN_WALLETS = 'GADMIN123, GOTHER456'; + mockVerifyWalletAccessToken.mockReturnValue({ wallet: 'GADMIN123' }); + const governor = queryCostGovernor(); + + for (let i = 0; i < 5; i++) { + const req = makeReq({ + path: '/creators/abc/holders', + headers: { authorization: 'Bearer admin-token' }, + }); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + } + }); + + it('bypasses the governor for internal service calls', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 0; + mockEnvConfig.INTERNAL_SERVICE_KEY = 'internal-secret'; + const governor = queryCostGovernor(); + + const req = makeReq({ headers: { 'x-internal-service-key': 'internal-secret' } }); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it('exempts /health and its own /internal/qcost routes', async () => { + const redis = buildFakeRedis(); + mockGetRedis.mockReturnValue(redis); + mockEnvConfig.QUERY_COST_BUDGET = 0; + const governor = queryCostGovernor(); + + for (const path of ['/health', '/internal/qcost/reset/GABC']) { + const req = makeReq({ path, method: 'POST' }); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + } + }); + + it('fails open when Redis is unavailable', async () => { + mockGetRedis.mockReturnValue(null); + mockEnvConfig.QUERY_COST_BUDGET = 0; + const governor = queryCostGovernor(); + + const req = makeReq(); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); + + it('fails open when a Redis command throws', async () => { + const redis = buildFakeRedis(); + redis.zremrangebyscore.mockRejectedValueOnce(new Error('connection reset')); + mockGetRedis.mockReturnValue(redis); + const governor = queryCostGovernor(); + + const req = makeReq(); + const res = makeRes(); + const next = jest.fn(); + await governor(req, res, next); + + expect(next).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + }); +}); diff --git a/src/middlewares/query-cost-governor.middleware.ts b/src/middlewares/query-cost-governor.middleware.ts new file mode 100644 index 0000000..0cec4eb --- /dev/null +++ b/src/middlewares/query-cost-governor.middleware.ts @@ -0,0 +1,223 @@ +// src/middlewares/query-cost-governor.middleware.ts +// Adaptive per-wallet (or per-IP, when unauthenticated) database query cost +// governor (#755). +// +// A single wallet firing expensive paginated/search/analytics queries can +// saturate the connection pool for everyone. This assigns a cost unit to +// every request (see src/utils/query-cost.utils.ts), tracks a rolling sum +// of costs per caller in a Redis sorted set, and rejects requests that would +// push the caller over budget with 429 query_budget_exceeded. +// +// Identity: most of the routes this is meant to protect (creator list, +// holders, search) are public reads with no wallet-auth middleware in front +// of them today, so "per-wallet" only applies when the caller sent a valid +// JWT (the same one requireJwtAuth checks) — decoded here without rejecting +// when absent/invalid, unlike requireJwtAuth. Anonymous callers still get a +// real budget, keyed on IP, so the governor actually protects the public +// routes named in the issue rather than only the already-authenticated ones. +// +// Concurrency note: like wallet-rate-limit.middleware.ts, this evicts/reads +// then conditionally writes in separate round trips rather than a single +// atomic Lua script — under a burst of truly concurrent requests from the +// same caller there's a small window where more than the budget could be +// admitted. Matches this codebase's existing accepted tradeoff for Redis +// rate limiting rather than introducing a different rigor level for this +// one feature. + +import type { Request, Response, NextFunction } from 'express'; +import { randomUUID } from 'crypto'; +import { getRedis } from '../utils/redis.utils'; +import { envConfig } from '../config'; +import { logger } from '../utils/logger.utils'; +import { extractBearerToken, verifyWalletAccessToken } from '../utils/jwt.utils'; +import { + buildQueryCostRedisKey, + compileCostMap, + computeQueryCost, + matchCostRoute, + type CompiledCostRoute, +} from '../utils/query-cost.utils'; +import { QUERY_COST_EXEMPT_PATH_PREFIXES } from '../constants/query-cost.constants'; + +const INTERNAL_SERVICE_HEADER = 'x-internal-service-key'; + +export interface QueryCostRequest extends Request { + walletAddress?: string; + queryCost?: { cost: number; identity: string }; +} + +function isInternalServiceCall(req: Request): boolean { + if (!envConfig.INTERNAL_SERVICE_KEY) return false; + const provided = req.headers[INTERNAL_SERVICE_HEADER]; + const value = Array.isArray(provided) ? provided[0] : provided; + return value === envConfig.INTERNAL_SERVICE_KEY; +} + +function isExemptPath(path: string): boolean { + return QUERY_COST_EXEMPT_PATH_PREFIXES.some(prefix => + path.startsWith(prefix) + ); +} + +/** Best-effort wallet resolution: never rejects on a missing/invalid token. */ +function resolveWalletAddress(req: Request): string | undefined { + const token = extractBearerToken(req.headers.authorization); + if (!token) return undefined; + try { + return verifyWalletAccessToken(token).wallet; + } catch { + return undefined; + } +} + +function parseAdminWallets(raw: string | undefined): Set { + if (!raw) return new Set(); + return new Set( + raw + .split(',') + .map(wallet => wallet.trim().toLowerCase()) + .filter(Boolean) + ); +} + +/** Encodes a sorted-set member as ":" so cost survives eviction/summation without a second data structure. */ +function encodeMember(cost: number): string { + return `${cost}:${randomUUID()}`; +} + +function decodeCost(member: string): number { + const separator = member.indexOf(':'); + const cost = Number.parseInt( + separator === -1 ? member : member.slice(0, separator), + 10 + ); + return Number.isFinite(cost) ? cost : 0; +} + +export interface QueryCostGovernorOptions { + /** Rolling window in milliseconds. Defaults to envConfig.QUERY_COST_WINDOW_MS. */ + windowMs?: number; + /** Budget per window. Defaults to envConfig.QUERY_COST_BUDGET. */ + budget?: number; +} + +export function queryCostGovernor(options: QueryCostGovernorOptions = {}) { + const windowMs = options.windowMs ?? envConfig.QUERY_COST_WINDOW_MS; + const budget = options.budget ?? envConfig.QUERY_COST_BUDGET; + const adminWallets = parseAdminWallets(envConfig.QUERY_COST_ADMIN_WALLETS); + + let compiledRoutes: CompiledCostRoute[]; + try { + compiledRoutes = compileCostMap(envConfig.QUERY_COST_MAP_JSON); + } catch (error) { + logger.error( + { type: 'query_cost_config_invalid', error }, + 'Invalid QUERY_COST_MAP_JSON; falling back to defaults' + ); + compiledRoutes = compileCostMap(undefined); + } + + return async ( + req: QueryCostRequest, + res: Response, + next: NextFunction + ): Promise => { + if (isExemptPath(req.path) || isInternalServiceCall(req)) { + next(); + return; + } + + const walletAddress = resolveWalletAddress(req); + req.walletAddress = walletAddress; + + if (walletAddress && adminWallets.has(walletAddress.toLowerCase())) { + next(); + return; + } + + const identity = walletAddress + ? `wallet:${walletAddress}` + : `ip:${req.ip ?? 'unknown'}`; + + const matched = matchCostRoute(compiledRoutes, req.method, req.path); + const cost = computeQueryCost(matched, req.query.limit); + + const redis = getRedis(); + if (!redis) { + // Fail open, same as wallet-rate-limit.middleware.ts: caching/rate + // infra being down must never take the API down. + req.queryCost = { cost, identity }; + res.set('X-Query-Cost', String(cost)); + next(); + return; + } + + const key = buildQueryCostRedisKey(identity); + const now = Date.now(); + const windowStart = now - windowMs; + + try { + await redis.zremrangebyscore(key, 0, windowStart); + const members = await redis.zrange(key, 0, '-1'); + const spent = members.reduce( + (sum, member) => sum + decodeCost(member), + 0 + ); + + if (spent + cost > budget) { + const oldest = await redis.zrange(key, 0, '0', 'WITHSCORES'); + const oldestTimestamp = oldest[1] ? Number(oldest[1]) : now; + const resetAtMs = oldestTimestamp + windowMs; + const retryAfterSeconds = Math.max( + 1, + Math.ceil((resetAtMs - now) / 1000) + ); + + logger.warn( + { + type: 'query_budget_exceeded', + identity, + route: req.path, + method: req.method, + cost, + spent, + budget, + }, + 'Query budget exceeded' + ); + + res + .status(429) + .set('Retry-After', String(retryAfterSeconds)) + .set('X-Query-Cost', String(cost)) + .set('X-Query-Budget-Remaining', String(Math.max(0, budget - spent))) + .set('X-Query-Budget-Reset', String(Math.floor(resetAtMs / 1000))) + .json({ + type: 'query_budget_exceeded', + message: 'Query budget exceeded for this window.', + retryAfterSeconds, + timestamp: new Date().toISOString(), + }); + return; + } + + await redis.zadd(key, now, encodeMember(cost)); + await redis.pexpire(key, windowMs); + + req.queryCost = { cost, identity }; + res.set('X-Query-Cost', String(cost)); + res.set( + 'X-Query-Budget-Remaining', + String(Math.max(0, budget - spent - cost)) + ); + next(); + } catch (error) { + logger.error( + { error, identity, route: req.path }, + 'Query cost governor check failed; allowing request through (fail open)' + ); + res.set('X-Query-Cost', String(cost)); + next(); + } + }; +} diff --git a/src/modules/admin/audit-log-endpoint.integration.test.ts b/src/modules/admin/audit-log-endpoint.integration.test.ts index 17c325a..8a89f34 100644 --- a/src/modules/admin/audit-log-endpoint.integration.test.ts +++ b/src/modules/admin/audit-log-endpoint.integration.test.ts @@ -1,15 +1,12 @@ import request from 'supertest'; -import { createServer } from '../../utils/server.utils'; +import app from '../../app'; import { prisma } from '../../utils/prisma.utils'; import { signWalletAccessToken } from '../../utils/jwt.utils'; describe('GET /admin/audit-log Endpoint Integration Tests', () => { - let app: any; let adminToken: string; beforeAll(async () => { - app = await createServer(); - // Create admin token const adminWallet = '0xadmintestwallet1111111111111111111111111'; adminToken = signWalletAccessToken(adminWallet, 'admin-sub', 3600); diff --git a/src/modules/admin/query-cost.controllers.test.ts b/src/modules/admin/query-cost.controllers.test.ts new file mode 100644 index 0000000..35bcef7 --- /dev/null +++ b/src/modules/admin/query-cost.controllers.test.ts @@ -0,0 +1,54 @@ +import { httpResetQueryCost } from './query-cost.controllers'; + +const mockDel = jest.fn(); + +jest.mock('../../utils/redis.utils', () => ({ + getRedis: jest.fn(() => ({ del: mockDel })), +})); + +jest.mock('../../utils/logger.utils', () => ({ + logger: { warn: jest.fn(), error: jest.fn(), debug: jest.fn(), info: jest.fn() }, +})); + +describe('httpResetQueryCost', () => { + const next = jest.fn(); + + function createRes(): any { + const res: any = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res; + } + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('clears the wallet-scoped Redis key and returns success', async () => { + const req: any = { params: { walletAddress: 'GABC123' } }; + const res = createRes(); + + await httpResetQueryCost(req, res, next); + + expect(mockDel).toHaveBeenCalledWith('qcost:wallet:GABC123'); + expect(res.json).toHaveBeenCalledWith( + expect.objectContaining({ + success: true, + data: expect.objectContaining({ + walletAddress: 'GABC123', + status: 'reset', + }), + }) + ); + }); + + it('rejects a missing walletAddress param', async () => { + const req: any = { params: {} }; + const res = createRes(); + + await httpResetQueryCost(req, res, next); + + expect(mockDel).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + }); +}); diff --git a/src/modules/admin/query-cost.controllers.ts b/src/modules/admin/query-cost.controllers.ts new file mode 100644 index 0000000..bb0eb53 --- /dev/null +++ b/src/modules/admin/query-cost.controllers.ts @@ -0,0 +1,41 @@ +import { AsyncController } from '../../types/auth.types'; +import { getRedis } from '../../utils/redis.utils'; +import { buildQueryCostRedisKey } from '../../utils/query-cost.utils'; +import { sendSuccess, sendValidationError } from '../../utils/api-response.utils'; +import { logger } from '../../utils/logger.utils'; + +/** + * POST /internal/qcost/reset/:walletAddress + * + * Clears a wallet's rolling query-cost budget immediately. Internal-network + * route (see src/modules/index.ts's mount and README) — same convention as + * the existing /internal/sequencer/clear-drift/:creatorWallet. + */ +export const httpResetQueryCost: AsyncController = async (req, res, next) => { + try { + const rawParam = req.params.walletAddress; + const walletAddress = Array.isArray(rawParam) ? rawParam[0] : rawParam; + if (!walletAddress) { + sendValidationError(res, 'Missing walletAddress parameter'); + return; + } + + const redis = getRedis(); + if (redis) { + await redis.del(buildQueryCostRedisKey(`wallet:${walletAddress}`)); + } + + logger.warn( + { type: 'query_cost_reset', walletAddress }, + 'Query cost budget reset by operator' + ); + + sendSuccess(res, { + walletAddress, + status: 'reset', + message: `Query cost budget cleared for ${walletAddress}`, + }); + } catch (err) { + next(err); + } +}; diff --git a/src/modules/admin/sequencer.routes.ts b/src/modules/admin/sequencer.routes.ts index fa6c605..81d2530 100644 --- a/src/modules/admin/sequencer.routes.ts +++ b/src/modules/admin/sequencer.routes.ts @@ -1,8 +1,11 @@ import { Router } from 'express'; import { httpClearDrift } from './sequencer.controllers'; +import { httpResetQueryCost } from './query-cost.controllers'; const sequencerRouter = Router(); sequencerRouter.post('/sequencer/clear-drift/:creatorWallet', httpClearDrift); +// Query cost governor admin override (#755). +sequencerRouter.post('/qcost/reset/:walletAddress', httpResetQueryCost); export default sequencerRouter; diff --git a/src/modules/creator/creator.routes.ts b/src/modules/creator/creator.routes.ts index dc290fa..70a2a93 100644 --- a/src/modules/creator/creator.routes.ts +++ b/src/modules/creator/creator.routes.ts @@ -76,7 +76,9 @@ creatorsRouter.post( return; } - const keyId = req.params.keyId; + const keyId = Array.isArray(req.params.keyId) + ? req.params.keyId[0] + : req.params.keyId; try { const creatorProfile = await prisma.creatorProfile.findFirst({ where: { OR: [{ id: keyId }, { handle: keyId }] }, diff --git a/src/modules/index.ts b/src/modules/index.ts index 6678b70..ed9ebae 100644 --- a/src/modules/index.ts +++ b/src/modules/index.ts @@ -1,4 +1,5 @@ import { routeBodySizeLimit } from '../middlewares/body-size-limit.middleware'; +import { queryCostGovernor } from '../middlewares/query-cost-governor.middleware'; import { Router } from 'express'; import authRouter from './auth/auth.routes'; import healthRouter from './health/health.routes'; @@ -26,6 +27,14 @@ import { BASE as CREATORS_BASE } from '../constants/creator.constants'; const router = Router(); +// Adaptive per-wallet/per-IP database query cost governor (#755). Mounted +// ahead of route resolution (so it matches on req.path, not req.route — see +// query-cost.utils.ts) and ahead of every group below, so it covers the +// whole API surface rather than needing to be threaded into each route +// individually. Exempts /health and its own /internal/qcost management +// routes (see QUERY_COST_EXEMPT_PATH_PREFIXES). +router.use(queryCostGovernor()); + // Each group gets its own JSON body parser so its size limit can be tuned // independently via BODY_SIZE_LIMIT_ env vars (see // docs/body-size-limits.md). Groups without a dedicated override share diff --git a/src/modules/wallets/wallet-following.integration.test.ts b/src/modules/wallets/wallet-following.integration.test.ts index 3a2d2ba..da57d14 100644 --- a/src/modules/wallets/wallet-following.integration.test.ts +++ b/src/modules/wallets/wallet-following.integration.test.ts @@ -9,7 +9,7 @@ import supertest from 'supertest'; import { Keypair } from '@stellar/stellar-base'; import app from '../../app'; import { prisma } from '../../utils/prisma.utils'; -import { signJwt } from '../../middlewares/jwt.middleware'; +import { signWalletAccessToken } from '../../utils/jwt.utils'; describe('GET /api/v1/wallets/:address/following', () => { const PREFIX = 'wallet-following-test'; @@ -166,10 +166,7 @@ describe('GET /api/v1/wallets/:address/following', () => { // ── Alphabetical ordering ──────────────────────────────────────────────── it('returns creators in alphabetical order by display name', async () => { - const token = signJwt({ - walletAddress: walletA.publicKey(), - sub: userIdWalletA, - }); + const token = signWalletAccessToken(walletA.publicKey(), userIdWalletA); const res = await supertest(app) .get(`/api/v1/wallets/${walletA.publicKey()}/following`) @@ -185,10 +182,7 @@ describe('GET /api/v1/wallets/:address/following', () => { // ── Completeness ───────────────────────────────────────────────────────── it('returns all followed creators', async () => { - const token = signJwt({ - walletAddress: walletA.publicKey(), - sub: userIdWalletA, - }); + const token = signWalletAccessToken(walletA.publicKey(), userIdWalletA); const res = await supertest(app) .get(`/api/v1/wallets/${walletA.publicKey()}/following`) @@ -206,10 +200,7 @@ describe('GET /api/v1/wallets/:address/following', () => { // ── Empty array for wallet with no follows ─────────────────────────────── it('returns an empty array for a wallet that follows no one', async () => { - const token = signJwt({ - walletAddress: walletB.publicKey(), - sub: userIdWalletB, - }); + const token = signWalletAccessToken(walletB.publicKey(), userIdWalletB); const res = await supertest(app) .get(`/api/v1/wallets/${walletB.publicKey()}/following`) diff --git a/src/utils/query-cost.utils.test.ts b/src/utils/query-cost.utils.test.ts new file mode 100644 index 0000000..be60e91 --- /dev/null +++ b/src/utils/query-cost.utils.test.ts @@ -0,0 +1,83 @@ +import { + compileCostMap, + computeQueryCost, + matchCostRoute, +} from './query-cost.utils'; + +describe('compileCostMap', () => { + it('compiles the default map and matches param segments', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc-123/holders'); + expect(matched?.baseCost).toBe(3); + }); + + it('does not match a different method for the same path', () => { + const routes = compileCostMap(); + expect(matchCostRoute(routes, 'POST', '/creators')).toBeNull(); + }); + + it('merges a JSON override over the defaults without dropping unrelated entries', () => { + const routes = compileCostMap('{"GET /custom": 9}'); + expect(matchCostRoute(routes, 'GET', '/custom')?.baseCost).toBe(9); + expect(matchCostRoute(routes, 'GET', '/creators')?.baseCost).toBe(1); + }); + + it('lets a JSON override replace a default entry', () => { + const routes = compileCostMap('{"GET /creators": 4}'); + expect(matchCostRoute(routes, 'GET', '/creators')?.baseCost).toBe(4); + }); + + it('rejects malformed JSON', () => { + expect(() => compileCostMap('{not json')).toThrow(/valid JSON/); + }); + + it('rejects a non-object JSON value', () => { + expect(() => compileCostMap('[1,2,3]')).toThrow(/JSON object/); + }); + + it('rejects a non-positive-number cost', () => { + expect(() => compileCostMap('{"GET /x": -1}')).toThrow(/positive number/); + expect(() => compileCostMap('{"GET /x": "5"}')).toThrow(/positive number/); + }); + + it('rejects a key with no method prefix', () => { + expect(() => compileCostMap('{"/no-method": 1}')).toThrow(/METHOD \/pattern/); + }); +}); + +describe('computeQueryCost', () => { + it('uses the default cost when no route matched', () => { + expect(computeQueryCost(null, undefined)).toBe(1); + }); + + it('uses the matched route base cost with no limit param', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc/holders'); + expect(computeQueryCost(matched, undefined)).toBe(3); + }); + + it('multiplies by a numeric limit param', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc/holders'); + expect(computeQueryCost(matched, '100')).toBe(300); + }); + + it('ignores a limit of 1 or less (no discount below base cost)', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc/holders'); + expect(computeQueryCost(matched, '1')).toBe(3); + expect(computeQueryCost(matched, '0')).toBe(3); + }); + + it('ignores a non-numeric limit param', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc/holders'); + expect(computeQueryCost(matched, 'not-a-number')).toBe(3); + }); + + it('uses the first value when limit is an array', () => { + const routes = compileCostMap(); + const matched = matchCostRoute(routes, 'GET', '/creators/abc/holders'); + expect(computeQueryCost(matched, ['50', '999'])).toBe(150); + }); +}); diff --git a/src/utils/query-cost.utils.ts b/src/utils/query-cost.utils.ts new file mode 100644 index 0000000..947224c --- /dev/null +++ b/src/utils/query-cost.utils.ts @@ -0,0 +1,140 @@ +// src/utils/query-cost.utils.ts +// Route-pattern matching and cost computation for the query cost governor +// (#755). Deliberately not path-to-regexp / req.route: the governor is +// mounted as router-level middleware ahead of route resolution, so req.route +// isn't populated yet when it runs — matching has to work off req.path +// directly. + +import { + DEFAULT_QUERY_COST, + DEFAULT_QUERY_COST_MAP, +} from '../constants/query-cost.constants'; + +export interface CompiledCostRoute { + method: string; + pattern: string; + regex: RegExp; + baseCost: number; +} + +/** Converts an Express-style `:param` pattern into a matching RegExp. */ +function compilePattern(pattern: string): RegExp { + const escaped = pattern + .split('/') + .map(segment => + segment.startsWith(':') + ? '[^/]+' + : segment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + ) + .join('/'); + return new RegExp(`^${escaped}/?$`); +} + +/** Parses a `"METHOD /pattern"` key into its parts. */ +function parseKey(key: string): { method: string; pattern: string } | null { + const spaceIndex = key.indexOf(' '); + if (spaceIndex === -1) return null; + return { + method: key.slice(0, spaceIndex).toUpperCase(), + pattern: key.slice(spaceIndex + 1), + }; +} + +/** + * Merges the default cost map with an optional JSON override (from + * QUERY_COST_MAP_JSON), compiling every entry into a matchable route once at + * startup rather than on every request. + */ +export function compileCostMap( + overrideJson?: string, + defaults: Record = DEFAULT_QUERY_COST_MAP +): CompiledCostRoute[] { + const merged: Record = { ...defaults }; + + if (overrideJson) { + let parsed: unknown; + try { + parsed = JSON.parse(overrideJson); + } catch { + throw new Error( + 'QUERY_COST_MAP_JSON is not valid JSON — expected an object of "METHOD /pattern": cost entries' + ); + } + if ( + typeof parsed !== 'object' || + parsed === null || + Array.isArray(parsed) + ) { + throw new Error( + 'QUERY_COST_MAP_JSON must be a JSON object of "METHOD /pattern": cost entries' + ); + } + for (const [key, value] of Object.entries( + parsed as Record + )) { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + throw new Error( + `QUERY_COST_MAP_JSON entry "${key}" must map to a positive number` + ); + } + merged[key] = value; + } + } + + const compiled: CompiledCostRoute[] = []; + for (const [key, baseCost] of Object.entries(merged)) { + const parts = parseKey(key); + if (!parts) { + throw new Error( + `Query cost map key "${key}" must be of the form "METHOD /pattern"` + ); + } + compiled.push({ + method: parts.method, + pattern: parts.pattern, + regex: compilePattern(parts.pattern), + baseCost, + }); + } + return compiled; +} + +/** Finds the first compiled route matching this method+path, if any. */ +export function matchCostRoute( + routes: CompiledCostRoute[], + method: string, + path: string +): CompiledCostRoute | null { + const upperMethod = method.toUpperCase(); + for (const route of routes) { + if (route.method === upperMethod && route.regex.test(path)) { + return route; + } + } + return null; +} + +/** + * Computes the cost of a request: the matched route's base cost, or + * DEFAULT_QUERY_COST when unmatched, multiplied by the `limit` query param + * when present (issue #755's "parameterised cost" requirement — a caller + * asking for more rows pays proportionally more). + */ +/** Redis key for a caller's rolling query-cost sorted set. */ +export function buildQueryCostRedisKey(identity: string): string { + return `qcost:${identity}`; +} + +export function computeQueryCost( + matched: CompiledCostRoute | null, + limitParam: unknown +): number { + const baseCost = matched?.baseCost ?? DEFAULT_QUERY_COST; + const limit = Array.isArray(limitParam) ? limitParam[0] : limitParam; + const parsedLimit = + typeof limit === 'string' ? Number.parseInt(limit, 10) : NaN; + if (Number.isFinite(parsedLimit) && parsedLimit > 1) { + return baseCost * parsedLimit; + } + return baseCost; +}