diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 61aeffb7d7..9c686e5fc5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,12 +2,14 @@ name: Run tests concurrency: # Include event_name so push and pull_request on the same branch do not cancel each other. - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'workflow_call' && github.sha || github.head_ref || github.ref_name || github.ref }} + # Include PR head SHA so re-triggers on the same commit do not cancel an in-flight run. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'workflow_call' && github.sha || (github.event_name == 'pull_request' && format('{0}-{1}', github.head_ref || github.ref_name, github.event.pull_request.head.sha) || format('{0}-{1}', github.head_ref || github.ref_name || github.ref, github.sha)) }} # Keep in-progress pull_request suites running; duplicate synchronize events were # cancelling shards mid-queue and leaving cancelled required checks on the PR. cancel-in-progress: ${{ github.event_name != 'pull_request' }} on: + pull_request: workflow_dispatch: push: branches-ignore: @@ -323,13 +325,13 @@ jobs: actions: write concurrency: # Isolate by event_name so push + pull_request on the same branch do not cancel pending jobs. - group: capgo-local-services-backend-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }} + group: capgo-local-services-backend-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: backend-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard_id }} SUPABASE_WORKTREE_PORT_OFFSET: ${{ matrix.supabase_port_offset }} strategy: - fail-fast: true + fail-fast: false matrix: include: - shard: 1/6 @@ -489,7 +491,7 @@ jobs: contents: read actions: write concurrency: - group: capgo-local-services-backend-plugin-${{ github.event_name }}-${{ github.repository }} + group: capgo-local-services-backend-plugin-${{ github.event_name }}-${{ github.repository }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: backend-plugin-${{ github.run_id }}-${{ github.run_attempt }} @@ -599,7 +601,7 @@ jobs: contents: read concurrency: # Isolate by event_name so push + pull_request on the same branch do not cancel pending jobs. - group: capgo-local-services-backend-sql-${{ github.event_name }}-${{ github.repository }} + group: capgo-local-services-backend-sql-${{ github.event_name }}-${{ github.repository }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: backend-sql-${{ github.run_id }}-${{ github.run_attempt }} @@ -669,7 +671,7 @@ jobs: contents: read concurrency: # Isolate by event_name so push + pull_request on the same branch do not cancel pending jobs. - group: capgo-local-services-backend-sql-catalog-${{ github.event_name }}-${{ github.repository }} + group: capgo-local-services-backend-sql-catalog-${{ github.event_name }}-${{ github.repository }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: backend-sql-catalog-${{ github.run_id }}-${{ github.run_attempt }} @@ -741,7 +743,7 @@ jobs: contents: read concurrency: # Isolate by event_name so push + pull_request on the same branch do not cancel pending jobs. - group: capgo-local-services-cloudflare-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }} + group: capgo-local-services-cloudflare-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: cloudflare-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard_id }} @@ -749,7 +751,7 @@ jobs: CLOUDFLARE_WORKER_PORT_OFFSET: ${{ matrix.cloudflare_worker_port_offset }} CLOUDFLARE_PERSIST_DIR: .context/cloudflare-workers-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.supabase_port_offset }} strategy: - fail-fast: true + fail-fast: false matrix: include: - shard: 1/8 @@ -883,7 +885,7 @@ jobs: permissions: contents: read concurrency: - group: capgo-local-services-cloudflare-plugin-${{ github.event_name }}-${{ github.repository }} + group: capgo-local-services-cloudflare-plugin-${{ github.event_name }}-${{ github.repository }}-${{ github.sha }} cancel-in-progress: false env: SUPABASE_WORKTREE_INSTANCE: cloudflare-plugin-${{ github.run_id }}-${{ github.run_attempt }} @@ -998,7 +1000,7 @@ jobs: contents: read concurrency: # Isolate by event_name so push + pull_request on the same branch do not cancel pending jobs. - group: capgo-local-services-playwright-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }} + group: capgo-local-services-playwright-${{ github.event_name }}-${{ github.repository }}-${{ matrix.shard }}-${{ github.sha }} cancel-in-progress: false strategy: fail-fast: false diff --git a/cloudflare_workers/files/index.ts b/cloudflare_workers/files/index.ts index c32af980e4..2c1056c872 100644 --- a/cloudflare_workers/files/index.ts +++ b/cloudflare_workers/files/index.ts @@ -1,4 +1,5 @@ import { WorkerEntrypoint } from 'cloudflare:workers' +import { FILE_READ_TRACKING_QUERY_PARAMS, getAttachmentFileIdFromReadPath, hasDeletedFileMarker } from '../../supabase/functions/_backend/files/file_read_cache.ts' import { app as files } from '../../supabase/functions/_backend/files/files.ts' import { handlePreviewRequest, isPreviewSubdomain } from '../../supabase/functions/_backend/files/preview.ts' import { app as download_link } from '../../supabase/functions/_backend/private/download_link.ts' @@ -11,7 +12,6 @@ export { AttachmentUploadHandler, UploadHandler } from '../../supabase/functions const functionName = 'files' const app = createHono(functionName, version) -const TRACKING_QUERY_PARAMS = ['device_id'] as const type CachedFilesLoopback = { fetch: (request: Request, init?: { cf?: { cacheKey: string } }) => Promise @@ -66,7 +66,7 @@ function isCacheablePreviewRead(request: Request): boolean { function buildWorkersCacheKey(request: Request): string | null { const url = new URL(request.url) if (isCacheableAttachmentRead(request)) - return `/files-cache${url.pathname}${normalizeSearch(url, TRACKING_QUERY_PARAMS)}` + return `/files-cache${url.pathname}${normalizeSearch(url, FILE_READ_TRACKING_QUERY_PARAMS)}` if (isCacheablePreviewRead(request)) { const hostname = getRequestHostname(request).toLowerCase() @@ -108,6 +108,23 @@ export const filesWorkerCacheTestUtils = { export default { async fetch(request: Request, env: Cloudflare.Env, ctx: FilesExecutionContext): Promise { + const rawFileId = getAttachmentFileIdFromReadPath(new URL(request.url).pathname) + let fileId: string | null = rawFileId + if (rawFileId) { + try { + fileId = decodeURIComponent(rawFileId) + } + catch { + // Let the files handler return its invalid-path response. + } + } + if (fileId && await hasDeletedFileMarker(fileId)) { + return new Response(JSON.stringify({ error: 'not_found', message: 'Not found' }), { + status: 404, + headers: { 'Content-Type': 'application/json' }, + }) + } + const cacheKey = buildWorkersCacheKey(request) const cachedFiles = ctx.exports?.CachedFiles if (cacheKey && cachedFiles) diff --git a/supabase/functions/_backend/files/file_read_cache.ts b/supabase/functions/_backend/files/file_read_cache.ts new file mode 100644 index 0000000000..e7e6eed7dd --- /dev/null +++ b/supabase/functions/_backend/files/file_read_cache.ts @@ -0,0 +1,225 @@ +import type { Context } from 'hono' +import { getRuntimeKey } from 'hono/adapter' +import { cloudlog } from '../utils/logging.ts' +import { getDatabaseURL, getPgClient } from '../utils/pg.ts' + +export const FILE_READ_TRACKING_QUERY_PARAMS = ['device_id'] as const +export const DELETED_FILE_CACHE_HEADER = 'x-capgo-file-deleted' +const DELETED_FILE_MARKER_ORIGIN = 'https://capgo-files-cache.internal' +const FILE_READ_CACHE_NAME = 'capgo-file-read-cache' +const FILE_READ_CACHE_ORIGINS = ['https://api.capgo.app'] as const +const FILE_READ_PATH_PREFIXES = [ + '/files/read/attachments/', + '/private/files/read/attachments/', + '/read/attachments/', +] as const + +export function isVersionDeleted(row: { deleted?: boolean | null, deleted_at?: string | Date | null } | null | undefined): boolean { + if (!row) + return false + return row.deleted === true || row.deleted_at != null +} + +export function getAttachmentFileIdFromReadPath(pathname: string): string | null { + for (const prefix of FILE_READ_PATH_PREFIXES) { + if (pathname.startsWith(prefix)) { + const fileId = pathname.slice(prefix.length) + return fileId || null + } + } + return null +} + +export function buildFileReadCacheRequest(request: Request): Request { + const cacheUrl = new URL(request.url) + for (const queryParam of FILE_READ_TRACKING_QUERY_PARAMS) { + cacheUrl.searchParams.delete(queryParam) + } + cacheUrl.searchParams.set('range', request.headers.get('range') || '') + cacheUrl.searchParams.sort() + return new Request(cacheUrl, request) +} + +export function buildDeletedFileMarkerRequest(fileId: string): Request { + return new Request(`${DELETED_FILE_MARKER_ORIGIN}/deleted/${encodeURIComponent(fileId)}`) +} + +function normalizeSearch(url: URL, ignoredParams: readonly string[] = []): string { + const searchParams = new URLSearchParams(url.search) + for (const param of ignoredParams) { + searchParams.delete(param) + } + searchParams.sort() + const search = searchParams.toString() + return search ? `?${search}` : '' +} + +export function buildWorkersFileCacheKey(pathname: string, search = ''): string { + const url = new URL(`https://capgo-files-cache.internal${pathname}${search}`) + return `/files-cache${pathname}${normalizeSearch(url, FILE_READ_TRACKING_QUERY_PARAMS)}` +} + +type CacheLike = Cache & { + default?: Cache + open?: (cacheName: string) => Promise +} + +let fileReadCachePromise: Promise | null = null + +async function resolveFileReadCache(): Promise { + if (typeof caches === 'undefined') + return null + + const cacheStorage = caches as unknown as CacheLike + if (getRuntimeKey() === 'workerd' && cacheStorage.default) + return cacheStorage.default + + if (typeof cacheStorage.open === 'function') { + try { + return await cacheStorage.open(FILE_READ_CACHE_NAME) + } + catch { + return null + } + } + + return null +} + +export async function getFileReadCache(): Promise { + fileReadCachePromise ??= resolveFileReadCache() + return fileReadCachePromise +} + +export async function hasDeletedFileMarker(fileId: string): Promise { + const cache = await getFileReadCache() + if (!cache?.match) + return false + + try { + const cached = await cache.match(buildDeletedFileMarkerRequest(fileId)) + return cached != null + } + catch { + return false + } +} + +export async function markFileDeletedInCache(fileId: string): Promise { + const cache = await getFileReadCache() + if (!cache?.put) + return + + await cache.put(buildDeletedFileMarkerRequest(fileId), new Response('deleted', { + status: 404, + headers: { + 'Cache-Control': 'public, max-age=31536000', + [DELETED_FILE_CACHE_HEADER]: '1', + }, + })) +} + +let sharedDeletedLookupPool: ReturnType | null = null +let sharedDeletedLookupPoolUrl: string | null = null + +function getDeletedLookupPgClient(c: Context): ReturnType { + const dbUrl = getDatabaseURL(c, false) + if (!sharedDeletedLookupPool || sharedDeletedLookupPoolUrl !== dbUrl) { + sharedDeletedLookupPool = getPgClient(c, false) + sharedDeletedLookupPoolUrl = dbUrl + } + return sharedDeletedLookupPool +} + +function buildFileReadCacheRequestsForPath(fileId: string, checksum?: string | null): Request[] { + const identityParamSets: Array> = [{}] + if (checksum) + identityParamSets.push({ key: checksum }) + + return FILE_READ_CACHE_ORIGINS.flatMap(origin => + FILE_READ_PATH_PREFIXES.flatMap((prefix) => { + return identityParamSets.map((identityParams) => { + const url = new URL(`${prefix}${fileId}`, origin) + for (const [key, value] of Object.entries(identityParams)) + url.searchParams.set(key, value) + url.searchParams.set('range', '') + url.searchParams.sort() + return new Request(url) + }) + }), + ) +} + +function buildWorkersFileCacheRequests(fileId: string, checksum?: string | null): Request[] { + const searchVariants = [''] + if (checksum) + searchVariants.push(`?key=${encodeURIComponent(checksum)}`) + + return FILE_READ_PATH_PREFIXES + .filter(prefix => prefix.startsWith('/files/') || prefix.startsWith('/private/')) + .flatMap(prefix => + searchVariants.map(search => + new Request(`${DELETED_FILE_MARKER_ORIGIN}${buildWorkersFileCacheKey(`${prefix}${fileId}`, search)}`), + ), + ) +} + +// Per-DC cache delete/marker calls are best-effort accelerators. The globally durable +// deletion gate is isAttachmentVersionDeleted's primary-DB lookup on app_versions.r2_path +// (deleted/deleted_at), which the serve path checks before returning cached bytes or +// restoring to R2 (see files.ts cache-hit guard). Workers CachedFiles also short-circuits on the local deleted marker. +// Checksum is required so key=checksum cache variants are purged alongside path-only entries. +export async function purgeFileReadCache(fileId: string, checksum?: string | null): Promise { + await markFileDeletedInCache(fileId) + + const cache = await getFileReadCache() + if (!cache || typeof cache.delete !== 'function') + return + + const requests = [ + ...buildFileReadCacheRequestsForPath(fileId, checksum), + ...buildWorkersFileCacheRequests(fileId, checksum), + ] + await Promise.all(requests.map(request => cache.delete(request).catch(() => false))) +} + +export async function isAttachmentVersionDeleted(c: Context, fileId: string): Promise { + if (await hasDeletedFileMarker(fileId)) + return true + + // app_versions.r2_path only tracks bundle zip objects, not arbitrary attachment uploads. + if (!fileId.endsWith('.zip')) + return false + + try { + const pgClient = getDeletedLookupPgClient(c) + const result = await pgClient.query<{ deleted: boolean | null, deleted_at: string | null }>( + ` + SELECT deleted, deleted_at + FROM public.app_versions + WHERE r2_path = $1 + AND (COALESCE(deleted, false) = true OR deleted_at IS NOT NULL) + LIMIT 1 + `, + [fileId], + ) + return result.rows.length > 0 + } + catch (error) { + cloudlog({ + requestId: c.get('requestId'), + message: 'isAttachmentVersionDeleted lookup failed, failing closed', + fileId, + error: error instanceof Error ? error.message : String(error), + }) + return true + } +} + +export const fileReadCacheTestUtils = { + buildFileReadCacheRequest, + buildDeletedFileMarkerRequest, + buildWorkersFileCacheKey, + getAttachmentFileIdFromReadPath, + isVersionDeleted, +} diff --git a/supabase/functions/_backend/files/files.ts b/supabase/functions/_backend/files/files.ts index 1473d77585..91dbdcc7ed 100644 --- a/supabase/functions/_backend/files/files.ts +++ b/supabase/functions/_backend/files/files.ts @@ -18,6 +18,7 @@ import { checkPermissionPg } from '../utils/rbac.ts' import { createStatsBandwidth } from '../utils/stats.ts' import { supabaseAdmin } from '../utils/supabase.ts' import { backgroundTask } from '../utils/utils.ts' +import { buildFileReadCacheRequest, getFileReadCache, isAttachmentVersionDeleted } from './file_read_cache.ts' import { app as files_config } from './files_config.ts' import { parseUploadMetadata } from './parse.ts' import { DEFAULT_RETRY_PARAMS, RetryBucket } from './retry.ts' @@ -32,7 +33,6 @@ const ATTACHMENT_PREFIX = 'attachments' const ATTACHMENT_PLAN_LIMIT: Array<'mau' | 'bandwidth' | 'storage'> = ['mau', 'bandwidth', 'storage'] const TUS_UPLOAD_CONTENT_TYPE = 'application/offset+octet-stream' const FILE_READ_CACHE_CONTROL = 'public, max-age=31536000, immutable' -const TRACKING_QUERY_PARAMS = ['device_id'] export const app = new Hono() @@ -406,9 +406,9 @@ async function getSupabaseStorageResponse(c: Context, fileId: string): Promise { const fileId = c.get('fileId') - // It is imperative that files are read without any database read to avoid bottlenecks and keep file downloads highly available, especially under heavy load. - // This was designed that way, and access to a file that is going to be deleted is not important compared to download availability. - // Do not add DB or R2 checks before serving the file; if the file is missing in R2, a 404 is expected. + // File reads stay off the primary DB. A deleted version may still be in the + // edge cache after R2 trash; check the deleted marker or one indexed r2_path + // lookup before serving or restoring that cache entry. cloudlog({ requestId: c.get('requestId'), message: 'getHandler files', fileId }) if (getRuntimeKey() !== 'workerd') { @@ -423,35 +423,39 @@ async function getHandler(c: Context): Promise { return c.json({ error: 'not_found', message: 'Not found' }, 404) } - // Support for deno cache or CF cache do not remove this - // @ts-expect-error-next-line - const cache = getRuntimeKey() === 'workerd' ? caches.default : caches + const cache = await getFileReadCache() const rawFileId = getRawAttachmentRouteId(c) const candidateKeys = getSafeAttachmentReadCandidateKeys(fileId, rawFileId) - const cacheUrl = new URL(c.req.url) - for (const queryParam of TRACKING_QUERY_PARAMS) { - cacheUrl.searchParams.delete(queryParam) - } - cacheUrl.searchParams.set('range', c.req.header('range') || '') - cacheUrl.searchParams.sort() - const cacheKey = new Request(cacheUrl, c.req) - let response = await cache.match(cacheKey) + const cacheKey = buildFileReadCacheRequest(c.req.raw) + let response = cache ? await cache.match(cacheKey) : null if (response != null) { - response = ensureNoTransformResponse(response) + if (await isAttachmentVersionDeleted(c, fileId)) { + cloudlog({ requestId: c.get('requestId'), message: 'getHandler files cache hit for deleted version', fileId }) + return c.json({ error: 'not_found', message: 'Not found' }, 404) + } + + const cachedResponse = ensureNoTransformResponse(response) + response = cachedResponse cloudlog({ requestId: c.get('requestId'), message: 'getHandler files cache hit' }) if (c.req.raw.method !== 'HEAD') { - await saveBandwidthUsage(c, getTransferredBytesFromResponse(response)) + await saveBandwidthUsage(c, getTransferredBytesFromResponse(cachedResponse)) } - // Best-effort restore: if file is cached but missing in R2, write it back. + // Best-effort restore: if a live file is cached but missing in R2, write it back. await backgroundTask(c, async () => { try { + if (await isAttachmentVersionDeleted(c, fileId)) + return + const retryBucket = new RetryBucket(bucket, DEFAULT_RETRY_PARAMS) const existingObject = await headFirstExistingAttachmentCandidate(retryBucket, candidateKeys) if (existingObject != null) return - const cached = response.clone() + const cached = cachedResponse.clone() const data = await cached.arrayBuffer() + if (await isAttachmentVersionDeleted(c, fileId)) + return + const contentType = cached.headers.get('content-type') || undefined const httpMetadata = buildFileHttpMetadata(contentType, cached.headers.get('cache-control')) await bucket.put(fileId, data, { httpMetadata }) @@ -464,7 +468,12 @@ async function getHandler(c: Context): Promise { cloudlog({ requestId: c.get('requestId'), message: 'Failed to restore cached file to R2', fileId, error: String(err) }) } }) - return response + return cachedResponse + } + + if (await isAttachmentVersionDeleted(c, fileId)) { + cloudlog({ requestId: c.get('requestId'), message: 'getHandler files cache miss for deleted version', fileId }) + return c.json({ error: 'not_found', message: 'Not found' }, 404) } const rangeHeaderFromRequest = c.req.header('range') @@ -521,10 +530,12 @@ async function getHandler(c: Context): Promise { } headers.set('Content-Disposition', `attachment; filename="${object.key}"`) response = new Response(object.body, { headers }) - await backgroundTask(c, () => { - cloudlog({ requestId: c.get('requestId'), message: 'getHandler files cache saved', fileId }) - cache.put(cacheKey, response.clone()) - }) + if (cache && !await isAttachmentVersionDeleted(c, fileId)) { + await backgroundTask(c, () => { + cloudlog({ requestId: c.get('requestId'), message: 'getHandler files cache saved', fileId }) + return cache.put(cacheKey, response.clone()) + }) + } return response } diff --git a/supabase/functions/_backend/triggers/on_version_update.ts b/supabase/functions/_backend/triggers/on_version_update.ts index 9229dba982..06bd531183 100644 --- a/supabase/functions/_backend/triggers/on_version_update.ts +++ b/supabase/functions/_backend/triggers/on_version_update.ts @@ -3,6 +3,7 @@ import type { MiddlewareKeyVariables } from '../utils/hono.ts' import type { Database } from '../utils/supabase.types.ts' import { eq } from 'drizzle-orm' import { Hono } from 'hono/tiny' +import { purgeFileReadCache } from '../files/file_read_cache.ts' import { BRES, middlewareAPISecret, simpleError, triggerValidator } from '../utils/hono.ts' import { cloudlog } from '../utils/logging.ts' import { persistVersionManifestEntries } from '../utils/manifest_persist.ts' @@ -425,6 +426,20 @@ async function deleteManifest(c: Context, record: Database['public']['Tables'][' export async function deleteIt(c: Context, record: Database['public']['Tables']['app_versions']['Row']) { cloudlog({ requestId: c.get('requestId'), message: 'Delete', r2_path: record.r2_path }) + if (record.r2_path) { + try { + await purgeFileReadCache(record.r2_path, record.checksum) + } + catch (error) { + cloudlog({ + requestId: c.get('requestId'), + message: 'purgeFileReadCache failed during version delete', + r2_path: record.r2_path, + error, + }) + } + } + // Manifest files: trash R2 first, then drop DB rows. Must finish before ACK. await deleteManifest(c, record) diff --git a/tests/channel-rate-limit.test.ts b/tests/channel-rate-limit.test.ts index 97f53ad23c..5b062726f5 100644 --- a/tests/channel-rate-limit.test.ts +++ b/tests/channel-rate-limit.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto' import { env } from 'node:process' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { BASE_URL, getBaseData, headers, PLUGIN_BASE_URL, resetAndSeedAppData, resetAppData, resetAppDataStats } from './test-utils.ts' +import { BASE_URL, getBaseData, headers, PLUGIN_BASE_URL, resetAndSeedAppData, resetAppData, resetAppDataStats, warmEdgeEndpoint } from './test-utils.ts' // Rate limiting uses Cloudflare Workers Cache API, which isn't available in Supabase Edge Functions const USE_CLOUDFLARE = env.USE_CLOUDFLARE_WORKERS === 'true' @@ -54,26 +54,17 @@ function sleep(ms: number): Promise { } // The limiter counts requests in a one-second window anchored to the first request -// (see channelSelfRateLimit.ts) and its cache counter is not atomic under concurrent -// requests, so bursts must be sequential AND finish inside the window to trip it. -// Send sequential requests within the window budget; when a slow runner lets the -// window expire before the limit trips, wait out the counter and retry the round. -const WINDOW_BUDGET_MS = 900 - +// (see channelSelfRateLimit.ts). Send limit+1 sequential requests as fast as the runner +// allows; when a slow endpoint lets the window expire mid-burst, wait out the counter +// and retry with a fresh burst. async function hitRateLimit(makeRequest: (deviceId: string) => Promise, deviceId: string): Promise { - for (let round = 0; round < 4; round++) { - const roundStart = Date.now() - let sent = 0 - while (Date.now() - roundStart < WINDOW_BUDGET_MS) { + for (let round = 0; round < 6; round++) { + // Send limit+1 back-to-back; slow runners may span multiple 1s windows. + for (let sent = 0; sent < OP_LIMIT_PER_SECOND + 1; sent++) { const response = await makeRequest(deviceId) - sent += 1 if (response.status === 429) return response - // Three times the limit landed inside one window without a 429: the limiter is broken. - if (sent >= OP_LIMIT_PER_SECOND * 3) - return null } - // Window expired before the limit could trip; let the counter reset and retry. await sleep(1100) } return null @@ -114,6 +105,18 @@ async function testRateLimitBehavior( beforeAll(async () => { await resetAndSeedAppData(APPNAME) + if (USE_CLOUDFLARE) { + await warmEdgeEndpoint(`${PLUGIN_BASE_URL}/channel_self`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(getBaseData(APPNAME)), + }) + await warmEdgeEndpoint(`${BASE_URL}/device`, { + method: 'DELETE', + headers, + body: JSON.stringify({ app_id: APPNAME, device_id: randomUUID().toLowerCase() }), + }) + } }) afterAll(async () => { diff --git a/tests/channel_self.test.ts b/tests/channel_self.test.ts index 934c0b2c66..654b750b10 100644 --- a/tests/channel_self.test.ts +++ b/tests/channel_self.test.ts @@ -3,7 +3,7 @@ import type { DeviceLink, HttpMethod } from './test-utils.ts' import { randomUUID } from 'node:crypto' import { env } from 'node:process' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { getBaseData, getSupabaseClient, PLUGIN_BASE_URL, resetAndSeedAppData, resetAppData, resetAppDataStats } from './test-utils.ts' +import { fetchTestRequest, getBaseData, getSupabaseClient, PLUGIN_BASE_URL, resetAndSeedAppData, resetAppData, resetAppDataStats, warmEdgeEndpoint } from './test-utils.ts' interface ChannelInfo { id: number @@ -32,12 +32,10 @@ async function fetchEndpoint(method: HttpMethod, bodyIn: object) { } const body = method !== 'DELETE' ? JSON.stringify(bodyIn) : undefined - const response = await fetch(url, { + return fetchTestRequest(url.toString(), { method, body, }) - - return response } async function fetchGetChannels(queryParams: Record) { @@ -45,11 +43,9 @@ async function fetchGetChannels(queryParams: Record) { for (const [key, value] of Object.entries(queryParams)) url.searchParams.append(key, value) - const response = await fetch(url, { + return fetchTestRequest(url.toString(), { method: 'GET', }) - - return response } async function getResponseErrorCode(response: Response) { @@ -67,6 +63,12 @@ async function withSupabaseCall { await resetAndSeedAppData(APPNAME) + // Cold first /channel_self request can 502 under Deno shard load; warm before assertions. + await warmEdgeEndpoint(`${PLUGIN_BASE_URL}/channel_self`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(getBaseData(APPNAME)), + }) }) afterAll(async () => { await resetAppData(APPNAME) diff --git a/tests/files-app-read-guard.unit.test.ts b/tests/files-app-read-guard.unit.test.ts index 50918a1650..d6bc6a8b2d 100644 --- a/tests/files-app-read-guard.unit.test.ts +++ b/tests/files-app-read-guard.unit.test.ts @@ -1,7 +1,9 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' const getAppByAppIdPgMock = vi.fn(async () => null) -const getPgClientMock = vi.fn(() => ({})) +const getPgClientMock = vi.fn(() => ({ + query: async () => ({ rows: [] }), +})) const originalCaches = globalThis.caches const cachedBodiesByPath = new Map([ ['/read/attachments/orgs/test-org/apps/test-app/orphan.txt', 'cached orphan bytes'], @@ -24,6 +26,7 @@ vi.mock('../supabase/functions/_backend/utils/discord.ts', () => ({ vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ closeClient: () => Promise.resolve(), getAppOwnerPostgres: vi.fn(), + getDatabaseURL: vi.fn(() => 'postgres://test'), getDrizzleClient: vi.fn(() => ({})), getPgClient: getPgClientMock, })) @@ -70,7 +73,7 @@ describe('files app-scoped cached reads', () => { globalThis.caches = originalCaches }) - it.concurrent('serves deleted app-scoped files from cache without a database lookup', async () => { + it.concurrent('serves orphan app-scoped files from cache when no deleted version row exists', async () => { const bucketPut = vi.fn() const appGlobal = await createFilesApp() @@ -85,11 +88,10 @@ describe('files app-scoped cached reads', () => { expect(response.status).toBe(200) expect(await response.text()).toBe('cached orphan bytes') expect(bucketPut).not.toHaveBeenCalled() - expect(getPgClientMock).not.toHaveBeenCalled() expect(getAppByAppIdPgMock).not.toHaveBeenCalled() }) - it.concurrent('serves malformed app-scoped paths from cache without a database lookup', async () => { + it.concurrent('serves malformed app-scoped paths from cache when no deleted version row exists', async () => { const bucketPut = vi.fn() const appGlobal = await createFilesApp() @@ -104,8 +106,6 @@ describe('files app-scoped cached reads', () => { expect(response.status).toBe(200) expect(await response.text()).toBe('cached malformed bytes') expect(bucketPut).not.toHaveBeenCalled() - expect(getPgClientMock).not.toHaveBeenCalled() expect(getAppByAppIdPgMock).not.toHaveBeenCalled() - expect(getPgClientMock).not.toHaveBeenCalled() }) }) diff --git a/tests/files-bandwidth.unit.test.ts b/tests/files-bandwidth.unit.test.ts index 57ea32e9b6..cccae41d54 100644 --- a/tests/files-bandwidth.unit.test.ts +++ b/tests/files-bandwidth.unit.test.ts @@ -2,6 +2,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const retryGetMock = vi.fn() const createStatsBandwidthMock = vi.fn() +const queryMock = vi.fn() +const closeClientMock = vi.fn() +const getPgClientMock = vi.fn(() => ({ query: queryMock })) vi.mock('hono/adapter', async (importOriginal) => { const actual = await importOriginal() @@ -16,6 +19,14 @@ vi.mock('../supabase/functions/_backend/utils/discord.ts', () => ({ sendDiscordAlert: () => Promise.resolve(), })) +vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ + closeClient: closeClientMock, + getAppOwnerPostgres: vi.fn(), + getDatabaseURL: vi.fn(() => 'postgres://test'), + getDrizzleClient: vi.fn(() => ({})), + getPgClient: getPgClientMock, +})) + vi.mock('../supabase/functions/_backend/files/retry.ts', () => ({ DEFAULT_RETRY_PARAMS: {}, RetryBucket: class RetryBucketMock { @@ -64,9 +75,14 @@ describe('files bandwidth tracking', () => { beforeEach(() => { vi.resetModules() vi.clearAllMocks() + queryMock.mockResolvedValue({ rows: [] }) globalThis.caches = { default: { - match: async () => null, + match: async (request: Request) => { + if (new URL(request.url).pathname.startsWith('/deleted/')) + return null + return null + }, put: async () => { }, }, } as any @@ -101,12 +117,16 @@ describe('files bandwidth tracking', () => { it('does not track bandwidth for cached HEAD reads', async () => { globalThis.caches = { default: { - match: async () => new Response(null, { - headers: { - 'cache-control': 'public, max-age=3600', - 'content-length': '3478395', - }, - }), + match: async (request: Request) => { + if (new URL(request.url).pathname.startsWith('/deleted/')) + return null + return new Response(null, { + headers: { + 'cache-control': 'public, max-age=3600', + 'content-length': '3478395', + }, + }) + }, put: async () => { }, }, } as any diff --git a/tests/files-deleted-cache.unit.test.ts b/tests/files-deleted-cache.unit.test.ts new file mode 100644 index 0000000000..1c373e5dd4 --- /dev/null +++ b/tests/files-deleted-cache.unit.test.ts @@ -0,0 +1,167 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const queryMock = vi.fn() +const closeClientMock = vi.fn() +const getPgClientMock = vi.fn(() => ({ query: queryMock })) + +vi.mock('hono/adapter', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + getRuntimeKey: () => 'workerd', + } +}) + +vi.mock('../supabase/functions/_backend/utils/discord.ts', () => ({ + sendDiscordAlert500: () => Promise.resolve(), + sendDiscordAlert: () => Promise.resolve(), +})) + +vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ + closeClient: closeClientMock, + getAppOwnerPostgres: vi.fn(), + getDatabaseURL: vi.fn(() => 'postgres://test'), + getDrizzleClient: vi.fn(() => ({})), + getPgClient: getPgClientMock, +})) + +vi.mock('../supabase/functions/_backend/files/retry.ts', () => ({ + DEFAULT_RETRY_PARAMS: {}, + RetryBucket: class RetryBucketMock { + head() { + return Promise.resolve(null) + } + + get() { + return Promise.resolve(null) + } + }, +})) + +const originalCaches = globalThis.caches + +function createCache(entries: Map = new Map()) { + return { + match: async (request: Request) => { + return entries.get(new URL(request.url).pathname) ?? entries.get(request.url) ?? null + }, + put: async (request: Request, response: Response) => { + entries.set(new URL(request.url).pathname, response) + entries.set(request.url, response) + }, + delete: async (request: Request) => { + entries.delete(new URL(request.url).pathname) + entries.delete(request.url) + return true + }, + entries, + } +} + +async function createFilesApp() { + const { app: files } = await import('../supabase/functions/_backend/files/files.ts') + const { createAllCatch, createHono } = await import('../supabase/functions/_backend/utils/hono.ts') + const { version } = await import('../supabase/functions/_backend/utils/version.ts') + + const appGlobal = createHono('files', version) + appGlobal.route('/', files) + createAllCatch(appGlobal, 'files') + return appGlobal +} + +describe('deleted bundle cache', () => { + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + queryMock.mockResolvedValue({ rows: [] }) + }) + + afterEach(() => { + globalThis.caches = originalCaches + }) + + it('does not serve or restore a cached file when the version is deleted', async () => { + queryMock.mockResolvedValue({ + rows: [{ deleted: true, deleted_at: '2026-08-16T00:00:00Z' }], + }) + + const cache = createCache(new Map([ + ['/read/attachments/orgs/test-org/apps/test-app/bundle.zip', new Response('cached deleted bytes', { + headers: { 'content-type': 'application/zip' }, + })], + ])) + globalThis.caches = { default: cache } as any + + const bucketPut = vi.fn() + const appGlobal = await createFilesApp() + const response = await appGlobal.fetch( + new Request('http://localhost/read/attachments/orgs/test-org/apps/test-app/bundle.zip'), + { ATTACHMENT_BUCKET: { put: bucketPut, head: vi.fn(), get: vi.fn() } }, + { waitUntil: () => { } } as any, + ) + + expect(response.status).toBe(404) + expect(await response.json()).toMatchObject({ error: 'not_found' }) + expect(bucketPut).not.toHaveBeenCalled() + expect(queryMock).toHaveBeenCalled() + }) + + it('does not serve or restore a cached file when a deleted marker is present', async () => { + const { buildDeletedFileMarkerRequest } = await import('../supabase/functions/_backend/files/file_read_cache.ts') + const cache = createCache(new Map([ + ['/read/attachments/orgs/test-org/apps/test-app/bundle.zip', new Response('cached deleted bytes', { + headers: { 'content-type': 'application/zip' }, + })], + ])) + await cache.put(buildDeletedFileMarkerRequest('orgs/test-org/apps/test-app/bundle.zip'), new Response('deleted')) + globalThis.caches = { default: cache } as any + + const bucketPut = vi.fn() + const appGlobal = await createFilesApp() + const response = await appGlobal.fetch( + new Request('http://localhost/read/attachments/orgs/test-org/apps/test-app/bundle.zip'), + { ATTACHMENT_BUCKET: { put: bucketPut, head: vi.fn(), get: vi.fn() } }, + { waitUntil: () => { } } as any, + ) + + expect(response.status).toBe(404) + expect(bucketPut).not.toHaveBeenCalled() + expect(queryMock).not.toHaveBeenCalled() + }) + + it('purges path and key=checksum cache entries and writes a deleted marker', async () => { + const cache = createCache() + globalThis.caches = { default: cache } as any + + const { buildDeletedFileMarkerRequest, buildFileReadCacheRequest, purgeFileReadCache } = await import('../supabase/functions/_backend/files/file_read_cache.ts') + const fileId = 'orgs/org-1/apps/com.cleanup.test/1.0.0.zip' + const checksum = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + const readRequest = buildFileReadCacheRequest(new Request(`https://api.capgo.app/files/read/attachments/${fileId}`)) + const keyedReadRequest = buildFileReadCacheRequest(new Request(`https://api.capgo.app/files/read/attachments/${fileId}?device_id=device-1&key=${checksum}`)) + await cache.put(readRequest, new Response('cached bundle bytes', { + headers: { 'content-type': 'application/zip' }, + })) + await cache.put(keyedReadRequest, new Response('cached keyed bundle bytes', { + headers: { 'content-type': 'application/zip' }, + })) + expect(await cache.match(readRequest)).not.toBeNull() + expect(await cache.match(keyedReadRequest)).not.toBeNull() + + await purgeFileReadCache(fileId, checksum) + + expect(await cache.match(readRequest)).toBeNull() + expect(await cache.match(keyedReadRequest)).toBeNull() + const marker = await cache.match(buildDeletedFileMarkerRequest(fileId)) + expect(marker).not.toBeNull() + expect(marker?.status).toBe(404) + }) + + it('treats deleted and deleted_at as deleted versions', async () => { + const { isVersionDeleted } = await import('../supabase/functions/_backend/files/file_read_cache.ts') + + expect(isVersionDeleted({ deleted: true, deleted_at: null })).toBe(true) + expect(isVersionDeleted({ deleted: false, deleted_at: '2026-08-16T00:00:00Z' })).toBe(true) + expect(isVersionDeleted({ deleted: false, deleted_at: null })).toBe(false) + expect(isVersionDeleted(null)).toBe(false) + }) +}) diff --git a/tests/files-local-read-proxy.unit.test.ts b/tests/files-local-read-proxy.unit.test.ts index c84904c09e..99c1274cd3 100644 --- a/tests/files-local-read-proxy.unit.test.ts +++ b/tests/files-local-read-proxy.unit.test.ts @@ -23,6 +23,7 @@ vi.mock('../supabase/functions/_backend/utils/discord.ts', () => ({ vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ closeClient: () => Promise.resolve(), getAppOwnerPostgres: vi.fn(), + getDatabaseURL: vi.fn(() => 'postgres://test'), getDrizzleClient: vi.fn(() => ({})), getPgClient: getPgClientMock, })) diff --git a/tests/files-r2-error.test.ts b/tests/files-r2-error.test.ts index 7843ce6000..6ad3e700bf 100644 --- a/tests/files-r2-error.test.ts +++ b/tests/files-r2-error.test.ts @@ -2,6 +2,9 @@ import { describe, expect, it, vi } from 'vitest' const retryGetMock = vi.fn() const retryHeadMock = vi.fn() +const queryMock = vi.fn() +const closeClientMock = vi.fn() +const getPgClientMock = vi.fn(() => ({ query: queryMock })) vi.mock('cloudflare:workers', () => ({ DurableObject: class DurableObjectMock {}, @@ -20,6 +23,14 @@ vi.mock('../supabase/functions/_backend/utils/discord.ts', () => ({ sendDiscordAlert: () => Promise.resolve(), })) +vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ + closeClient: closeClientMock, + getAppOwnerPostgres: vi.fn(), + getDatabaseURL: vi.fn(() => 'postgres://test'), + getDrizzleClient: vi.fn(() => ({})), + getPgClient: getPgClientMock, +})) + vi.mock('../supabase/functions/_backend/files/retry.ts', () => ({ DEFAULT_RETRY_PARAMS: {}, RetryBucket: class RetryBucketMock { @@ -37,6 +48,7 @@ vi.mock('../supabase/functions/_backend/files/retry.ts', () => ({ describe('files R2 error handling', () => { it('should return 503 when R2 get fails', async () => { vi.resetModules() + queryMock.mockResolvedValue({ rows: [] }) retryHeadMock.mockResolvedValue(null) retryGetMock.mockImplementation(() => { throw new Error('r2 unavailable') @@ -71,11 +83,15 @@ describe('files R2 error handling', () => { it('should add immutable cache control and strip tracking params on cached responses', async () => { vi.resetModules() + queryMock.mockResolvedValue({ rows: [] }) retryHeadMock.mockResolvedValue(null) retryGetMock.mockResolvedValue(null) const matchMock = vi.fn(async (request: Request) => { const cacheUrl = new URL(request.url) + if (cacheUrl.pathname.startsWith('/deleted/')) + return null + expect(cacheUrl.searchParams.get('device_id')).toBeNull() expect(cacheUrl.searchParams.get('key')).toBe('checksum') expect(cacheUrl.searchParams.get('range')).toBe('') @@ -112,7 +128,7 @@ describe('files R2 error handling', () => { expect(response.status).toBe(200) expect(response.headers.get('cache-control')).toBe('public, max-age=31536000, immutable, no-transform') - expect(matchMock).toHaveBeenCalledTimes(1) + expect(matchMock).toHaveBeenCalledTimes(2) }) it('should persist no-transform in file metadata written to R2', async () => { diff --git a/tests/on-version-update-cleanup.unit.test.ts b/tests/on-version-update-cleanup.unit.test.ts index 436f2c7b89..425fe1c129 100644 --- a/tests/on-version-update-cleanup.unit.test.ts +++ b/tests/on-version-update-cleanup.unit.test.ts @@ -13,6 +13,7 @@ const { manifestSelectWhere, moveObjectToTrash, pgQuery, + purgeFileReadCache, supabaseAdmin, } = vi.hoisted(() => { const callOrder: string[] = [] @@ -75,10 +76,15 @@ const { manifestSelectWhere, moveObjectToTrash, pgQuery, + purgeFileReadCache: vi.fn(async () => {}), supabaseAdmin: vi.fn(() => ({ from: supabaseFrom })), } }) +vi.mock('../supabase/functions/_backend/files/file_read_cache.ts', () => ({ + purgeFileReadCache, +})) + vi.mock('../supabase/functions/_backend/utils/s3.ts', () => ({ getPath: vi.fn(), s3: { @@ -124,6 +130,7 @@ function createVersion(overrides: Record = {}) { name: '1.0.0', owner_org: 'org-1', r2_path: 'orgs/org-1/apps/com.cleanup.test/1.0.0.zip', + checksum: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', storage_provider: 'r2', ...overrides, } as any @@ -179,6 +186,10 @@ describe('on_version_update deleted version cleanup', () => { it('moves the bundle to trash and clears stored size for soft-deleted versions', async () => { const response = await deleteIt(createContext(), createVersion()) expect(response.status).toBe(200) + expect(purgeFileReadCache).toHaveBeenCalledWith( + 'orgs/org-1/apps/com.cleanup.test/1.0.0.zip', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + ) expect(moveObjectToTrash).toHaveBeenCalledWith(expect.anything(), 'orgs/org-1/apps/com.cleanup.test/1.0.0.zip') expect(appVersionsMetaUpdate).toHaveBeenCalledWith({ size: 0 }) }) diff --git a/tests/rbac-apikey-request-identity-rpc.test.ts b/tests/rbac-apikey-request-identity-rpc.test.ts index 6936c0b17f..50f569f298 100644 --- a/tests/rbac-apikey-request-identity-rpc.test.ts +++ b/tests/rbac-apikey-request-identity-rpc.test.ts @@ -278,7 +278,7 @@ describe('app_versions RBAC update policy', () => { .select('id, deleted') if (deletedError) { - expect(deletedError.message.toLowerCase()).toMatch(/row|permission_denied_bundle_delete/) + expect(deletedError.message).toMatch(/PERMISSION_DENIED_BUNDLE_DELETE/i) } else { expect(deletedUpdate).toEqual([])