feat(enterprise): add trace processor lease holders

backend/package.json

@@ -41,7 +41,7 @@
     "prepack": "npm run build",
     "typecheck": "tsc --noEmit",
     "test": "jest",
-    "test:core": "jest --runInBand --forceExit src/agent/communication/__tests__/agentMessageBus.test.ts src/agent/core/executors/__tests__/strategyExecutor.test.ts src/agent/core/executors/__tests__/hypothesisExecutor.test.ts src/agent/context/__tests__/enhancedSessionContext.test.ts src/tests/adbTools.test.ts src/services/__tests__/sessionLogger.test.ts src/services/__tests__/traceAnalysisSkillConfig.test.ts src/agent/agents/domain/__tests__/registry.test.ts src/agentv3/__tests__/sqlIncludeInjector.test.ts src/agentv3/__tests__/analysisPatternMemory.test.ts src/agentv3/__tests__/claudeRuntimeRuntimeSnapshots.test.ts src/middleware/__tests__/auth.test.ts src/services/__tests__/rbac.test.ts src/routes/__tests__/agentRoutesRbac.test.ts src/routes/__tests__/ownerGuardRoutes.test.ts src/routes/__tests__/requestContextRouteCoverage.test.ts src/middleware/__tests__/legacyApiCompatibility.test.ts src/services/__tests__/enterpriseDb.test.ts src/services/__tests__/enterpriseSchema.test.ts src/services/__tests__/enterpriseRepository.test.ts src/services/__tests__/processRss.test.ts src/scripts/__tests__/benchmarkTraceProcessorRss.test.ts src/services/__tests__/enterpriseKnowledgeScope.test.ts src/services/__tests__/enterpriseMigration.test.ts src/services/__tests__/runtimeSnapshotStore.test.ts src/services/providerManager/__tests__/localSecretStore.test.ts src/services/providerManager/__tests__/enterpriseProviderStore.test.ts src/routes/__tests__/enterpriseTraceMetadataRoutes.test.ts src/routes/__tests__/enterpriseReportRoutes.test.ts src/routes/__tests__/enterpriseRestartPersistence.test.ts",
+    "test:core": "jest --runInBand --forceExit src/agent/communication/__tests__/agentMessageBus.test.ts src/agent/core/executors/__tests__/strategyExecutor.test.ts src/agent/core/executors/__tests__/hypothesisExecutor.test.ts src/agent/context/__tests__/enhancedSessionContext.test.ts src/tests/adbTools.test.ts src/services/__tests__/sessionLogger.test.ts src/services/__tests__/traceAnalysisSkillConfig.test.ts src/agent/agents/domain/__tests__/registry.test.ts src/agentv3/__tests__/sqlIncludeInjector.test.ts src/agentv3/__tests__/analysisPatternMemory.test.ts src/agentv3/__tests__/claudeRuntimeRuntimeSnapshots.test.ts src/middleware/__tests__/auth.test.ts src/services/__tests__/rbac.test.ts src/routes/__tests__/agentRoutesRbac.test.ts src/routes/__tests__/ownerGuardRoutes.test.ts src/routes/__tests__/requestContextRouteCoverage.test.ts src/middleware/__tests__/legacyApiCompatibility.test.ts src/services/__tests__/enterpriseDb.test.ts src/services/__tests__/enterpriseSchema.test.ts src/services/__tests__/enterpriseRepository.test.ts src/services/__tests__/processRss.test.ts src/services/__tests__/traceProcessorLeaseStore.test.ts src/scripts/__tests__/benchmarkTraceProcessorRss.test.ts src/services/__tests__/enterpriseKnowledgeScope.test.ts src/services/__tests__/enterpriseMigration.test.ts src/services/__tests__/runtimeSnapshotStore.test.ts src/services/providerManager/__tests__/localSecretStore.test.ts src/services/providerManager/__tests__/enterpriseProviderStore.test.ts src/routes/__tests__/enterpriseTraceMetadataRoutes.test.ts src/routes/__tests__/enterpriseReportRoutes.test.ts src/routes/__tests__/enterpriseRestartPersistence.test.ts",
     "test:watch": "jest --watch",
     "test:coverage": "jest --coverage",
     "test:unit": "jest --testPathPatterns=src/tests",

backend/src/routes/__tests__/agentRoutesRbac.test.ts

@@ -2,13 +2,28 @@
 // Copyright (C) 2024-2026 Gracker (Chris)
 // This file is part of SmartPerfetto. See LICENSE for details.
 
-import { afterEach, describe, expect, it } from '@jest/globals';
+import { afterEach, describe, expect, it, jest } from '@jest/globals';
 import express from 'express';
+import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
 import request from 'supertest';
+import { ENTERPRISE_FEATURE_FLAG_ENV } from '../../config';
+import { ENTERPRISE_DB_PATH_ENV } from '../../services/enterpriseDb';
+import { ENTERPRISE_DATA_DIR_ENV, writeTraceMetadata } from '../../services/traceMetadataStore';
+import {
+  getTraceProcessorLeaseStore,
+  setTraceProcessorLeaseStoreForTests,
+} from '../../services/traceProcessorLeaseStore';
+import { setTraceProcessorServiceForTests } from '../../services/traceProcessorService';
 import agentRoutes from '../agentRoutes';
 
 const originalApiKey = process.env.SMARTPERFETTO_API_KEY;
 const originalSsoTrustedHeaders = process.env.SMARTPERFETTO_SSO_TRUSTED_HEADERS;
+const originalEnterprise = process.env[ENTERPRISE_FEATURE_FLAG_ENV];
+const originalEnterpriseDbPath = process.env[ENTERPRISE_DB_PATH_ENV];
+const originalEnterpriseDataDir = process.env[ENTERPRISE_DATA_DIR_ENV];
+const originalUploadDir = process.env.UPLOAD_DIR;
 
 function makeApp(): express.Express {
   const app = express();
@@ -27,17 +42,38 @@ function viewerHeaders(req: request.Test): request.Test {
     .set('X-SmartPerfetto-SSO-Scopes', 'trace:read,report:read');
 }
 
-afterEach(() => {
+function analystHeaders(req: request.Test): request.Test {
+  return req
+    .set('X-SmartPerfetto-SSO-User-Id', 'analyst-user')
+    .set('X-SmartPerfetto-SSO-Email', 'analyst@example.test')
+    .set('X-SmartPerfetto-SSO-Tenant-Id', 'tenant-a')
+    .set('X-SmartPerfetto-SSO-Workspace-Id', 'workspace-a')
+    .set('X-SmartPerfetto-SSO-Roles', 'analyst')
+    .set('X-SmartPerfetto-SSO-Scopes', 'trace:read,trace:write,agent:run,report:read');
+}
+
+function restoreEnvValue(key: string, value: string | undefined): void {
+  if (value === undefined) {
+    delete process.env[key];
+  } else {
+    process.env[key] = value;
+  }
+}
+
+afterEach(async () => {
+  jest.restoreAllMocks();
+  setTraceProcessorServiceForTests(null);
+  setTraceProcessorLeaseStoreForTests(null);
   if (originalApiKey === undefined) {
     delete process.env.SMARTPERFETTO_API_KEY;
   } else {
     process.env.SMARTPERFETTO_API_KEY = originalApiKey;
   }
-  if (originalSsoTrustedHeaders === undefined) {
-    delete process.env.SMARTPERFETTO_SSO_TRUSTED_HEADERS;
-  } else {
-    process.env.SMARTPERFETTO_SSO_TRUSTED_HEADERS = originalSsoTrustedHeaders;
-  }
+  restoreEnvValue('SMARTPERFETTO_SSO_TRUSTED_HEADERS', originalSsoTrustedHeaders);
+  restoreEnvValue(ENTERPRISE_FEATURE_FLAG_ENV, originalEnterprise);
+  restoreEnvValue(ENTERPRISE_DB_PATH_ENV, originalEnterpriseDbPath);
+  restoreEnvValue(ENTERPRISE_DATA_DIR_ENV, originalEnterpriseDataDir);
+  restoreEnvValue('UPLOAD_DIR', originalUploadDir);
 });
 
 describe('agent route RBAC', () => {
@@ -52,4 +88,62 @@ describe('agent route RBAC', () => {
     expect(res.body.error).toBe('Forbidden');
     expect(res.body.details).toContain('agent:run');
   });
+
+  it('rejects analyze when the scoped trace processor lease is draining', async () => {
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'smartperfetto-agent-lease-'));
+    let leaseStore: ReturnType<typeof getTraceProcessorLeaseStore> | null = null;
+    try {
+      const traceId = 'trace-draining';
+      const tracePath = path.join(tmpDir, `${traceId}.trace`);
+      await fs.writeFile(tracePath, 'trace bytes');
+      delete process.env.SMARTPERFETTO_API_KEY;
+      process.env.SMARTPERFETTO_SSO_TRUSTED_HEADERS = 'true';
+      process.env[ENTERPRISE_FEATURE_FLAG_ENV] = 'true';
+      process.env[ENTERPRISE_DB_PATH_ENV] = path.join(tmpDir, 'enterprise.sqlite');
+      process.env[ENTERPRISE_DATA_DIR_ENV] = path.join(tmpDir, 'data');
+      process.env.UPLOAD_DIR = path.join(tmpDir, 'uploads');
+
+      await writeTraceMetadata({
+        id: traceId,
+        filename: `${traceId}.trace`,
+        size: 11,
+        uploadedAt: new Date().toISOString(),
+        status: 'ready',
+        path: tracePath,
+        tenantId: 'tenant-a',
+        workspaceId: 'workspace-a',
+        userId: 'analyst-user',
+      });
+      setTraceProcessorServiceForTests({
+        getOrLoadTrace: jest.fn(async () => ({
+          id: traceId,
+          filename: `${traceId}.trace`,
+          size: 11,
+          filePath: tracePath,
+          uploadTime: new Date(),
+          status: 'ready',
+        })),
+      } as any);
+
+      const scope = { tenantId: 'tenant-a', workspaceId: 'workspace-a', userId: 'analyst-user' };
+      leaseStore = getTraceProcessorLeaseStore();
+      const lease = leaseStore.acquireHolder(scope, traceId, {
+        holderType: 'manual_register',
+        holderRef: 'port:9100',
+      });
+      leaseStore.markStarting(scope, lease.id);
+      leaseStore.markReady(scope, lease.id);
+      leaseStore.beginDraining(scope, lease.id);
+
+      const res = await analystHeaders(request(makeApp()).post('/api/agent/v1/analyze'))
+        .send({ traceId, query: 'analyze this trace' });
+
+      expect(res.status).toBe(409);
+      expect(res.body.code).toBe('TRACE_PROCESSOR_LEASE_UNAVAILABLE');
+    } finally {
+      leaseStore?.close();
+      setTraceProcessorLeaseStoreForTests(null);
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  });
 });

backend/src/routes/__tests__/enterpriseTraceMetadataRoutes.test.ts

@@ -83,6 +83,31 @@ function readTraceAsset(traceId: string): TraceAssetRow | null {
   }
 }
 
+function readTraceProcessorLeases(traceId: string): Array<{
+  id: string;
+  state: string;
+  holder_type: string;
+  holder_ref: string;
+}> {
+  const db = openEnterpriseDb(dbPath);
+  try {
+    return db.prepare<unknown[], {
+      id: string;
+      state: string;
+      holder_type: string;
+      holder_ref: string;
+    }>(`
+      SELECT l.id, l.state, h.holder_type, h.holder_ref
+      FROM trace_processor_leases l
+      JOIN trace_processor_holders h ON h.lease_id = l.id
+      WHERE l.trace_id = ?
+      ORDER BY h.holder_type
+    `).all(traceId);
+  } finally {
+    db.close();
+  }
+}
+
 beforeEach(async () => {
   tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'smartperfetto-enterprise-trace-routes-'));
   dbPath = path.join(tmpDir, 'enterprise.sqlite');
@@ -158,6 +183,12 @@ describe('enterprise trace metadata routes', () => {
     expect(JSON.parse(row!.metadata_json)).toEqual(expect.objectContaining({
       filename: 'fixture.trace',
     }));
+    expect(readTraceProcessorLeases(traceId)).toEqual([
+      expect.objectContaining({
+        state: 'active',
+        holder_type: 'frontend_http_rpc',
+      }),
+    ]);
 
     const listRes = await ssoHeaders(request(app).get('/api/traces'));
     expect(listRes.status).toBe(200);

backend/src/routes/agentRoutes.ts

@@ -57,7 +57,11 @@ import { FileSystemSceneReportStore } from '../services/sceneReport/sceneReportS
 import { SceneReportMemoryCache } from '../services/sceneReport/sceneReportMemoryCache';
 import { computeTraceContentHash } from '../agent/scene/traceHash';
 import { probeTraceDuration } from '../agent/scene/sceneTraceDurationProbe';
-import { sceneStoryConfig } from '../config';
+import { resolveFeatureConfig, sceneStoryConfig } from '../config';
+import {
+  getTraceProcessorLeaseStore,
+  type TraceProcessorLeaseRecord,
+} from '../services/traceProcessorLeaseStore';
 import { registerAgentLogsRoutes } from './agentLogsRoutes';
 import { registerAgentQuickSceneRoutes } from './agentQuickSceneRoutes';
 import { registerAgentReportRoutes } from './agentReportRoutes';
@@ -142,6 +146,37 @@ function buildRunId(sessionId: string, sequence: number): string {
   return `run-${sessionId}-${sequence}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
 }
 
+function enterpriseLeasesEnabled(): boolean {
+  return resolveFeatureConfig().enterprise;
+}
+
+function leaseScopeFromRequestContext(context: RequestContext) {
+  return {
+    tenantId: context.tenantId,
+    workspaceId: context.workspaceId,
+    userId: context.userId,
+  };
+}
+
+function leaseScopeFromSession(session: AnalysisSession) {
+  if (!session.tenantId || !session.workspaceId) return null;
+  return {
+    tenantId: session.tenantId,
+    workspaceId: session.workspaceId,
+    userId: session.userId,
+  };
+}
+
+function markLeaseReadyIfNew(
+  lease: TraceProcessorLeaseRecord,
+  scope: { tenantId: string; workspaceId: string; userId?: string },
+): TraceProcessorLeaseRecord {
+  if (lease.state !== 'pending') return lease;
+  const store = getTraceProcessorLeaseStore();
+  const starting = store.markStarting(scope, lease.id);
+  return store.markReady(scope, starting.id);
+}
+
 function buildSessionObservability(
   session: AnalysisSession
 ): { runId: string; requestId: string; runSequence: number; status: string } | undefined {
@@ -931,6 +966,38 @@ async function handleAnalyzeRequest(
       runSequence: runContext.sequence,
     });
 
+    let agentRunLease: TraceProcessorLeaseRecord | null = null;
+    if (enterpriseLeasesEnabled()) {
+      try {
+        const scope = leaseScopeFromRequestContext(requestContext);
+        agentRunLease = getTraceProcessorLeaseStore().acquireHolder(
+          scope,
+          traceId,
+          {
+            holderType: 'agent_run',
+            holderRef: runContext.runId,
+            runId: runContext.runId,
+            sessionId,
+            metadata: {
+              requestId: runContext.requestId,
+              runSequence: runContext.sequence,
+            },
+          },
+        );
+        agentRunLease = markLeaseReadyIfNew(agentRunLease, scope);
+      } catch (leaseError: any) {
+        sessionForRun.status = 'failed';
+        sessionForRun.error = leaseError.message;
+        markSessionRunStatus(sessionForRun, 'failed', leaseError.message);
+        res.status(409).json({
+          success: false,
+          code: 'TRACE_PROCESSOR_LEASE_UNAVAILABLE',
+          error: leaseError.message,
+        });
+        return;
+      }
+    }
+
     // Validate traceContext — must be array of objects with columns/rows
     const traceContext = Array.isArray(rawTraceContext)
       ? rawTraceContext.filter(
@@ -960,6 +1027,18 @@ async function handleAnalyzeRequest(
           timestamp: Date.now(),
         });
       }
+    }).finally(() => {
+      if (!agentRunLease) return;
+      try {
+        getTraceProcessorLeaseStore().releaseHolder(
+          leaseScopeFromRequestContext(requestContext),
+          agentRunLease.id,
+          'agent_run',
+          runContext.runId,
+        );
+      } catch (releaseError: any) {
+        console.warn(`[AgentRoutes] Failed to release agent_run lease ${agentRunLease.id}: ${releaseError.message}`);
+      }
     });
 
     res.json({
@@ -974,6 +1053,8 @@ async function handleAnalyzeRequest(
       providerSnapshotChanged: preparedSession?.providerSnapshotChanged || undefined,
       architecture: 'agent-driven',
       runId: runContext.runId,
+      leaseId: agentRunLease?.id,
+      leaseState: agentRunLease?.state,
       requestId: runContext.requestId,
       runSequence: runContext.sequence,
       observability: {
@@ -4043,7 +4124,27 @@ function sendAgentDrivenResult(res: express.Response, session: AnalysisSession)
   // Generate HTML report
   let reportUrl: string | undefined;
   let reportError: string | undefined;
+  const reportId = `agent-report-${session.sessionId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  let reportLease: TraceProcessorLeaseRecord | null = null;
   try {
+    if (enterpriseLeasesEnabled()) {
+      const scope = leaseScopeFromSession(session);
+      if (scope) {
+        reportLease = getTraceProcessorLeaseStore().acquireHolder(
+          scope,
+          session.traceId,
+          {
+            holderType: 'report_generation',
+            holderRef: reportId,
+            reportId,
+            sessionId: session.sessionId,
+            runId: session.lastRun?.runId || session.activeRun?.runId,
+          },
+        );
+        reportLease = markLeaseReadyIfNew(reportLease, scope);
+      }
+    }
+
     const generator = getHTMLReportGenerator();
     // Report assembly (cumulative findings dedup, empty-conclusion fallback,
     // snapshot-first analysisNotes/Plan/Flags) lives in the shared builder so
@@ -4073,7 +4174,6 @@ function sendAgentDrivenResult(res: express.Response, session: AnalysisSession)
     const html = generator.generateAgentDrivenHTML(reportData);
 
     // Store report
-    const reportId = `agent-report-${session.sessionId}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
     persistReport(reportId, {
       html,
       generatedAt: Date.now(),
@@ -4097,6 +4197,22 @@ function sendAgentDrivenResult(res: express.Response, session: AnalysisSession)
       resultConfidence: result?.confidence,
       resultRounds: result?.rounds,
     });
+  } finally {
+    if (reportLease) {
+      const scope = leaseScopeFromSession(session);
+      if (scope) {
+        try {
+          getTraceProcessorLeaseStore().releaseHolder(
+            scope,
+            reportLease.id,
+            'report_generation',
+            reportId,
+          );
+        } catch (releaseError: any) {
+          console.warn(`[AgentRoutes] Failed to release report_generation lease ${reportLease.id}: ${releaseError.message}`);
+        }
+      }
+    }
   }
 
   // Send analysis_completed event with full result. Keep it replayable so a