diff --git a/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts b/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts index c2070cb6..4814bc0e 100644 --- a/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts +++ b/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts @@ -31,7 +31,7 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { await fs.rm(tempRoot, { recursive: true, force: true }); }); - test('covers current D1/D2/D4/D5/D6/D7/D8/D9 isolation invariants without invoking a real provider', async () => { + test('covers current D1/D2/D4/D5/D6/D7/D8/D9/D10 isolation invariants without invoking a real provider', async () => { const tracePath = path.join(tempRoot, 'fixture.pftrace'); const uploadRoot = path.join(tempRoot, 'uploads'); const outputPath = path.join(tempRoot, 'report.json'); @@ -54,6 +54,7 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { D7: true, D8: true, D9: true, + D10: true, }); expect(Object.values(report.scenarios.D1.checks)).toEqual(expect.arrayContaining([true])); expect(Object.values(report.scenarios.D1.checks).every(Boolean)).toBe(true); @@ -64,6 +65,7 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { expect(Object.values(report.scenarios.D7.checks).every(Boolean)).toBe(true); expect(Object.values(report.scenarios.D8.checks).every(Boolean)).toBe(true); expect(Object.values(report.scenarios.D9.checks).every(Boolean)).toBe(true); + expect(Object.values(report.scenarios.D10.checks).every(Boolean)).toBe(true); expect(report.coverageLimitations.join('\n')).toContain('TraceProcessorLease'); expect(report.scenarios.D6.details).toEqual(expect.objectContaining({ reportUrl: expect.stringMatching(/^\/api\/reports\//), @@ -79,6 +81,10 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { expect.arrayContaining([expect.stringMatching(/^run-/), 'running']), ]), })); + expect(report.scenarios.D10.details).toEqual(expect.objectContaining({ + candidateLeaseCount: 0, + activeHolderCountAfterShare: 2, + })); expect(process.env.UPLOAD_DIR).toBe(previousUploadDir); const written = JSON.parse(await fs.readFile(outputPath, 'utf8')) as EnterpriseWindowRegressionReport; @@ -87,6 +93,6 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { const traceFiles = (await fs.readdir(path.join(uploadRoot, 'traces'))) .filter(file => file.endsWith('.trace')); - expect(traceFiles).toHaveLength(12); + expect(traceFiles).toHaveLength(14); }); }); diff --git a/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts b/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts index a4b7ce43..25e142a5 100644 --- a/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts +++ b/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts @@ -17,10 +17,17 @@ import { writeTraceMetadata, } from '../services/traceMetadataStore'; import { TraceProcessorLeaseStore } from '../services/traceProcessorLeaseStore'; +import { + TP_ADMISSION_CONTROL_ENV, + TP_ESTIMATE_MULTIPLIER_ENV, + TP_MIN_ESTIMATE_BYTES_ENV, + TP_RAM_BUDGET_BYTES_ENV, + decideTraceProcessorAdmission, +} from '../services/traceProcessorRamBudget'; import { ownerFieldsFromContext } from '../services/resourceOwnership'; import type { RequestContext } from '../middleware/auth'; -type ScenarioName = 'D1' | 'D2' | 'D4' | 'D5' | 'D6' | 'D7' | 'D8' | 'D9'; +type ScenarioName = 'D1' | 'D2' | 'D4' | 'D5' | 'D6' | 'D7' | 'D8' | 'D9' | 'D10'; interface VerifyOptions { tracePath?: string; @@ -1361,6 +1368,122 @@ async function scenarioD9( } } +async function scenarioD10( + db: Database.Database, + tracePath: string, + userAWindow1: WindowContext, +): Promise { + const mib = 1024 * 1024; + const activeUpload = await simulateUpload(db, userAWindow1, tracePath, 'd10-active-window.pftrace'); + const candidateUpload = await simulateUpload(db, userAWindow1, tracePath, 'd10-rejected-new-lease.pftrace'); + const store = new TraceProcessorLeaseStore(db); + const scope = leaseScope(userAWindow1); + const windowId = userAWindow1.context.windowId ?? userAWindow1.label; + const activeRssBytes = 448 * mib; + const budgetBytes = 512 * mib; + const minEstimateBytes = 128 * mib; + + let activeLease = store.acquireHolder(scope, activeUpload.traceId, { + holderType: 'frontend_http_rpc', + holderRef: windowId, + windowId, + frontendVisibility: 'visible', + metadata: { scenario: 'D10' }, + }); + store.markStarting(scope, activeLease.id); + activeLease = store.markReady(scope, activeLease.id); + activeLease = store.recordRss(scope, activeLease.id, activeRssBytes); + + const admissionEnv: NodeJS.ProcessEnv = { + ...process.env, + [TP_ADMISSION_CONTROL_ENV]: 'true', + [TP_RAM_BUDGET_BYTES_ENV]: String(budgetBytes), + [TP_ESTIMATE_MULTIPLIER_ENV]: '1', + [TP_MIN_ESTIMATE_BYTES_ENV]: String(minEstimateBytes), + }; + const processorSamples = store.listLeases(scope, { + states: ['pending', 'starting', 'ready', 'idle', 'active', 'crashed', 'restarting'], + }).map(lease => ({ + traceId: lease.traceId, + rssBytes: lease.rssBytes, + })); + const decision = decideTraceProcessorAdmission({ + traceId: candidateUpload.traceId, + traceSizeBytes: candidateUpload.sizeBytes, + processors: processorSamples, + env: admissionEnv, + }); + + let newLeaseAttempted = false; + if (decision.admitted) { + newLeaseAttempted = true; + store.acquireHolder(scope, candidateUpload.traceId, { + holderType: 'agent_run', + holderRef: 'run-d10-candidate', + metadata: { scenario: 'D10' }, + }); + } + + const activeLeaseAfterRejectedAdmission = store.getLeaseById(scope, activeLease.id); + const candidateLeases = store.listLeases(scope, { traceId: candidateUpload.traceId }); + const activeHolderCountBeforeShare = activeLeaseAfterRejectedAdmission?.holderCount ?? 0; + const sharedLease = store.acquireHolder(scope, activeUpload.traceId, { + holderType: 'agent_run', + holderRef: 'run-d10-existing-trace', + runId: 'run-d10-existing-trace', + metadata: { scenario: 'D10', reuse: 'existing-lease' }, + }); + const activeTraceLeases = store.listLeases(scope, { traceId: activeUpload.traceId }); + const activeTraceAssetPresent = scalar( + db, + 'SELECT COUNT(*) AS value FROM trace_assets WHERE id = ?', + [activeUpload.traceId], + ) === 1; + const candidateTraceAssetPresent = scalar( + db, + 'SELECT COUNT(*) AS value FROM trace_assets WHERE id = ?', + [candidateUpload.traceId], + ) === 1; + + const checks = { + admissionRejectsNewLeaseNearRamBudget: !decision.admitted + && decision.estimatedRssBytes === minEstimateBytes + && decision.stats.availableForNewLeaseBytes === budgetBytes - activeRssBytes + && Boolean(decision.reason?.includes('exceeds available budget')), + rejectedAdmissionDoesNotCreateCandidateLease: !newLeaseAttempted + && candidateLeases.length === 0 + && candidateTraceAssetPresent + && fs.existsSync(candidateUpload.localPath), + activeLeaseSurvivesRejectedAdmission: activeLeaseAfterRejectedAdmission?.id === activeLease.id + && activeLeaseAfterRejectedAdmission.state === 'active' + && activeLeaseAfterRejectedAdmission.rssBytes === activeRssBytes + && activeLeaseAfterRejectedAdmission.holderCount === 1, + existingTraceCanReuseSharedLeaseWithoutNewProcessor: sharedLease.id === activeLease.id + && sharedLease.holderCount === activeHolderCountBeforeShare + 1 + && activeTraceLeases.length === 1, + noOomStyleCleanupOfExistingWindow: activeTraceAssetPresent + && fs.existsSync(activeUpload.localPath) + && store.getLeaseById(scope, activeLease.id)?.state === 'active', + }; + + return { + checks, + details: { + activeTraceId: activeUpload.traceId, + candidateTraceId: candidateUpload.traceId, + activeLeaseId: activeLease.id, + activeRssBytes, + budgetBytes, + estimatedRssBytes: decision.estimatedRssBytes, + availableForNewLeaseBytes: decision.stats.availableForNewLeaseBytes, + admissionReason: decision.reason, + activeHolderCountBeforeShare, + activeHolderCountAfterShare: sharedLease.holderCount, + candidateLeaseCount: candidateLeases.length, + }, + }; +} + function allChecksPassed(report: EnterpriseWindowRegressionReport): boolean { return Object.values(report.scenarios).every(scenario => Object.values(scenario.checks).every(Boolean), @@ -1428,6 +1551,7 @@ export async function runEnterpriseWindowRegression( D7: await scenarioD7(db, tracePath, windows.userAWindow1), D8: await scenarioD8(db, tracePath, windows.userAWindow1), D9: await scenarioD9(tracePath, windows.userAWindow1), + D10: await scenarioD10(db, tracePath, windows.userAWindow1), }; const report: EnterpriseWindowRegressionReport = { timestamp: new Date().toISOString(), @@ -1441,6 +1565,7 @@ export async function runEnterpriseWindowRegression( D7: scenarioPassed(scenarios.D7), D8: scenarioPassed(scenarios.D8), D9: scenarioPassed(scenarios.D9), + D10: scenarioPassed(scenarios.D10), }, uploadRoot, tracePath, @@ -1454,7 +1579,8 @@ export async function runEnterpriseWindowRegression( 'D7 covers running run, active lease, report_generation holder, and draining rejection invariants; actual route blocking is covered by enterpriseTraceMetadataRoutes tests.', 'D8 covers the DB ProviderSnapshot pin/hash-mismatch invariant in the enterprise window regression; AgentAnalyzeSessionService tests cover actual in-memory and persisted SDK session non-reuse.', 'D9 covers file-backed DB close/reopen recovery for trace metadata, run states, and AgentEvent replay; enterpriseRestartPersistence tests cover the route-level restart recovery path.', - 'Production backend proxy and queue behavior remain future §0.7 D1/D2/D4/D5/D6/D7/D8/D9 final-acceptance work against a live browser and trace_processor_shell.', + 'D10 covers RAM-admission rejection without creating a new lease or cleaning up existing active holders; TraceProcessorFactory tests cover pre-spawn rejection before a real processor starts.', + 'Production backend proxy and queue behavior remain future §0.7 D1/D2/D3/D4/D5/D6/D7/D8/D9/D10 final-acceptance work against a live browser and trace_processor_shell.', ], }; report.passed = allChecksPassed(report); diff --git a/docs/features/enterprise-multi-tenant/README.md b/docs/features/enterprise-multi-tenant/README.md index fdfebc50..d64d8100 100644 --- a/docs/features/enterprise-multi-tenant/README.md +++ b/docs/features/enterprise-multi-tenant/README.md @@ -114,7 +114,7 @@ - [x] D7 手动 cleanup / delete → running run / active lease / 正在生成的 report 被 draining 保护 - [x] D8 Provider 配置在 session 中途变更 → resume 校验 ProviderSnapshot hash,不复用错误 SDK session - [x] D9 后端进程重启 → pending/running/terminal run 状态、events、trace metadata 可恢复或转 failed -- [ ] D10 机器内存接近上限 → admission 拒绝新 lease,不通过 OOM 杀已有窗口 +- [x] D10 机器内存接近上限 → admission 拒绝新 lease,不通过 OOM 杀已有窗口 ### 0.8 §19 总验收 - [ ] 50 在线用户 + 5-15 running run + pending 排队稳定