diff --git a/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts b/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts index b122b1a7..d853aa79 100644 --- a/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts +++ b/backend/src/scripts/__tests__/verifyEnterpriseMultiTenantWindows.test.ts @@ -31,7 +31,7 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { await fs.rm(tempRoot, { recursive: true, force: true }); }); - test('covers current D1/D2 isolation invariants without invoking a real provider', async () => { + test('covers current D1/D2/D5 isolation invariants without invoking a real provider', async () => { const tracePath = path.join(tempRoot, 'fixture.pftrace'); const uploadRoot = path.join(tempRoot, 'uploads'); const outputPath = path.join(tempRoot, 'report.json'); @@ -45,10 +45,11 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { }); expect(report.passed).toBe(true); - expect(report.checks).toEqual({ D1: true, D2: true }); + expect(report.checks).toEqual({ D1: true, D2: true, D5: true }); expect(Object.values(report.scenarios.D1.checks)).toEqual(expect.arrayContaining([true])); expect(Object.values(report.scenarios.D1.checks).every(Boolean)).toBe(true); expect(Object.values(report.scenarios.D2.checks).every(Boolean)).toBe(true); + expect(Object.values(report.scenarios.D5.checks).every(Boolean)).toBe(true); expect(report.coverageLimitations.join('\n')).toContain('TraceProcessorLease'); expect(process.env.UPLOAD_DIR).toBe(previousUploadDir); @@ -58,6 +59,6 @@ describe('verifyEnterpriseMultiTenantWindows script', () => { const traceFiles = (await fs.readdir(path.join(uploadRoot, 'traces'))) .filter(file => file.endsWith('.trace')); - expect(traceFiles).toHaveLength(6); + expect(traceFiles).toHaveLength(7); }); }); diff --git a/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts b/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts index b05c4ae3..24712bba 100644 --- a/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts +++ b/backend/src/scripts/verifyEnterpriseMultiTenantWindows.ts @@ -9,16 +9,18 @@ import os from 'os'; import path from 'path'; import Database from 'better-sqlite3'; import { applyEnterpriseMinimalSchema } from '../services/enterpriseSchema'; +import type { EnterpriseRepositoryScope } from '../services/enterpriseRepository'; import { getTracesDir, listTraceMetadata, readTraceMetadataForContext, writeTraceMetadata, } from '../services/traceMetadataStore'; +import { TraceProcessorLeaseStore } from '../services/traceProcessorLeaseStore'; import { ownerFieldsFromContext } from '../services/resourceOwnership'; import type { RequestContext } from '../middleware/auth'; -type ScenarioName = 'D1' | 'D2'; +type ScenarioName = 'D1' | 'D2' | 'D5'; interface VerifyOptions { tracePath?: string; @@ -170,6 +172,14 @@ function context(label: string, input: { }; } +function leaseScope(wc: WindowContext): EnterpriseRepositoryScope { + return { + tenantId: wc.context.tenantId, + workspaceId: wc.context.workspaceId, + userId: wc.context.userId, + }; +} + function seedIdentity(db: Database.Database, wc: WindowContext): void { const now = Date.now(); const { tenantId, workspaceId, userId } = wc.context; @@ -502,6 +512,96 @@ async function scenarioD2( }; } +async function scenarioD5( + db: Database.Database, + tracePath: string, + userAWindow1: WindowContext, +): Promise { + const upload = await simulateUpload(db, userAWindow1, tracePath, 'd5-user-a-sleep.pftrace'); + const store = new TraceProcessorLeaseStore(db); + const scope = leaseScope(userAWindow1); + const holderRef = userAWindow1.context.windowId ?? userAWindow1.label; + const startedAt = 1_777_000_000_000; + const offlineAt = startedAt + 30_000; + const insideGraceAt = offlineAt + 30 * 60 * 1000 - 1; + const afterGraceAt = offlineAt + 30 * 60 * 1000 + 1; + const recoveredAt = afterGraceAt + 1_000; + + let lease = store.acquireHolder(scope, upload.traceId, { + holderType: 'frontend_http_rpc', + holderRef, + windowId: holderRef, + frontendVisibility: 'visible', + metadata: { scenario: 'D5' }, + }, { now: startedAt }); + store.markStarting(scope, lease.id); + lease = store.markReady(scope, lease.id); + + lease = store.acquireHolderForLease(scope, lease.id, { + holderType: 'frontend_http_rpc', + holderRef, + windowId: holderRef, + frontendVisibility: 'offline', + metadata: { + heartbeat: 'frontend', + scenario: 'D5', + }, + }, { now: offlineAt }); + const offlineHolder = lease.holders.find(holder => holder.holderRef === holderRef); + + const insideGraceSweep = store.sweepExpired(insideGraceAt); + const insideGraceLease = store.getLeaseById(scope, lease.id); + + const afterGraceSweep = store.sweepExpired(afterGraceAt); + const afterGraceLease = store.getLeaseById(scope, lease.id); + + const recoveredLease = store.acquireHolderForLease(scope, lease.id, { + holderType: 'frontend_http_rpc', + holderRef, + windowId: holderRef, + frontendVisibility: 'visible', + metadata: { + heartbeat: 'frontend', + scenario: 'D5', + recovery: 'pageshow', + }, + }, { now: recoveredAt }); + const recoveredHolder = recoveredLease.holders.find(holder => holder.holderRef === holderRef); + + const checks = { + offlineHeartbeatKeepsThirtyMinuteGrace: (offlineHolder?.expiresAt ?? 0) >= insideGraceAt, + leaseStaysActiveInsideOfflineGrace: insideGraceSweep.holdersRemoved === 0 + && insideGraceLease?.state === 'active' + && insideGraceLease.holderCount === 1, + staleOfflineHolderDoesNotReleaseLease: afterGraceSweep.holdersRemoved === 1 + && afterGraceLease?.state === 'idle' + && afterGraceLease.holderCount === 0, + pageshowHeartbeatReacquiresSameLease: recoveredLease.id === lease.id + && recoveredLease.state === 'active' + && recoveredLease.holderCount === 1 + && recoveredHolder?.metadata?.frontendVisibility === 'visible' + && recoveredHolder.expiresAt !== null + && recoveredHolder.expiresAt >= recoveredAt + 90_000, + }; + + return { + checks, + details: { + traceId: upload.traceId, + leaseId: lease.id, + offlineHolderExpiresAt: offlineHolder?.expiresAt ?? null, + insideGraceAt, + afterGraceAt, + recoveredAt, + insideGraceSweep, + afterGraceSweep, + afterGraceLeaseState: afterGraceLease?.state ?? null, + recoveredLeaseState: recoveredLease.state, + recoveredHolder, + }, + }; +} + function allChecksPassed(report: EnterpriseWindowRegressionReport): boolean { return Object.values(report.scenarios).every(scenario => Object.values(scenario.checks).every(Boolean), @@ -563,6 +663,7 @@ export async function runEnterpriseWindowRegression( windows.userBWindow1, input.longSqlMs ?? 100, ), + D5: await scenarioD5(db, tracePath, windows.userAWindow1), }; const report: EnterpriseWindowRegressionReport = { timestamp: new Date().toISOString(), @@ -570,6 +671,7 @@ export async function runEnterpriseWindowRegression( checks: { D1: scenarioPassed(scenarios.D1), D2: scenarioPassed(scenarios.D2), + D5: scenarioPassed(scenarios.D5), }, uploadRoot, tracePath, @@ -577,7 +679,8 @@ export async function runEnterpriseWindowRegression( coverageLimitations: [ 'D1 covers same-name trace isolation across three users and two windows at the trace metadata, TraceAsset, workspace RBAC, and analysis session/run schema layers.', 'D2 covers a deterministic long-SQL window at the run/event metadata layer without invoking a real LLM provider.', - 'Production TraceProcessorLease holder/state assertions are covered by the §0.4.4 lease store and route tests; backend proxy and queue behavior remain future §0.7 D1/D2 final-acceptance work.', + 'D5 covers TraceProcessorLease holder grace and pageshow-style reacquire semantics after offline heartbeat expiry; frontend stale-lease reload signaling is covered by HttpRpcEngine unit tests.', + 'Production backend proxy and queue behavior remain future §0.7 D1/D2/D5 final-acceptance work against a live browser and trace_processor_shell.', ], }; report.passed = allChecksPassed(report); diff --git a/docs/features/enterprise-multi-tenant/README.md b/docs/features/enterprise-multi-tenant/README.md index 82ec2118..62b6431c 100644 --- a/docs/features/enterprise-multi-tenant/README.md +++ b/docs/features/enterprise-multi-tenant/README.md @@ -109,7 +109,7 @@ - [ ] D2 A 长 SQL 中，B 上传并分析另一个 trace → A 的 SSE 不断、A lease 不被 destroy、B 能排队或运行 - [ ] D3 A 前端 timeline，B 跑 full agent → A WebSocket 走 lease；P0 不被 P2 长任务无限阻塞 - [ ] D4 trace_processor_shell crash → leaseId 稳定；前端不持有旧 port；supervisor 单点重启 -- [ ] D5 浏览器断网 / 休眠 30 分钟后恢复 → frontend grace 生效；reacquire lease 或自动 reload +- [x] D5 浏览器断网 / 休眠 30 分钟后恢复 → frontend grace 生效；reacquire lease 或自动 reload - [ ] D6 SSE 在 conclusion 后、analysis_completed 前断开 → AgentEvent replay 能补回 reportUrl - [ ] D7 手动 cleanup / delete → running run / active lease / 正在生成的 report 被 draining 保护 - [ ] D8 Provider 配置在 session 中途变更 → resume 校验 ProviderSnapshot hash，不复用错误 SDK session diff --git a/frontend/index.html b/frontend/index.html index a62d0e45..b03f9492 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -9,7 +9,7 @@ - +