diff --git a/backend/package.json b/backend/package.json index 2252a7d3..3406d8a2 100644 --- a/backend/package.json +++ b/backend/package.json @@ -41,7 +41,7 @@ "prepack": "npm run build", "typecheck": "tsc --noEmit", "test": "jest", - "test:core": "jest --runInBand --forceExit src/agent/communication/__tests__/agentMessageBus.test.ts src/agent/core/executors/__tests__/strategyExecutor.test.ts src/agent/core/executors/__tests__/hypothesisExecutor.test.ts src/agent/context/__tests__/enhancedSessionContext.test.ts src/tests/adbTools.test.ts src/services/__tests__/sessionLogger.test.ts src/services/__tests__/traceAnalysisSkillConfig.test.ts src/agent/agents/domain/__tests__/registry.test.ts src/agentv3/__tests__/sqlIncludeInjector.test.ts src/agentv3/__tests__/analysisPatternMemory.test.ts src/agentv3/__tests__/claudeRuntimeRuntimeSnapshots.test.ts src/middleware/__tests__/auth.test.ts src/services/__tests__/rbac.test.ts src/routes/__tests__/agentRoutesRbac.test.ts src/routes/__tests__/ownerGuardRoutes.test.ts src/routes/__tests__/requestContextRouteCoverage.test.ts src/middleware/__tests__/legacyApiCompatibility.test.ts src/services/__tests__/enterpriseDb.test.ts src/services/__tests__/enterpriseSchema.test.ts src/services/__tests__/enterpriseRepository.test.ts src/services/__tests__/enterpriseKnowledgeScope.test.ts src/services/__tests__/runtimeSnapshotStore.test.ts src/services/providerManager/__tests__/enterpriseProviderStore.test.ts src/routes/__tests__/enterpriseTraceMetadataRoutes.test.ts src/routes/__tests__/enterpriseReportRoutes.test.ts", + "test:core": "jest --runInBand --forceExit src/agent/communication/__tests__/agentMessageBus.test.ts src/agent/core/executors/__tests__/strategyExecutor.test.ts src/agent/core/executors/__tests__/hypothesisExecutor.test.ts src/agent/context/__tests__/enhancedSessionContext.test.ts src/tests/adbTools.test.ts src/services/__tests__/sessionLogger.test.ts src/services/__tests__/traceAnalysisSkillConfig.test.ts src/agent/agents/domain/__tests__/registry.test.ts src/agentv3/__tests__/sqlIncludeInjector.test.ts src/agentv3/__tests__/analysisPatternMemory.test.ts src/agentv3/__tests__/claudeRuntimeRuntimeSnapshots.test.ts src/middleware/__tests__/auth.test.ts src/services/__tests__/rbac.test.ts src/routes/__tests__/agentRoutesRbac.test.ts src/routes/__tests__/ownerGuardRoutes.test.ts src/routes/__tests__/requestContextRouteCoverage.test.ts src/middleware/__tests__/legacyApiCompatibility.test.ts src/services/__tests__/enterpriseDb.test.ts src/services/__tests__/enterpriseSchema.test.ts src/services/__tests__/enterpriseRepository.test.ts src/services/__tests__/enterpriseKnowledgeScope.test.ts src/services/__tests__/enterpriseMigration.test.ts src/services/__tests__/runtimeSnapshotStore.test.ts src/services/providerManager/__tests__/enterpriseProviderStore.test.ts src/routes/__tests__/enterpriseTraceMetadataRoutes.test.ts src/routes/__tests__/enterpriseReportRoutes.test.ts", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "test:unit": "jest --testPathPatterns=src/tests", @@ -56,6 +56,7 @@ "test:all": "npm run test:gate", "trace-processor:ensure": "node scripts/ensure-trace-processor.cjs", "verify:pr": "npm run validate:skills && npm run validate:strategies && npm run typecheck && npm run build && node scripts/check-cli-pack.cjs && npm run test:core && npm run trace-processor:ensure && npm run test:scene-trace-regression", + "enterprise:migration": "tsx src/scripts/enterpriseMigrationSnapshot.ts", "skill:validate": "tsx src/cli/index.ts validate", "validate:skills": "tsx src/cli/index.ts validate --contracts --all", "validate:strategies": "tsx src/cli/index.ts validate --strategies", diff --git a/backend/src/agentv3/analysisPatternMemory.ts b/backend/src/agentv3/analysisPatternMemory.ts index 988da039..072cddab 100644 --- a/backend/src/agentv3/analysisPatternMemory.ts +++ b/backend/src/agentv3/analysisPatternMemory.ts @@ -265,7 +265,7 @@ function patternMatchesKnowledgeScope( pattern: {provenance?: PatternProvenance}, scope: KnowledgeScope | undefined, ): boolean { - if (!enterpriseKnowledgeStoreEnabled()) return true; + if (!enterpriseKnowledgeStoreEnabled() && !scope) return true; const resolved = resolveKnowledgeScope(scope); return ( pattern.provenance?.sourceTenantId === resolved.tenantId && diff --git a/backend/src/agentv3/claudeMcpServer.ts b/backend/src/agentv3/claudeMcpServer.ts index ffdf5267..2eab3639 100644 --- a/backend/src/agentv3/claudeMcpServer.ts +++ b/backend/src/agentv3/claudeMcpServer.ts @@ -179,7 +179,7 @@ const REASONING_NUDGE_EN = '\n\n[REFLECT] Before the next action: what is the ke export const MIN_PHASE_SUMMARY_CHARS = 15; function sqlErrorLogFile(scope?: KnowledgeScope): string { - if (!enterpriseKnowledgeStoreEnabled()) { + if (!enterpriseKnowledgeStoreEnabled() && !scope) { return path.join(SQL_ERROR_LOG_DIR, 'error_fix_pairs.json'); } const resolved = resolveKnowledgeScope(scope); diff --git a/backend/src/agentv3/claudeRuntime.ts b/backend/src/agentv3/claudeRuntime.ts index 8d428bcd..4951b784 100644 --- a/backend/src/agentv3/claudeRuntime.ts +++ b/backend/src/agentv3/claudeRuntime.ts @@ -83,13 +83,17 @@ import { applyCapturedEntities, } from '../agent/core/entityCapture'; import { DEFAULT_OUTPUT_LANGUAGE, localize } from './outputLanguage'; -import { resolveFeatureConfig } from '../config'; import { deleteClaudeSessionMapRuntimeSnapshots, loadClaudeSessionMapFromRuntimeSnapshots, saveClaudeSessionMapToRuntimeSnapshots, type ClaudeSessionMapRuntimeEntry, } from '../services/runtimeSnapshotStore'; +import { + enterpriseDbWritesEnabled, + legacyFilesystemReadAuthorityEnabled, + legacyFilesystemWritesEnabled, +} from '../services/enterpriseMigration'; import type { ProviderScope } from '../services/providerManager'; import type { KnowledgeScope } from '../services/scopedKnowledgeStore'; @@ -104,8 +108,12 @@ interface SessionMapEntry { updatedAt: number; } -function enterpriseSessionMapStoreEnabled(): boolean { - return resolveFeatureConfig(process.env).enterprise; +function enterpriseSessionMapDbWritesEnabled(): boolean { + return enterpriseDbWritesEnabled(); +} + +function legacySessionMapWritesEnabled(): boolean { + return legacyFilesystemWritesEnabled(); } function loadPersistedSessionMap(): Map { @@ -130,22 +138,16 @@ function loadPersistedSessionMap(): Map { } function loadSessionMapForCurrentMode(): Map { - if (!enterpriseSessionMapStoreEnabled()) { + if (legacyFilesystemReadAuthorityEnabled()) { return loadPersistedSessionMap(); } try { - const dbMap = loadClaudeSessionMapFromRuntimeSnapshots(SESSION_MAP_MAX_AGE_MS); - if (dbMap.size > 0) return dbMap; + return loadClaudeSessionMapFromRuntimeSnapshots(SESSION_MAP_MAX_AGE_MS); } catch (err) { console.warn('[ClaudeRuntime] Failed to load runtime_snapshots session map:', (err as Error).message); } - - const legacyMap = loadPersistedSessionMap(); - if (legacyMap.size > 0) { - console.warn('[ClaudeRuntime] Loaded legacy logs/claude_session_map.json for migration; future enterprise writes use runtime_snapshots'); - } - return legacyMap; + return new Map(); } function providerScopeFromOptions(options: AnalysisOptions): ProviderScope | undefined { @@ -420,11 +422,12 @@ export class ClaudeRuntime extends EventEmitter implements IOrchestrator { entry: ClaudeSessionMapRuntimeEntry, options: AnalysisOptions, ): void { - if (!enterpriseSessionMapStoreEnabled()) { + if (legacySessionMapWritesEnabled()) { savePersistedSessionMap(this.sessionMap); - return; } + if (!enterpriseSessionMapDbWritesEnabled()) return; + if (!options.tenantId || !options.workspaceId) { console.warn('[ClaudeRuntime] Enterprise session map persistence skipped: missing tenant/workspace scope'); return; @@ -600,7 +603,7 @@ export class ClaudeRuntime extends EventEmitter implements IOrchestrator { && (Date.now() - (existingSessionMapEntry.updatedAt || 0) < SDK_SESSION_FRESHNESS_MS) ? existingSessionMapEntry.sdkSessionId : undefined; - if (existingSessionMapEntry && existingSdkSessionId && enterpriseSessionMapStoreEnabled()) { + if (existingSessionMapEntry && existingSdkSessionId && enterpriseSessionMapDbWritesEnabled()) { this.persistSessionMapEntry(sessionId, traceId, ctx.sessionMapKey, existingSessionMapEntry, options); } @@ -1716,7 +1719,7 @@ export class ClaudeRuntime extends EventEmitter implements IOrchestrator { && (Date.now() - (sessionMapEntry.updatedAt || 0) < SDK_SESSION_FRESHNESS_MS) ? sessionMapEntry.sdkSessionId : undefined; - if (sessionMapEntry && existingSdkSessionId && enterpriseSessionMapStoreEnabled()) { + if (sessionMapEntry && existingSdkSessionId && enterpriseSessionMapDbWritesEnabled()) { this.persistSessionMapEntry(sessionId, traceId, sessionMapKey, sessionMapEntry, options); } const sdkEnv = createSdkEnv(options.providerId, providerScope); @@ -1965,13 +1968,14 @@ export class ClaudeRuntime extends EventEmitter implements IOrchestrator { this.sessionHypotheses.delete(sessionId); this.sessionUncertaintyFlags.delete(sessionId); this.activeAnalyses.delete(sessionId); - if (enterpriseSessionMapStoreEnabled()) { + if (enterpriseSessionMapDbWritesEnabled()) { try { deleteClaudeSessionMapRuntimeSnapshots(sessionId); } catch (err) { console.warn('[ClaudeRuntime] Failed to delete runtime_snapshots session map:', (err as Error).message); } - } else { + } + if (legacySessionMapWritesEnabled()) { // Use immediate save — session is being removed, must persist before cleanup completes savePersistedSessionMapSync(this.sessionMap); } diff --git a/backend/src/agentv3/projectMemory.ts b/backend/src/agentv3/projectMemory.ts index 2735227f..b7d4eb85 100644 --- a/backend/src/agentv3/projectMemory.ts +++ b/backend/src/agentv3/projectMemory.ts @@ -48,7 +48,9 @@ import { type MemoryScope, } from '../types/sparkContracts'; import { + enterpriseKnowledgeDbWritesEnabled, enterpriseKnowledgeStoreEnabled, + legacyKnowledgeFilesystemWritesEnabled, type KnowledgeScope, getScopedKnowledgeRecord, listScopedKnowledgeRecords, @@ -155,7 +157,11 @@ export class ProjectMemory { ): void { this.load(); this.assertSaveInvariants(entry); - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.entries.set(entry.entryId, entry); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, entry.entryId, @@ -168,10 +174,7 @@ export class ProjectMemory { sourceRunId: storageScope?.sourceRunId ?? storageScope?.runId, }, ); - return; } - this.entries.set(entry.entryId, entry); - this.persist(); } /** Get an entry by id. */ @@ -196,13 +199,17 @@ export class ProjectMemory { entryId: string, storageScope?: KnowledgeScope, ): boolean { - if (enterpriseKnowledgeStoreEnabled()) { - return removeScopedKnowledgeRecord(KNOWLEDGE_KIND, entryId, storageScope); + let removed = false; + if (enterpriseKnowledgeDbWritesEnabled()) { + removed = removeScopedKnowledgeRecord(KNOWLEDGE_KIND, entryId, storageScope) || removed; } - this.load(); - const had = this.entries.delete(entryId); - if (had) this.persist(); - return had; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.load(); + const had = this.entries.delete(entryId); + if (had) this.persist(); + removed = had || removed; + } + return removed; } /** Filtered list, deterministically ordered by entryId. */ @@ -329,7 +336,15 @@ export class ProjectMemory { promotionLevel: (entry.promotionLevel ?? 0) + 1, promotionPolicy: policy, }; - if (enterpriseKnowledgeStoreEnabled()) { + const auditEntry = {entryId, policy, auditedAt: Date.now()}; + let auditRecorded = false; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.entries.set(entryId, promoted); + this.auditLog.push(auditEntry); + this.persist(); + auditRecorded = true; + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, entryId, @@ -342,13 +357,8 @@ export class ProjectMemory { sourceRunId: storageScope?.sourceRunId ?? storageScope?.runId, }, ); - this.auditLog.push({entryId, policy, auditedAt: Date.now()}); - this.persist(); - return; + if (!auditRecorded) this.auditLog.push(auditEntry); } - this.entries.set(entryId, promoted); - this.auditLog.push({entryId, policy, auditedAt: Date.now()}); - this.persist(); } /** Read-only view of the audit log, sorted by audit time ascending. */ diff --git a/backend/src/routes/reportRoutes.ts b/backend/src/routes/reportRoutes.ts index fbb933ff..5d76d8d3 100644 --- a/backend/src/routes/reportRoutes.ts +++ b/backend/src/routes/reportRoutes.ts @@ -15,8 +15,12 @@ import * as fs from 'fs'; import * as path from 'path'; import type Database from 'better-sqlite3'; import { attachRequestContext, requireRequestContext } from '../middleware/auth'; -import { resolveFeatureConfig } from '../config'; import { openEnterpriseDb } from '../services/enterpriseDb'; +import { + enterpriseDbReadAuthorityEnabled, + enterpriseDbWritesEnabled, + legacyFilesystemWritesEnabled, +} from '../services/enterpriseMigration'; import { REPORT_CAUSAL_MAP_CSS, REPORT_CAUSAL_MAP_SCRIPT } from '../services/reportCausalMapAssets'; import { localize, parseOutputLanguage } from '../agentv3/outputLanguage'; import { resolveEnterpriseDataRoot } from '../services/traceMetadataStore'; @@ -71,7 +75,15 @@ interface ReportArtifactRow { const SAFE_REPORT_ID_RE = /^[a-zA-Z0-9._:-]+$/; function enterpriseReportStoreEnabled(): boolean { - return resolveFeatureConfig(process.env).enterprise; + return enterpriseDbReadAuthorityEnabled(); +} + +function enterpriseReportDbWritesEnabled(): boolean { + return enterpriseDbWritesEnabled(); +} + +function legacyReportWritesEnabled(): boolean { + return legacyFilesystemWritesEnabled(); } function assertSafeReportSegment(value: string, label: string): string { @@ -259,6 +271,22 @@ function persistEnterpriseReport(reportId: string, entry: PersistedReport): void }); } +function persistLegacyReport(reportId: string, entry: PersistedReport): void { + const filePath = path.join(REPORTS_DIR, `${reportId}.html`); + fs.writeFileSync(filePath, entry.html, 'utf-8'); + const metaPath = path.join(REPORTS_DIR, `${reportId}.meta.json`); + fs.writeFileSync(metaPath, JSON.stringify({ + generatedAt: entry.generatedAt, + sessionId: entry.sessionId, + runId: entry.runId, + traceId: entry.traceId, + tenantId: entry.tenantId, + workspaceId: entry.workspaceId, + userId: entry.userId, + visibility: entry.visibility, + })); +} + function loadEnterpriseReport(reportId: string): PersistedReport | null { if (!SAFE_REPORT_ID_RE.test(reportId)) return null; try { @@ -316,24 +344,12 @@ export function upgradeLegacyReportHtml(html: string): string { export function persistReport(reportId: string, entry: PersistedReport): void { reportStore.set(reportId, entry); try { - if (enterpriseReportStoreEnabled()) { + if (legacyReportWritesEnabled()) { + persistLegacyReport(reportId, entry); + } + if (enterpriseReportDbWritesEnabled()) { persistEnterpriseReport(reportId, entry); - return; } - const filePath = path.join(REPORTS_DIR, `${reportId}.html`); - fs.writeFileSync(filePath, entry.html, 'utf-8'); - // Write metadata alongside - const metaPath = path.join(REPORTS_DIR, `${reportId}.meta.json`); - fs.writeFileSync(metaPath, JSON.stringify({ - generatedAt: entry.generatedAt, - sessionId: entry.sessionId, - runId: entry.runId, - traceId: entry.traceId, - tenantId: entry.tenantId, - workspaceId: entry.workspaceId, - userId: entry.userId, - visibility: entry.visibility, - })); } catch (err) { console.warn('[ReportRoutes] Failed to persist report to disk:', (err as Error).message); } @@ -344,6 +360,10 @@ function loadReportFromDisk(reportId: string): PersistedReport | null { if (enterpriseReportStoreEnabled()) { return loadEnterpriseReport(reportId); } + return loadLegacyReportFromDisk(reportId); +} + +function loadLegacyReportFromDisk(reportId: string): PersistedReport | null { try { const filePath = path.join(REPORTS_DIR, `${reportId}.html`); if (!fs.existsSync(filePath)) return null; @@ -388,30 +408,7 @@ function loadReportFromDisk(reportId: string): PersistedReport | null { } } -function deletePersistedReport(reportId: string): boolean { - if (enterpriseReportStoreEnabled()) { - if (!SAFE_REPORT_ID_RE.test(reportId)) return false; - try { - return withEnterpriseReportDb((db) => { - const row = db.prepare( - 'SELECT * FROM report_artifacts WHERE id = ?', - ).get(reportId); - if (!row) return false; - db.prepare('DELETE FROM report_artifacts WHERE id = ?').run(reportId); - try { - const reportDir = path.dirname(row.local_path); - const metadataPath = path.join(reportDir, 'report.json'); - if (fs.existsSync(row.local_path)) fs.unlinkSync(row.local_path); - if (fs.existsSync(metadataPath)) fs.unlinkSync(metadataPath); - fs.rmSync(reportDir, { recursive: true, force: true }); - } catch { /* non-fatal */ } - return true; - }); - } catch { - return false; - } - } - +function deleteLegacyReport(reportId: string): boolean { try { const htmlPath = path.join(REPORTS_DIR, `${reportId}.html`); const metaPath = path.join(REPORTS_DIR, `${reportId}.meta.json`); @@ -424,6 +421,40 @@ function deletePersistedReport(reportId: string): boolean { } } +function deleteEnterpriseReport(reportId: string): boolean { + if (!SAFE_REPORT_ID_RE.test(reportId)) return false; + try { + return withEnterpriseReportDb((db) => { + const row = db.prepare( + 'SELECT * FROM report_artifacts WHERE id = ?', + ).get(reportId); + if (!row) return false; + db.prepare('DELETE FROM report_artifacts WHERE id = ?').run(reportId); + try { + const reportDir = path.dirname(row.local_path); + const metadataPath = path.join(reportDir, 'report.json'); + if (fs.existsSync(row.local_path)) fs.unlinkSync(row.local_path); + if (fs.existsSync(metadataPath)) fs.unlinkSync(metadataPath); + fs.rmSync(reportDir, { recursive: true, force: true }); + } catch { /* non-fatal */ } + return true; + }); + } catch { + return false; + } +} + +function deletePersistedReport(reportId: string): boolean { + let deleted = false; + if (enterpriseReportDbWritesEnabled()) { + deleted = deleteEnterpriseReport(reportId) || deleted; + } + if (legacyReportWritesEnabled()) { + deleted = deleteLegacyReport(reportId) || deleted; + } + return deleted; +} + function getReportForContext(reportId: string, req: express.Request): PersistedReport | null { const context = requireRequestContext(req); const report = reportStore.get(reportId) || loadReportFromDisk(reportId); @@ -445,23 +476,24 @@ const reportCleanupInterval = setInterval(() => { } } - // Clean disk files - try { - const files = fs.readdirSync(REPORTS_DIR); - for (const file of files) { - if (!file.endsWith('.meta.json')) continue; - const metaPath = path.join(REPORTS_DIR, file); - try { - const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); - if (meta.generatedAt && now - meta.generatedAt > maxAge) { - const reportId = file.replace('.meta.json', ''); - fs.unlinkSync(metaPath); - const htmlPath = path.join(REPORTS_DIR, `${reportId}.html`); - if (fs.existsSync(htmlPath)) fs.unlinkSync(htmlPath); - } - } catch { /* skip individual file errors */ } - } - } catch { /* non-fatal */ } + if (legacyReportWritesEnabled()) { + try { + const files = fs.readdirSync(REPORTS_DIR); + for (const file of files) { + if (!file.endsWith('.meta.json')) continue; + const metaPath = path.join(REPORTS_DIR, file); + try { + const meta = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); + if (meta.generatedAt && now - meta.generatedAt > maxAge) { + const reportId = file.replace('.meta.json', ''); + fs.unlinkSync(metaPath); + const htmlPath = path.join(REPORTS_DIR, `${reportId}.html`); + if (fs.existsSync(htmlPath)) fs.unlinkSync(htmlPath); + } + } catch { /* skip individual file errors */ } + } + } catch { /* non-fatal */ } + } }, 30 * 60 * 1000); reportCleanupInterval.unref?.(); diff --git a/backend/src/scripts/enterpriseMigrationSnapshot.ts b/backend/src/scripts/enterpriseMigrationSnapshot.ts new file mode 100644 index 00000000..a2ebbc69 --- /dev/null +++ b/backend/src/scripts/enterpriseMigrationSnapshot.ts @@ -0,0 +1,65 @@ +#!/usr/bin/env tsx +// SPDX-License-Identifier: AGPL-3.0-or-later +// Copyright (C) 2024-2026 Gracker (Chris) +// This file is part of SmartPerfetto. See LICENSE for details. + +import { + buildEnterpriseMigrationDryRun, + createEnterpriseMigrationSnapshot, + describeEnterpriseMigrationRollback, + restoreEnterpriseMigrationSnapshot, +} from '../services/enterpriseMigration'; + +function usage(): never { + console.error([ + 'Usage:', + ' tsx src/scripts/enterpriseMigrationSnapshot.ts --dry-run [--snapshot-dir ]', + ' tsx src/scripts/enterpriseMigrationSnapshot.ts --snapshot [--snapshot-dir ]', + ' tsx src/scripts/enterpriseMigrationSnapshot.ts --restore ', + ].join('\n')); + process.exit(2); +} + +function argValue(args: string[], name: string): string | undefined { + const idx = args.indexOf(name); + if (idx < 0) return undefined; + const value = args[idx + 1]; + if (!value || value.startsWith('--')) usage(); + return value; +} + +async function main(): Promise { + const args = process.argv.slice(2); + const snapshotRoot = argValue(args, '--snapshot-dir'); + const restoreDir = argValue(args, '--restore'); + if (restoreDir) { + const result = await restoreEnterpriseMigrationSnapshot(restoreDir); + console.log(JSON.stringify(result, null, 2)); + return; + } + + if (args.includes('--dry-run')) { + const report = buildEnterpriseMigrationDryRun({snapshotRoot}); + console.log(JSON.stringify({ + ...report, + rollback: describeEnterpriseMigrationRollback(), + }, null, 2)); + return; + } + + if (args.includes('--snapshot')) { + const manifest = await createEnterpriseMigrationSnapshot({snapshotRoot}); + console.log(JSON.stringify({ + ...manifest, + rollback: describeEnterpriseMigrationRollback(), + }, null, 2)); + return; + } + + usage(); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/backend/src/services/__tests__/enterpriseMigration.test.ts b/backend/src/services/__tests__/enterpriseMigration.test.ts new file mode 100644 index 00000000..1f1e07d9 --- /dev/null +++ b/backend/src/services/__tests__/enterpriseMigration.test.ts @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +// Copyright (C) 2024-2026 Gracker (Chris) +// This file is part of SmartPerfetto. See LICENSE for details. + +import fs from 'fs'; +import fsp from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { ENTERPRISE_FEATURE_FLAG_ENV } from '../../config'; +import { ENTERPRISE_DB_PATH_ENV, openEnterpriseDb } from '../enterpriseDb'; +import { + ENTERPRISE_MIGRATION_PHASE_ENV, + buildEnterpriseMigrationDryRun, + createEnterpriseMigrationSnapshot, + resolveEnterpriseMigrationPlan, + restoreEnterpriseMigrationSnapshot, +} from '../enterpriseMigration'; +import { + getTraceMetadataPath, + readTraceMetadata, + writeTraceMetadata, +} from '../traceMetadataStore'; + +const DATA_DIR_ENV = 'SMARTPERFETTO_DATA_DIR'; +const LOGS_DIR_ENV = 'SMARTPERFETTO_LOGS_DIR'; +const PROVIDER_DATA_DIR_ENV = 'PROVIDER_DATA_DIR_OVERRIDE'; +const UPLOAD_DIR_ENV = 'UPLOAD_DIR'; + +const originalEnv = { + enterprise: process.env[ENTERPRISE_FEATURE_FLAG_ENV], + migrationPhase: process.env[ENTERPRISE_MIGRATION_PHASE_ENV], + enterpriseDbPath: process.env[ENTERPRISE_DB_PATH_ENV], + dataDir: process.env[DATA_DIR_ENV], + logsDir: process.env[LOGS_DIR_ENV], + providerDataDir: process.env[PROVIDER_DATA_DIR_ENV], + uploadDir: process.env[UPLOAD_DIR_ENV], +}; + +function restoreEnvValue(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} + +function applyEnv(env: Record): void { + for (const [key, value] of Object.entries(env)) { + process.env[key] = value; + } +} + +function testEnv(tmpDir: string, phase = 'cutover'): Record { + return { + [ENTERPRISE_FEATURE_FLAG_ENV]: 'true', + [ENTERPRISE_MIGRATION_PHASE_ENV]: phase, + [ENTERPRISE_DB_PATH_ENV]: path.join(tmpDir, 'enterprise.sqlite'), + [DATA_DIR_ENV]: path.join(tmpDir, 'data'), + [LOGS_DIR_ENV]: path.join(tmpDir, 'logs'), + [PROVIDER_DATA_DIR_ENV]: path.join(tmpDir, 'provider'), + [UPLOAD_DIR_ENV]: path.join(tmpDir, 'uploads'), + }; +} + +async function writeText(filePath: string, text: string): Promise { + await fsp.mkdir(path.dirname(filePath), {recursive: true}); + await fsp.writeFile(filePath, text, 'utf-8'); +} + +afterEach(() => { + restoreEnvValue(ENTERPRISE_FEATURE_FLAG_ENV, originalEnv.enterprise); + restoreEnvValue(ENTERPRISE_MIGRATION_PHASE_ENV, originalEnv.migrationPhase); + restoreEnvValue(ENTERPRISE_DB_PATH_ENV, originalEnv.enterpriseDbPath); + restoreEnvValue(DATA_DIR_ENV, originalEnv.dataDir); + restoreEnvValue(LOGS_DIR_ENV, originalEnv.logsDir); + restoreEnvValue(PROVIDER_DATA_DIR_ENV, originalEnv.providerDataDir); + restoreEnvValue(UPLOAD_DIR_ENV, originalEnv.uploadDir); +}); + +describe('enterprise migration phases', () => { + it('resolves legacy, dual-write, cutover, and retired storage semantics', () => { + expect(resolveEnterpriseMigrationPlan({}).phase).toBe('legacy'); + + expect(resolveEnterpriseMigrationPlan({ + [ENTERPRISE_FEATURE_FLAG_ENV]: 'true', + }).phase).toBe('cutover'); + + expect(resolveEnterpriseMigrationPlan({ + [ENTERPRISE_FEATURE_FLAG_ENV]: 'true', + [ENTERPRISE_MIGRATION_PHASE_ENV]: 'P-A', + })).toEqual(expect.objectContaining({ + phase: 'dual-write', + readAuthority: 'filesystem', + writeFilesystem: true, + writeDb: true, + rollback: 'delete-db', + })); + + expect(resolveEnterpriseMigrationPlan({ + [ENTERPRISE_FEATURE_FLAG_ENV]: 'true', + [ENTERPRISE_MIGRATION_PHASE_ENV]: 'cutover', + })).toEqual(expect.objectContaining({ + phase: 'cutover', + readAuthority: 'db', + writeFilesystem: false, + writeDb: true, + rollback: 'return-to-dual-write', + })); + + expect(resolveEnterpriseMigrationPlan({ + [ENTERPRISE_FEATURE_FLAG_ENV]: 'true', + [ENTERPRISE_MIGRATION_PHASE_ENV]: 'retired', + })).toEqual(expect.objectContaining({ + phase: 'retired', + readAuthority: 'db', + writeFilesystem: false, + writeDb: true, + rollback: 'restore-snapshots', + })); + }); + + it('dual-writes trace metadata while keeping legacy JSON authoritative until cutover', async () => { + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'smartperfetto-migration-trace-')); + try { + applyEnv(testEnv(tmpDir, 'dual-write')); + await writeTraceMetadata({ + id: 'trace-a', + filename: 'legacy-authority.perfetto-trace', + size: 123, + uploadedAt: new Date(0).toISOString(), + status: 'ready', + tenantId: 'tenant-a', + workspaceId: 'workspace-a', + userId: 'user-a', + }); + + const legacyPath = getTraceMetadataPath('trace-a')!; + expect(fs.existsSync(legacyPath)).toBe(true); + + let db = openEnterpriseDb(process.env[ENTERPRISE_DB_PATH_ENV]!); + try { + const row = db.prepare(` + SELECT tenant_id, workspace_id, metadata_json + FROM trace_assets + WHERE id = 'trace-a' + `).get() as {tenant_id: string; workspace_id: string; metadata_json: string}; + expect(row.tenant_id).toBe('tenant-a'); + expect(row.workspace_id).toBe('workspace-a'); + expect(row.metadata_json).toContain('legacy-authority.perfetto-trace'); + db.prepare(` + UPDATE trace_assets + SET metadata_json = ? + WHERE id = 'trace-a' + `).run(JSON.stringify({ + filename: 'db-shadow-only.perfetto-trace', + uploadedAt: new Date(0).toISOString(), + })); + } finally { + db.close(); + } + + await expect(readTraceMetadata('trace-a')).resolves.toEqual( + expect.objectContaining({filename: 'legacy-authority.perfetto-trace'}), + ); + + process.env[ENTERPRISE_MIGRATION_PHASE_ENV] = 'cutover'; + await writeTraceMetadata({ + id: 'trace-b', + filename: 'db-authority.perfetto-trace', + size: 456, + uploadedAt: new Date(1).toISOString(), + status: 'ready', + tenantId: 'tenant-a', + workspaceId: 'workspace-a', + userId: 'user-a', + }); + + expect(fs.existsSync(getTraceMetadataPath('trace-b')!)).toBe(false); + await expect(readTraceMetadata('trace-b')).resolves.toEqual( + expect.objectContaining({filename: 'db-authority.perfetto-trace'}), + ); + await expect(readTraceMetadata('trace-a')).resolves.toEqual( + expect.objectContaining({filename: 'db-shadow-only.perfetto-trace'}), + ); + } finally { + await fsp.rm(tmpDir, {recursive: true, force: true}); + } + }); + + it('creates dry-run fingerprints, filesystem snapshots, DB snapshots, and restores them', async () => { + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'smartperfetto-migration-snapshot-')); + try { + const env = testEnv(tmpDir, 'retired'); + await writeText(path.join(env[UPLOAD_DIR_ENV], 'traces', 'trace-a.json'), '{"id":"trace-a"}'); + await writeText(path.join(env[LOGS_DIR_ENV], 'reports', 'report-a.html'), 'report'); + await writeText(path.join(env[DATA_DIR_ENV], 'tenant-a', 'workspace-a', 'traces', 'trace-a.trace'), 'trace bytes'); + await writeText(path.join(env[PROVIDER_DATA_DIR_ENV], 'providers.json'), '[{"id":"legacy-provider"}]'); + + const db = openEnterpriseDb(env[ENTERPRISE_DB_PATH_ENV]); + try { + db.prepare(` + INSERT INTO organizations (id, name, status, plan, created_at, updated_at) + VALUES ('tenant-a', 'Tenant A', 'active', 'enterprise', 1, 1) + `).run(); + } finally { + db.close(); + } + + const dryRun = buildEnterpriseMigrationDryRun({ + env: env as NodeJS.ProcessEnv, + snapshotRoot: path.join(tmpDir, 'snapshots'), + now: new Date('2026-05-08T00:00:00.000Z'), + }); + expect(dryRun.phase).toBe('retired'); + expect(dryRun.filesystem.find(item => item.label === 'uploads')).toEqual( + expect.objectContaining({exists: true, fileCount: 1}), + ); + expect(dryRun.database.tableCounts.organizations).toBe(1); + expect(dryRun.fingerprint).toMatch(/^[a-f0-9]{64}$/); + + const manifest = await createEnterpriseMigrationSnapshot({ + env: env as NodeJS.ProcessEnv, + snapshotRoot: path.join(tmpDir, 'snapshots'), + snapshotId: 'snapshot-a', + now: new Date('2026-05-08T00:00:00.000Z'), + }); + expect(fs.existsSync(path.join(manifest.snapshotDir, 'manifest.json'))).toBe(true); + expect(fs.existsSync(manifest.databaseSnapshot.snapshotPath)).toBe(true); + + await fsp.rm(env[UPLOAD_DIR_ENV], {recursive: true, force: true}); + await fsp.rm(env[LOGS_DIR_ENV], {recursive: true, force: true}); + await fsp.rm(env[DATA_DIR_ENV], {recursive: true, force: true}); + await fsp.rm(env[PROVIDER_DATA_DIR_ENV], {recursive: true, force: true}); + await fsp.rm(env[ENTERPRISE_DB_PATH_ENV], {force: true}); + + const restored = await restoreEnterpriseMigrationSnapshot(manifest.snapshotDir, { + env: env as NodeJS.ProcessEnv, + now: new Date('2026-05-08T00:01:00.000Z'), + }); + expect(restored.restoredDatabase.restored).toBe(true); + expect(fs.existsSync(path.join(env[UPLOAD_DIR_ENV], 'traces', 'trace-a.json'))).toBe(true); + expect(fs.existsSync(path.join(env[LOGS_DIR_ENV], 'reports', 'report-a.html'))).toBe(true); + expect(fs.existsSync(path.join(env[DATA_DIR_ENV], 'tenant-a', 'workspace-a', 'traces', 'trace-a.trace'))).toBe(true); + expect(fs.existsSync(path.join(env[PROVIDER_DATA_DIR_ENV], 'providers.json'))).toBe(true); + + const restoredDb = openEnterpriseDb(env[ENTERPRISE_DB_PATH_ENV]); + try { + const row = restoredDb.prepare(` + SELECT name + FROM organizations + WHERE id = 'tenant-a' + `).get() as {name: string}; + expect(row.name).toBe('Tenant A'); + } finally { + restoredDb.close(); + } + } finally { + await fsp.rm(tmpDir, {recursive: true, force: true}); + } + }); +}); diff --git a/backend/src/services/baselineStore.ts b/backend/src/services/baselineStore.ts index be044978..ff509b66 100644 --- a/backend/src/services/baselineStore.ts +++ b/backend/src/services/baselineStore.ts @@ -34,7 +34,9 @@ import { type PerfBaselineKey, } from '../types/sparkContracts'; import { + enterpriseKnowledgeDbWritesEnabled, enterpriseKnowledgeStoreEnabled, + legacyKnowledgeFilesystemWritesEnabled, type KnowledgeScope, getScopedKnowledgeRecord, listScopedKnowledgeRecords, @@ -132,7 +134,11 @@ export class BaselineStore { addBaseline(record: BaselineRecord, scope?: KnowledgeScope): void { this.load(); this.assertPublishInvariants(record); - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.baselines.set(record.baselineId, record); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, record.baselineId, @@ -141,10 +147,7 @@ export class BaselineStore { scope, {createdAt: record.capturedAt, updatedAt: Date.now()}, ); - return; } - this.baselines.set(record.baselineId, record); - this.persist(); } /** Get a baseline by id. */ @@ -165,13 +168,17 @@ export class BaselineStore { /** Remove a baseline. Returns whether anything was actually removed. */ removeBaseline(baselineId: string, scope?: KnowledgeScope): boolean { - if (enterpriseKnowledgeStoreEnabled()) { - return removeScopedKnowledgeRecord(KNOWLEDGE_KIND, baselineId, scope); + let removed = false; + if (enterpriseKnowledgeDbWritesEnabled()) { + removed = removeScopedKnowledgeRecord(KNOWLEDGE_KIND, baselineId, scope) || removed; } - this.load(); - const had = this.baselines.delete(baselineId); - if (had) this.persist(); - return had; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.load(); + const had = this.baselines.delete(baselineId); + if (had) this.persist(); + removed = had || removed; + } + return removed; } /** diff --git a/backend/src/services/caseGraph.ts b/backend/src/services/caseGraph.ts index 477e8690..6bff9f39 100644 --- a/backend/src/services/caseGraph.ts +++ b/backend/src/services/caseGraph.ts @@ -39,7 +39,9 @@ import * as path from 'path'; import {type CaseEdge} from '../types/sparkContracts'; import { + enterpriseKnowledgeDbWritesEnabled, enterpriseKnowledgeStoreEnabled, + legacyKnowledgeFilesystemWritesEnabled, type KnowledgeScope, listScopedKnowledgeRecords, removeScopedKnowledgeRecord, @@ -111,7 +113,11 @@ export class CaseGraph { `Self-loops are not permitted: edge '${edge.edgeId}' has fromCaseId === toCaseId === '${edge.fromCaseId}'`, ); } - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.edges.set(edgeKey(edge), edge); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { for (const existing of this.listEnterpriseEdges(scope)) { if ( edgeKey(existing.record) === edgeKey(edge) && @@ -131,29 +137,31 @@ export class CaseGraph { edge, scope, ); - return; } - this.edges.set(edgeKey(edge), edge); - this.persist(); } /** Remove an edge by canonical id. Returns whether it was present. */ removeEdge(edgeId: string, scope?: KnowledgeScope): boolean { - if (enterpriseKnowledgeStoreEnabled()) { - return removeScopedKnowledgeRecord(KNOWLEDGE_KIND, edgeId, scope); + let removed = false; + if (enterpriseKnowledgeDbWritesEnabled()) { + removed = removeScopedKnowledgeRecord(KNOWLEDGE_KIND, edgeId, scope) || removed; } - this.load(); - let foundKey: string | undefined; - for (const [k, e] of this.edges) { - if (e.edgeId === edgeId) { - foundKey = k; - break; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.load(); + let foundKey: string | undefined; + for (const [k, e] of this.edges) { + if (e.edgeId === edgeId) { + foundKey = k; + break; + } + } + if (foundKey) { + this.edges.delete(foundKey); + this.persist(); + removed = true; } } - if (!foundKey) return false; - this.edges.delete(foundKey); - this.persist(); - return true; + return removed; } /** Get all edges originating at the case. */ diff --git a/backend/src/services/caseLibrary.ts b/backend/src/services/caseLibrary.ts index 17c2b328..69e4aa1c 100644 --- a/backend/src/services/caseLibrary.ts +++ b/backend/src/services/caseLibrary.ts @@ -36,7 +36,9 @@ import { makeSparkProvenance, } from '../types/sparkContracts'; import { + enterpriseKnowledgeDbWritesEnabled, enterpriseKnowledgeStoreEnabled, + legacyKnowledgeFilesystemWritesEnabled, type KnowledgeScope, getScopedKnowledgeRecord, listScopedKnowledgeRecords, @@ -109,7 +111,11 @@ export class CaseLibrary { `Use publishCase() to advance a case to 'published'; saveCase() rejects published records to keep the gate auditable`, ); } - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.cases.set(record.caseId, record); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, record.caseId, @@ -118,10 +124,7 @@ export class CaseLibrary { scope, {createdAt: record.createdAt, updatedAt: Date.now()}, ); - return; } - this.cases.set(record.caseId, record); - this.persist(); } getCase(caseId: string, scope?: KnowledgeScope): CaseNode | undefined { @@ -137,13 +140,17 @@ export class CaseLibrary { } removeCase(caseId: string, scope?: KnowledgeScope): boolean { - if (enterpriseKnowledgeStoreEnabled()) { - return removeScopedKnowledgeRecord(KNOWLEDGE_KIND, caseId, scope); + let removed = false; + if (enterpriseKnowledgeDbWritesEnabled()) { + removed = removeScopedKnowledgeRecord(KNOWLEDGE_KIND, caseId, scope) || removed; } - this.load(); - const had = this.cases.delete(caseId); - if (had) this.persist(); - return had; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.load(); + const had = this.cases.delete(caseId); + if (had) this.persist(); + removed = had || removed; + } + return removed; } listCases(opts: ListOptions = {}, scope?: KnowledgeScope): CaseNode[] { @@ -213,7 +220,11 @@ export class CaseLibrary { curatedBy: trimmedReviewer, curatedAt: Date.now(), }; - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.cases.set(caseId, published); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, caseId, @@ -222,10 +233,7 @@ export class CaseLibrary { scope, {createdAt: published.createdAt, updatedAt: published.curatedAt}, ); - return published; } - this.cases.set(caseId, published); - this.persist(); return published; } @@ -265,7 +273,11 @@ export class CaseLibrary { traceArtifactId: undefined, traceUnavailableReason: reason, }; - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.cases.set(caseId, archived); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, caseId, @@ -274,10 +286,7 @@ export class CaseLibrary { scope, {createdAt: archived.createdAt, updatedAt: Date.now()}, ); - return archived; } - this.cases.set(caseId, archived); - this.persist(); return archived; } diff --git a/backend/src/services/enterpriseMigration.ts b/backend/src/services/enterpriseMigration.ts new file mode 100644 index 00000000..7c5b27be --- /dev/null +++ b/backend/src/services/enterpriseMigration.ts @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +// Copyright (C) 2024-2026 Gracker (Chris) +// This file is part of SmartPerfetto. See LICENSE for details. + +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +import { ENTERPRISE_FEATURE_FLAG_ENV, resolveFeatureConfig } from '../config'; +import { ENTERPRISE_DB_PATH_ENV, openEnterpriseDb, resolveEnterpriseDbPath } from './enterpriseDb'; +import { ENTERPRISE_MINIMAL_SCHEMA_TABLES } from './enterpriseSchema'; + +export const ENTERPRISE_MIGRATION_PHASE_ENV = 'SMARTPERFETTO_ENTERPRISE_MIGRATION_PHASE'; +export const ENTERPRISE_MIGRATION_SNAPSHOT_DIR_ENV = 'SMARTPERFETTO_ENTERPRISE_MIGRATION_SNAPSHOT_DIR'; + +export type EnterpriseMigrationPhase = 'legacy' | 'dual-write' | 'cutover' | 'retired'; +export type EnterpriseReadAuthority = 'filesystem' | 'db'; + +export interface EnterpriseMigrationPlan { + enterpriseEnabled: boolean; + phase: EnterpriseMigrationPhase; + readAuthority: EnterpriseReadAuthority; + writeFilesystem: boolean; + writeDb: boolean; + legacyReadOnly: boolean; + rollback: 'disable-enterprise' | 'delete-db' | 'return-to-dual-write' | 'restore-snapshots'; +} + +export interface MigrationFilesystemFingerprint { + label: string; + path: string; + exists: boolean; + kind: 'file' | 'directory'; + fileCount: number; + totalBytes: number; + sha256: string | null; + failures: string[]; +} + +export interface MigrationDatabaseFingerprint { + path: string; + exists: boolean; + totalRows: number; + tableCounts: Record; + sha256: string | null; + failures: string[]; +} + +export interface EnterpriseMigrationDryRunReport { + generatedAt: string; + phase: EnterpriseMigrationPhase; + plan: EnterpriseMigrationPlan; + filesystem: MigrationFilesystemFingerprint[]; + database: MigrationDatabaseFingerprint; + failures: string[]; + fingerprint: string; +} + +export interface EnterpriseMigrationSnapshotOptions { + env?: NodeJS.ProcessEnv; + now?: Date; + snapshotRoot?: string; + snapshotId?: string; +} + +export interface EnterpriseMigrationSnapshotManifest extends EnterpriseMigrationDryRunReport { + snapshotId: string; + snapshotDir: string; + filesystemSnapshots: Array<{ + label: string; + sourcePath: string; + snapshotPath: string; + kind: 'file' | 'directory'; + copied: boolean; + }>; + databaseSnapshot: { + sourcePath: string; + snapshotPath: string; + copied: boolean; + }; +} + +export interface EnterpriseMigrationRestoreResult { + snapshotId: string; + restoredAt: string; + restoredFilesystem: Array<{ + label: string; + targetPath: string; + restored: boolean; + }>; + restoredDatabase: { + targetPath: string; + restored: boolean; + }; +} + +interface FilesystemSource { + label: string; + path: string; + kind: 'file' | 'directory'; +} + +const DATA_DIR_ENV = 'SMARTPERFETTO_DATA_DIR'; +const LOGS_DIR_ENV = 'SMARTPERFETTO_LOGS_DIR'; +const PROVIDER_DATA_DIR_ENV = 'PROVIDER_DATA_DIR_OVERRIDE'; +const UPLOAD_DIR_ENV = 'UPLOAD_DIR'; + +function parseEnterpriseMigrationPhase( + value: string | undefined, + enterpriseEnabled: boolean, +): EnterpriseMigrationPhase { + if (!enterpriseEnabled) return 'legacy'; + if (!value || value.trim().length === 0) return 'cutover'; + const normalized = value.trim().toLowerCase().replace(/_/g, '-'); + if (['legacy', 'off', 'filesystem'].includes(normalized)) return 'legacy'; + if (['p-a', 'pa', 'dual', 'dualwrite', 'dual-write'].includes(normalized)) return 'dual-write'; + if (['p-b', 'pb', 'cut-read', 'cutover', 'db', 'db-authoritative'].includes(normalized)) return 'cutover'; + if (['p-c', 'pc', 'retire', 'retired'].includes(normalized)) return 'retired'; + throw new Error( + `Invalid ${ENTERPRISE_MIGRATION_PHASE_ENV}: ${value}. Expected dual-write, cutover, or retired.`, + ); +} + +export function resolveEnterpriseMigrationPlan( + env: NodeJS.ProcessEnv = process.env, +): EnterpriseMigrationPlan { + const enterpriseEnabled = resolveFeatureConfig(env).enterprise; + const phase = parseEnterpriseMigrationPhase( + env[ENTERPRISE_MIGRATION_PHASE_ENV], + enterpriseEnabled, + ); + + if (!enterpriseEnabled || phase === 'legacy') { + return { + enterpriseEnabled, + phase: 'legacy', + readAuthority: 'filesystem', + writeFilesystem: true, + writeDb: false, + legacyReadOnly: false, + rollback: 'disable-enterprise', + }; + } + + if (phase === 'dual-write') { + return { + enterpriseEnabled: true, + phase, + readAuthority: 'filesystem', + writeFilesystem: true, + writeDb: true, + legacyReadOnly: false, + rollback: 'delete-db', + }; + } + + if (phase === 'cutover') { + return { + enterpriseEnabled: true, + phase, + readAuthority: 'db', + writeFilesystem: false, + writeDb: true, + legacyReadOnly: true, + rollback: 'return-to-dual-write', + }; + } + + return { + enterpriseEnabled: true, + phase: 'retired', + readAuthority: 'db', + writeFilesystem: false, + writeDb: true, + legacyReadOnly: false, + rollback: 'restore-snapshots', + }; +} + +export function enterpriseDbReadAuthorityEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveEnterpriseMigrationPlan(env).readAuthority === 'db'; +} + +export function enterpriseDbWritesEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveEnterpriseMigrationPlan(env).writeDb; +} + +export function legacyFilesystemReadAuthorityEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveEnterpriseMigrationPlan(env).readAuthority === 'filesystem'; +} + +export function legacyFilesystemWritesEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveEnterpriseMigrationPlan(env).writeFilesystem; +} + +function resolveUploadRoot(env: NodeJS.ProcessEnv): string { + return path.resolve(env[UPLOAD_DIR_ENV] || './uploads'); +} + +function resolveDataRoot(env: NodeJS.ProcessEnv): string { + const configured = env[DATA_DIR_ENV]; + return path.resolve(configured && configured.trim().length > 0 ? configured : 'data'); +} + +function resolveLogsRoot(env: NodeJS.ProcessEnv): string { + const configured = env[LOGS_DIR_ENV]; + return path.resolve(configured && configured.trim().length > 0 ? configured : 'logs'); +} + +function resolveProviderFile(env: NodeJS.ProcessEnv): string { + const providerDir = env[PROVIDER_DATA_DIR_ENV] || path.resolve(process.cwd(), 'data'); + return path.join(path.resolve(providerDir), 'providers.json'); +} + +function defaultSnapshotRoot(env: NodeJS.ProcessEnv): string { + const configured = env[ENTERPRISE_MIGRATION_SNAPSHOT_DIR_ENV]; + return path.resolve( + configured && configured.trim().length > 0 + ? configured + : path.join(process.cwd(), 'enterprise-migration-snapshots'), + ); +} + +function snapshotIdForDate(date: Date): string { + return date.toISOString().replace(/[:.]/g, '-'); +} + +function filesystemSources(env: NodeJS.ProcessEnv): FilesystemSource[] { + return [ + {label: 'uploads', path: resolveUploadRoot(env), kind: 'directory'}, + {label: 'logs', path: resolveLogsRoot(env), kind: 'directory'}, + {label: 'data', path: resolveDataRoot(env), kind: 'directory'}, + {label: 'provider-file', path: resolveProviderFile(env), kind: 'file'}, + ]; +} + +function emptyHash(): string { + return crypto.createHash('sha256').digest('hex'); +} + +function hashFile(filePath: string): {bytes: number; sha256: string} { + const hash = crypto.createHash('sha256'); + const data = fs.readFileSync(filePath); + hash.update(data); + return {bytes: data.byteLength, sha256: hash.digest('hex')}; +} + +function fingerprintPath(source: FilesystemSource, snapshotRoot: string): MigrationFilesystemFingerprint { + const failures: string[] = []; + if (!fs.existsSync(source.path)) { + return { + label: source.label, + path: source.path, + exists: false, + kind: source.kind, + fileCount: 0, + totalBytes: 0, + sha256: null, + failures, + }; + } + + const aggregate = crypto.createHash('sha256'); + let fileCount = 0; + let totalBytes = 0; + + try { + if (source.kind === 'file') { + const stat = fs.statSync(source.path); + if (!stat.isFile()) { + throw new Error(`Expected file, got non-file path`); + } + const fileHash = hashFile(source.path); + aggregate.update(path.basename(source.path)); + aggregate.update('\0'); + aggregate.update(String(fileHash.bytes)); + aggregate.update('\0'); + aggregate.update(fileHash.sha256); + fileCount = 1; + totalBytes = fileHash.bytes; + } else { + const files = listFiles(source.path, snapshotRoot); + for (const file of files) { + const rel = path.relative(source.path, file).split(path.sep).join('/'); + const fileHash = hashFile(file); + aggregate.update(rel); + aggregate.update('\0'); + aggregate.update(String(fileHash.bytes)); + aggregate.update('\0'); + aggregate.update(fileHash.sha256); + aggregate.update('\0'); + fileCount += 1; + totalBytes += fileHash.bytes; + } + } + } catch (err) { + failures.push((err as Error).message); + } + + return { + label: source.label, + path: source.path, + exists: true, + kind: source.kind, + fileCount, + totalBytes, + sha256: failures.length > 0 ? null : (fileCount === 0 ? emptyHash() : aggregate.digest('hex')), + failures, + }; +} + +function isUnderPath(candidate: string, parent: string): boolean { + const relative = path.relative(parent, candidate); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +} + +function listFiles(root: string, snapshotRoot: string): string[] { + const out: string[] = []; + const stack = [root]; + while (stack.length > 0) { + const current = stack.pop()!; + if (isUnderPath(current, snapshotRoot)) continue; + const entries = fs.readdirSync(current, {withFileTypes: true}) + .sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const child = path.join(current, entry.name); + if (isUnderPath(child, snapshotRoot)) continue; + if (entry.isDirectory()) { + stack.push(child); + } else if (entry.isFile()) { + out.push(child); + } + } + } + return out.sort((a, b) => a.localeCompare(b)); +} + +function fingerprintDatabase(env: NodeJS.ProcessEnv): MigrationDatabaseFingerprint { + const dbPath = resolveEnterpriseDbPath(env); + const failures: string[] = []; + let tableCounts: Record = {}; + let totalRows = 0; + + try { + const db = openEnterpriseDb(dbPath); + try { + db.pragma('wal_checkpoint(FULL)'); + for (const table of ENTERPRISE_MINIMAL_SCHEMA_TABLES) { + try { + const row = db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get() as {count: number}; + tableCounts[table] = row.count; + totalRows += row.count; + } catch (err) { + tableCounts[table] = 0; + failures.push(`${table}: ${(err as Error).message}`); + } + } + } finally { + db.close(); + } + } catch (err) { + failures.push((err as Error).message); + tableCounts = {}; + } + + let sha256: string | null = null; + if (fs.existsSync(dbPath)) { + try { + sha256 = hashFile(dbPath).sha256; + } catch (err) { + failures.push((err as Error).message); + } + } + + return { + path: dbPath, + exists: fs.existsSync(dbPath), + totalRows, + tableCounts, + sha256, + failures, + }; +} + +function stableJson(value: unknown): string { + return JSON.stringify(sortJson(value)); +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => [key, sortJson(entry)]), + ); + } + return value; +} + +function reportFingerprint(report: Omit): string { + return crypto.createHash('sha256').update(stableJson({ + phase: report.phase, + filesystem: report.filesystem.map(item => ({ + label: item.label, + path: item.path, + exists: item.exists, + kind: item.kind, + fileCount: item.fileCount, + totalBytes: item.totalBytes, + sha256: item.sha256, + })), + database: { + path: report.database.path, + exists: report.database.exists, + totalRows: report.database.totalRows, + tableCounts: report.database.tableCounts, + sha256: report.database.sha256, + }, + })).digest('hex'); +} + +export function buildEnterpriseMigrationDryRun( + options: EnterpriseMigrationSnapshotOptions = {}, +): EnterpriseMigrationDryRunReport { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const snapshotRoot = options.snapshotRoot + ? path.resolve(options.snapshotRoot) + : defaultSnapshotRoot(env); + const plan = resolveEnterpriseMigrationPlan(env); + const filesystem = filesystemSources(env) + .map(source => fingerprintPath(source, snapshotRoot)); + const database = fingerprintDatabase(env); + const failures = [ + ...filesystem.flatMap(item => item.failures.map(failure => `${item.label}: ${failure}`)), + ...database.failures.map(failure => `database: ${failure}`), + ]; + const reportWithoutFingerprint = { + generatedAt: now.toISOString(), + phase: plan.phase, + plan, + filesystem, + database, + failures, + }; + return { + ...reportWithoutFingerprint, + fingerprint: reportFingerprint(reportWithoutFingerprint), + }; +} + +function copySnapshotSource( + source: FilesystemSource, + snapshotDir: string, +): EnterpriseMigrationSnapshotManifest['filesystemSnapshots'][number] { + const snapshotPath = path.join(snapshotDir, 'filesystem', source.label); + if (!fs.existsSync(source.path)) { + return { + label: source.label, + sourcePath: source.path, + snapshotPath, + kind: source.kind, + copied: false, + }; + } + fs.mkdirSync(path.dirname(snapshotPath), {recursive: true}); + if (source.kind === 'file') { + fs.copyFileSync(source.path, snapshotPath); + } else { + fs.cpSync(source.path, snapshotPath, { + recursive: true, + force: true, + filter: candidate => !isUnderPath(candidate, snapshotDir), + }); + } + return { + label: source.label, + sourcePath: source.path, + snapshotPath, + kind: source.kind, + copied: true, + }; +} + +async function backupDatabase(env: NodeJS.ProcessEnv, snapshotDir: string): Promise { + const sourcePath = resolveEnterpriseDbPath(env); + const snapshotPath = path.join(snapshotDir, 'database', path.basename(sourcePath)); + fs.mkdirSync(path.dirname(snapshotPath), {recursive: true}); + const db = openEnterpriseDb(sourcePath); + try { + db.pragma('wal_checkpoint(FULL)'); + await (db as unknown as {backup: (destination: string) => Promise}).backup(snapshotPath); + } finally { + db.close(); + } + return { + sourcePath, + snapshotPath, + copied: fs.existsSync(snapshotPath), + }; +} + +export async function createEnterpriseMigrationSnapshot( + options: EnterpriseMigrationSnapshotOptions = {}, +): Promise { + const env = options.env ?? process.env; + const now = options.now ?? new Date(); + const snapshotId = options.snapshotId ?? snapshotIdForDate(now); + const snapshotRoot = options.snapshotRoot + ? path.resolve(options.snapshotRoot) + : defaultSnapshotRoot(env); + const snapshotDir = path.join(snapshotRoot, snapshotId); + fs.mkdirSync(snapshotDir, {recursive: true}); + + const dryRun = buildEnterpriseMigrationDryRun({ + env, + now, + snapshotRoot, + snapshotId, + }); + const filesystemSnapshots = filesystemSources(env) + .map(source => copySnapshotSource(source, snapshotDir)); + const databaseSnapshot = await backupDatabase(env, snapshotDir); + const manifest: EnterpriseMigrationSnapshotManifest = { + ...dryRun, + snapshotId, + snapshotDir, + filesystemSnapshots, + databaseSnapshot, + }; + fs.writeFileSync( + path.join(snapshotDir, 'manifest.json'), + JSON.stringify(manifest, null, 2), + 'utf-8', + ); + return manifest; +} + +function loadSnapshotManifest(snapshotDir: string): EnterpriseMigrationSnapshotManifest { + const manifestPath = path.join(snapshotDir, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as EnterpriseMigrationSnapshotManifest; + if (!manifest.snapshotId || !Array.isArray(manifest.filesystemSnapshots)) { + throw new Error(`Invalid migration snapshot manifest: ${manifestPath}`); + } + return manifest; +} + +function assertRestoreTargetSafe(targetPath: string): void { + const resolved = path.resolve(targetPath); + const parsed = path.parse(resolved); + if (resolved === parsed.root || resolved.length < parsed.root.length + 2) { + throw new Error(`Refusing to restore over unsafe path: ${targetPath}`); + } +} + +export async function restoreEnterpriseMigrationSnapshot( + snapshotDir: string, + options: {env?: NodeJS.ProcessEnv; now?: Date} = {}, +): Promise { + const manifest = loadSnapshotManifest(path.resolve(snapshotDir)); + const restoredFilesystem: EnterpriseMigrationRestoreResult['restoredFilesystem'] = []; + for (const item of manifest.filesystemSnapshots) { + if (!item.copied || !fs.existsSync(item.snapshotPath)) { + restoredFilesystem.push({ + label: item.label, + targetPath: item.sourcePath, + restored: false, + }); + continue; + } + assertRestoreTargetSafe(item.sourcePath); + fs.rmSync(item.sourcePath, {recursive: true, force: true}); + fs.mkdirSync(path.dirname(item.sourcePath), {recursive: true}); + if (item.kind === 'file') { + fs.copyFileSync(item.snapshotPath, item.sourcePath); + } else { + fs.cpSync(item.snapshotPath, item.sourcePath, {recursive: true, force: true}); + } + restoredFilesystem.push({ + label: item.label, + targetPath: item.sourcePath, + restored: true, + }); + } + + const env = options.env ?? process.env; + const targetDbPath = resolveEnterpriseDbPath(env); + let restoredDb = false; + if (manifest.databaseSnapshot.copied && fs.existsSync(manifest.databaseSnapshot.snapshotPath)) { + assertRestoreTargetSafe(targetDbPath); + fs.mkdirSync(path.dirname(targetDbPath), {recursive: true}); + fs.rmSync(targetDbPath, {force: true}); + fs.rmSync(`${targetDbPath}-wal`, {force: true}); + fs.rmSync(`${targetDbPath}-shm`, {force: true}); + fs.copyFileSync(manifest.databaseSnapshot.snapshotPath, targetDbPath); + restoredDb = true; + } + + return { + snapshotId: manifest.snapshotId, + restoredAt: (options.now ?? new Date()).toISOString(), + restoredFilesystem, + restoredDatabase: { + targetPath: targetDbPath, + restored: restoredDb, + }, + }; +} + +export function describeEnterpriseMigrationRollback( + env: NodeJS.ProcessEnv = process.env, +): string { + const plan = resolveEnterpriseMigrationPlan(env); + if (!plan.enterpriseEnabled) { + return `Enterprise mode is disabled via ${ENTERPRISE_FEATURE_FLAG_ENV}; legacy filesystem storage is authoritative.`; + } + if (plan.phase === 'dual-write') { + return 'P-A rollback: stop enterprise mode or delete the SQLite DB; legacy filesystem data remains authoritative.'; + } + if (plan.phase === 'cutover') { + return `P-B rollback: set ${ENTERPRISE_MIGRATION_PHASE_ENV}=dual-write; keep the DB snapshot for investigation.`; + } + return 'P-C rollback: restore the pre-retirement filesystem snapshot and SQLite DB snapshot; reverse conversion is not promised.'; +} + +export const ENTERPRISE_MIGRATION_ENV_KEYS = [ + ENTERPRISE_FEATURE_FLAG_ENV, + ENTERPRISE_MIGRATION_PHASE_ENV, + ENTERPRISE_MIGRATION_SNAPSHOT_DIR_ENV, + ENTERPRISE_DB_PATH_ENV, + DATA_DIR_ENV, + LOGS_DIR_ENV, + PROVIDER_DATA_DIR_ENV, + UPLOAD_DIR_ENV, +] as const; diff --git a/backend/src/services/providerManager/providerStore.ts b/backend/src/services/providerManager/providerStore.ts index a5cd185d..6c4520eb 100644 --- a/backend/src/services/providerManager/providerStore.ts +++ b/backend/src/services/providerManager/providerStore.ts @@ -3,8 +3,12 @@ import * as fs from 'fs'; import * as path from 'path'; -import { resolveFeatureConfig } from '../../config'; import { openEnterpriseDb } from '../enterpriseDb'; +import { + enterpriseDbReadAuthorityEnabled, + enterpriseDbWritesEnabled, + legacyFilesystemWritesEnabled, +} from '../enterpriseMigration'; import type { ProviderConfig, ProviderConnection, ProviderScope } from './types'; import { LocalEncryptedSecretStore } from './localSecretStore'; @@ -53,7 +57,15 @@ const SENSITIVE_CONNECTION_FIELDS: Array = [ ]; function enterpriseProviderStoreEnabled(): boolean { - return resolveFeatureConfig(process.env).enterprise; + return enterpriseDbReadAuthorityEnabled(); +} + +function enterpriseProviderDbWritesEnabled(): boolean { + return enterpriseDbWritesEnabled(); +} + +function legacyProviderWritesEnabled(): boolean { + return legacyFilesystemWritesEnabled(); } function assertSafeScopeSegment(value: string, label: string): string { @@ -213,20 +225,24 @@ export class ProviderStore { } set(provider: ProviderConfig, scope?: ProviderScope): void { - if (enterpriseProviderStoreEnabled()) { + if (!enterpriseProviderStoreEnabled()) { + this.providers.set(provider.id, provider); + this.persist(); + } + if (enterpriseProviderDbWritesEnabled()) { this.setEnterprise(provider, scope); - return; } - this.providers.set(provider.id, provider); - this.persist(); } delete(id: string, scope?: ProviderScope): boolean { - if (enterpriseProviderStoreEnabled()) { - return this.deleteEnterprise(id, scope); + let deleted = false; + if (!enterpriseProviderStoreEnabled()) { + deleted = this.providers.delete(id); + if (deleted) this.persist(); + } + if (enterpriseProviderDbWritesEnabled()) { + deleted = this.deleteEnterprise(id, scope) || deleted; } - const deleted = this.providers.delete(id); - if (deleted) this.persist(); return deleted; } @@ -394,7 +410,7 @@ export class ProviderStore { } private persist(): void { - if (enterpriseProviderStoreEnabled()) return; + if (!legacyProviderWritesEnabled()) return; const dir = path.dirname(this.filePath); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); const tmp = `${this.filePath}.tmp`; diff --git a/backend/src/services/ragStore.ts b/backend/src/services/ragStore.ts index 45265856..9de8b806 100644 --- a/backend/src/services/ragStore.ts +++ b/backend/src/services/ragStore.ts @@ -33,7 +33,9 @@ import { makeSparkProvenance, } from '../types/sparkContracts'; import { + enterpriseKnowledgeDbWritesEnabled, enterpriseKnowledgeStoreEnabled, + legacyKnowledgeFilesystemWritesEnabled, type KnowledgeScope, getScopedKnowledgeRecord, listScopedKnowledgeRecords, @@ -154,7 +156,11 @@ export class RagStore { `License required for source kind '${chunk.kind}' but missing on chunk '${chunk.chunkId}'`, ); } - if (enterpriseKnowledgeStoreEnabled()) { + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.chunks.set(chunk.chunkId, chunk); + this.persist(); + } + if (enterpriseKnowledgeDbWritesEnabled()) { upsertScopedKnowledgeRecord( KNOWLEDGE_KIND, chunk.chunkId, @@ -163,21 +169,22 @@ export class RagStore { scope, {createdAt: chunk.indexedAt, updatedAt: chunk.indexedAt}, ); - return; } - this.chunks.set(chunk.chunkId, chunk); - this.persist(); } /** Remove a chunk. Returns whether anything was actually removed. */ removeChunk(chunkId: string, scope?: KnowledgeScope): boolean { - if (enterpriseKnowledgeStoreEnabled()) { - return removeScopedKnowledgeRecord(KNOWLEDGE_KIND, chunkId, scope); + let removed = false; + if (enterpriseKnowledgeDbWritesEnabled()) { + removed = removeScopedKnowledgeRecord(KNOWLEDGE_KIND, chunkId, scope) || removed; } - this.load(); - const had = this.chunks.delete(chunkId); - if (had) this.persist(); - return had; + if (legacyKnowledgeFilesystemWritesEnabled()) { + this.load(); + const had = this.chunks.delete(chunkId); + if (had) this.persist(); + removed = had || removed; + } + return removed; } /** Get a chunk by id, or undefined when absent. */ diff --git a/backend/src/services/scopedKnowledgeStore.ts b/backend/src/services/scopedKnowledgeStore.ts index 86a20725..ee517e2a 100644 --- a/backend/src/services/scopedKnowledgeStore.ts +++ b/backend/src/services/scopedKnowledgeStore.ts @@ -5,10 +5,14 @@ import crypto from 'crypto'; import type Database from 'better-sqlite3'; -import { resolveFeatureConfig } from '../config'; import type { RequestContext } from '../middleware/auth'; import { openEnterpriseDb } from './enterpriseDb'; import { createEnterpriseWorkspaceRepository } from './enterpriseRepository'; +import { + enterpriseDbReadAuthorityEnabled, + enterpriseDbWritesEnabled, + legacyFilesystemWritesEnabled, +} from './enterpriseMigration'; const DEFAULT_TENANT_ID = 'default-dev-tenant'; const DEFAULT_WORKSPACE_ID = 'default-workspace'; @@ -77,7 +81,19 @@ interface UpsertOptions { export function enterpriseKnowledgeStoreEnabled( env: NodeJS.ProcessEnv = process.env, ): boolean { - return resolveFeatureConfig(env).enterprise; + return enterpriseDbReadAuthorityEnabled(env); +} + +export function enterpriseKnowledgeDbWritesEnabled( + env: NodeJS.ProcessEnv = process.env, +): boolean { + return enterpriseDbWritesEnabled(env); +} + +export function legacyKnowledgeFilesystemWritesEnabled( + env: NodeJS.ProcessEnv = process.env, +): boolean { + return legacyFilesystemWritesEnabled(env); } export function knowledgeScopeFromRequestContext( diff --git a/backend/src/services/traceMetadataStore.ts b/backend/src/services/traceMetadataStore.ts index 26ffe72f..a8b68100 100644 --- a/backend/src/services/traceMetadataStore.ts +++ b/backend/src/services/traceMetadataStore.ts @@ -5,9 +5,13 @@ import path from 'path'; import fs from 'fs/promises'; import type Database from 'better-sqlite3'; -import { resolveFeatureConfig } from '../config'; import type { RequestContext } from '../middleware/auth'; import { openEnterpriseDb } from './enterpriseDb'; +import { + enterpriseDbReadAuthorityEnabled, + enterpriseDbWritesEnabled, + legacyFilesystemWritesEnabled, +} from './enterpriseMigration'; import { ownerFieldsFromContext, type ResourceOwnerFields, @@ -55,7 +59,15 @@ export function getTracesDir(): string { } function enterpriseTraceStoreEnabled(env: NodeJS.ProcessEnv = process.env): boolean { - return resolveFeatureConfig(env).enterprise; + return enterpriseDbReadAuthorityEnabled(env); +} + +function enterpriseTraceDbWritesEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return enterpriseDbWritesEnabled(env); +} + +function legacyTraceMetadataWritesEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return legacyFilesystemWritesEnabled(env); } function assertSafePathSegment(value: string, label: string): string { @@ -198,6 +210,19 @@ function enterpriseLocalPathForMetadata(metadata: TraceMetadata): string { if (metadata.externalRpc && typeof metadata.port === 'number') { return `external-rpc:${metadata.port}`; } + if ( + enterpriseTraceStoreEnabled() && + metadata.tenantId && + metadata.workspaceId + ) { + return path.join( + resolveEnterpriseDataRoot(), + assertSafePathSegment(metadata.tenantId, 'tenant id'), + assertSafePathSegment(metadata.workspaceId, 'workspace id'), + 'traces', + `${assertSafePathSegment(metadata.id, 'trace id')}.trace`, + ); + } const fallbackPath = getTraceFilePath(metadata.id); if (!fallbackPath) throw new Error(`Unsafe trace id: ${metadata.id}`); return fallbackPath; @@ -249,10 +274,10 @@ export async function writeTraceMetadata(metadata: TraceMetadata): Promise if (!isSafeTraceId(metadata.id)) { throw new Error(`Unsafe trace id: ${metadata.id}`); } - if (enterpriseTraceStoreEnabled()) { + if (enterpriseTraceDbWritesEnabled()) { writeEnterpriseTraceMetadata(metadata); - return; } + if (!legacyTraceMetadataWritesEnabled()) return; const tracesDir = getTracesDir(); await fs.mkdir(tracesDir, { recursive: true }); await fs.writeFile( @@ -351,12 +376,12 @@ export async function listTraceMetadataForContext(context: RequestContext): Prom export async function deleteTraceMetadata(traceId: string): Promise { if (!isSafeTraceId(traceId)) return; - if (enterpriseTraceStoreEnabled()) { + if (enterpriseTraceDbWritesEnabled()) { withEnterpriseTraceDb((db) => { db.prepare('DELETE FROM trace_assets WHERE id = ?').run(traceId); }); - return; } + if (!legacyTraceMetadataWritesEnabled()) return; const metadataPath = getTraceMetadataPath(traceId); if (!metadataPath) return; try { diff --git a/docs/features/enterprise-multi-tenant/README.md b/docs/features/enterprise-multi-tenant/README.md index 4837e95e..02a17b11 100644 --- a/docs/features/enterprise-multi-tenant/README.md +++ b/docs/features/enterprise-multi-tenant/README.md @@ -46,7 +46,7 @@ - [x] 3.5 `logs/claude_session_map.json` 迁到 `runtime_snapshots` - [x] 3.6 provider 从 `data/providers.json` 迁到 DB metadata + encrypted SecretStore - [x] 3.7 Memory / RAG / Case / Baseline 表加 scope（§14.1，先 filter 后语义召回） -- [ ] 3.8 双写 → 切读 → 退役 三阶段（§17），每阶段都能回滚；准备 filesystem + DB snapshot +- [x] 3.8 双写 → 切读 → 退役 三阶段（§17），每阶段都能回滚；准备 filesystem + DB snapshot - [ ] 3.9 SecretStore：libsodium 加密 + OS keyring 解 master key + secret rotation + 读取审计 - [ ] 3.10 集成测试：backend restart 后 session/report/trace metadata 可恢复 @@ -100,7 +100,7 @@ - [ ] 6.6 Runtime：lease acquire / release / heartbeat / stale / crash recovery - [ ] 6.7 Persistence：backend restart / queue shadow 恢复 / DB reconnect / SecretStore failure - [ ] 6.8 SSE：fetch-stream reconnect / cursor replay / terminal event 落库 -- [ ] 6.9 Migration：dry-run / 双写 / 切读 / 退役 / snapshot restore +- [x] 6.9 Migration：dry-run / 双写 / 切读 / 退役 / snapshot restore - [ ] 6.10 Regression：每次 PR 跑 `cd backend && npm run test:scene-trace-regression` - [ ] 6.11 PR Gate：合入前 `npm run verify:pr` 通过 @@ -991,6 +991,12 @@ request delete 旧数据默认迁移到 `default-dev-tenant/default-workspace` 或管理员指定 tenant/workspace。 +当前实现入口： + +- 阶段开关：`SMARTPERFETTO_ENTERPRISE_MIGRATION_PHASE=dual-write|cutover|retired`。企业模式关闭时仍走 legacy filesystem；企业模式开启但未显式设置阶段时默认 `cutover`，保持 DB 权威路径。 +- dry-run / snapshot / restore：`cd backend && npm run enterprise:migration -- --dry-run`、`--snapshot`、`--restore `。 +- P-A `dual-write`：trace/report/provider/runtime snapshot/RAG-baseline-case-project-memory 写旧文件与 DB，读取旧文件；P-B/P-C 读取 DB，旧文件不再写入。 + ## 18. 四条并行开发主线 原 P0 到 P6 线性阶段改为 4 条并行主线。观测、审计和并发回归贯穿所有主线。