From 02830c92e6b4289bc3a281736c309bcd717ac1e1 Mon Sep 17 00:00:00 2001 From: "ca.mathieu" Date: Wed, 1 Apr 2026 19:03:11 +0200 Subject: [PATCH] feat(server): add plikd migrate command for live backend-to-backend migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new `plikd migrate` CLI command that migrates metadata and file data directly between backends without intermediate archive files. ## Metadata migration (server/metadata/migrator.go) - Streams all records from source to destination in FK-safe order: users → tokens → uploads (unscoped) → files → settings - Uses src.log.Infof/Warningf for progress (consistent with exporter/importer) - Supports --ignore-errors to skip individual record failures ## Data migration (server/data/migrator.go) - Parallel worker pool (default: 4, configurable via --workers) - Uses context cancellation for clean fatal-error propagation (no panics) - Copies only `uploaded` and `removed` files (both have backing data) - Skips `missing`, `uploading`, `deleted` (no data in backend) - Supports --dry-run and --ignore-errors ## CLI command (server/cmd/migrate.go) - Flags: --to (required), --metadata-only, --data-only, --workers, --dry-run, --ignore-errors - --data-only and --metadata-only are mutually exclusive - Dry-run enumerates and prints all items without writing; errors are reported ## Tests - server/metadata/migrator_test.go: basic migration, soft-deleted uploads, ignore-errors - server/data/migrator_test.go: blob streaming, status filtering, dry-run, ignore-errors, multi-worker concurrency - testing/migrate/run.sh: e2e test using `plikd fakedb` to seed a source SQLite DB, migrate to destination, verify record counts, dry-run writes nothing, re-run is idempotent ## Docs - docs/operations/migration.md: full migration guide with 4 real-world scenarios - Side-nav entry in docs/.vitepress/config.js - Cross-link in docs/backends/metadata.md - server/ARCHITECTURE.md + testing/ARCHITECTURE.md updated - AGENTS.md Key Files table updated --- AGENTS.md | 3 + docs/.vitepress/config.js | 1 + docs/backends/metadata.md | 5 +- docs/operations/migration.md | 120 +++++++++++++++++++ server/ARCHITECTURE.md | 13 +- server/cmd/migrate.go | 194 ++++++++++++++++++++++++++++++ server/data/migrator.go | 154 ++++++++++++++++++++++++ server/data/migrator_test.go | 184 ++++++++++++++++++++++++++++ server/metadata/migrator.go | 123 +++++++++++++++++++ server/metadata/migrator_test.go | 128 ++++++++++++++++++++ testing/ARCHITECTURE.md | 2 + testing/migrate/run.sh | 199 +++++++++++++++++++++++++++++++ testing/test_backends.sh | 1 + 13 files changed, 1125 insertions(+), 2 deletions(-) create mode 100644 docs/operations/migration.md create mode 100644 server/cmd/migrate.go create mode 100644 server/data/migrator.go create mode 100644 server/data/migrator_test.go create mode 100644 server/metadata/migrator.go create mode 100644 server/metadata/migrator_test.go create mode 100755 testing/migrate/run.sh diff --git a/AGENTS.md b/AGENTS.md index 6ba8ee96..327df471 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,6 +113,9 @@ make vuln # govulncheck (report only) | `webapp/src/components/UploadControls.vue` | Shared sort/order/badge-filter control bar used by AdminView and HomeView | | `server/common/feature_flags.go` | Feature flag types (`disabled`/`enabled`/`default`/`forced`) | | `server/server/server.go` | `ensureDefaultAdmin()` — idempotent bootstrap of `DefaultAdminLogin`/`DefaultAdminPassword` on startup | +| `server/cmd/migrate.go` | `plikd migrate` cobra command — orchestrates backend-to-backend migration | +| `server/metadata/migrator.go` | `Migrate(src, dst, opts)` — streams all metadata from one backend to another | +| `server/data/migrator.go` | `MigrateFiles(src, dst, meta, opts)` — parallel file blob copy worker pool | ## Conventions diff --git a/docs/.vitepress/config.js b/docs/.vitepress/config.js index 0e09b5df..17c34612 100644 --- a/docs/.vitepress/config.js +++ b/docs/.vitepress/config.js @@ -74,6 +74,7 @@ export default withMermaid(defineConfig({ { text: 'Reverse Proxy', link: '/operations/reverse-proxy' }, { text: 'Server CLI', link: '/operations/server-cli' }, { text: 'Import / Export', link: '/operations/import-export' }, + { text: 'Migration', link: '/operations/migration' }, { text: 'Cross Compilation', link: '/operations/cross-compilation' }, ], }, diff --git a/docs/backends/metadata.md b/docs/backends/metadata.md index 12ffd2c2..37e58a3a 100644 --- a/docs/backends/metadata.md +++ b/docs/backends/metadata.md @@ -62,4 +62,7 @@ Plik uses [gormigrate](https://github.com/go-gormigrate/gormigrate) for automati ## Migrating Between Backends -To migrate data between different metadata backends (e.g. SQLite → PostgreSQL), use the `plikd export` and `plikd import` commands. See the [Import / Export](/operations/import-export) guide for details. +Plik provides two approaches for migrating between metadata backends: + +- **`plikd export` / `plikd import`** — Export metadata to a portable file and import it into a new backend. Good for offline backups or scheduled migrations. See the [Import / Export](/operations/import-export) guide. +- **`plikd migrate`** — Live, direct backend-to-backend migration that also handles file data blobs in parallel. No intermediate files needed. See the [Migration](/operations/migration) guide. diff --git a/docs/operations/migration.md b/docs/operations/migration.md new file mode 100644 index 00000000..9a6c490b --- /dev/null +++ b/docs/operations/migration.md @@ -0,0 +1,120 @@ +# Migration Guide + +Plik supports live, direct backend-to-backend migration via the `plikd migrate` command. This lets you move all data and/or metadata from one backend configuration to another **without creating intermediate archive files**. File data is streamed directly from the source backend to the target backend. + +## Overview + +The `plikd migrate` command: + +- Reads the source backends from the current `plikd.cfg` (same config used to start the server) +- Reads the **target** backends from a second config file (`--to` flag) +- Migrates metadata (users, tokens, uploads, files, settings) in order, respecting foreign key dependencies +- Migrates file blobs in parallel (configurable workers, default: 4) +- Supports dry-run mode, error tolerance, and selective migration + +> [!IMPORTANT] +> Stop the Plik server before running `plikd migrate`. Running a migration against a live server may produce inconsistent results. + +## Basic Syntax + +```sh +plikd migrate --to /path/to/target.cfg [options] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--to` | *(required)* | Path to the target `plikd.cfg` | +| `--metadata-only` | false | Only migrate metadata (skip file blobs) | +| `--data-only` | false | Only migrate file blobs (skip metadata) | +| `--workers` | 4 | Number of parallel file copy workers | +| `--dry-run` | false | Print every item that would be migrated, write nothing | +| `--ignore-errors` | false | Log errors per item and continue (don't abort) | + +## Common Scenarios + +### 1. SQLite → PostgreSQL (metadata only) + +Your current `plikd.cfg`: +```toml +MetadataBackend = "sqlite3" +[MetadataBackendConfig] + ConnectionString = "/home/plik/server/db/plik.db" +``` + +Create a `plikd-pg.cfg` with just the fields that differ: +```toml +MetadataBackend = "postgres" +[MetadataBackendConfig] + ConnectionString = "host=localhost user=plik password=secret dbname=plik sslmode=disable" +``` + +Run: +```sh +plikd migrate --to plikd-pg.cfg --metadata-only +``` + +### 2. Local Files → S3 (data only) + +You already have PostgreSQL running. Create `plikd-s3.cfg`: +```toml +DataBackend = "s3" +[DataBackendConfig] + Bucket = "my-plik-bucket" + Region = "us-east-1" + # Credentials via AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars +``` + +Run: +```sh +plikd migrate --to plikd-s3.cfg --data-only --workers 8 +``` + +### 3. Full Migration (SQLite + Local → PostgreSQL + S3) + +1. Prepare `plikd-new.cfg` with both new backends configured. +2. First preview with dry-run: + ```sh + plikd migrate --to plikd-new.cfg --dry-run + ``` +3. Run the full migration: + ```sh + plikd migrate --to plikd-new.cfg + ``` +4. Update your main `plikd.cfg` to use the new backends and start the server. + +### 4. Resuming a Failed Migration + +If a migration is interrupted, re-run with `--ignore-errors` to skip already-migrated records (unique constraint violations) and continue: +```sh +plikd migrate --to plikd-new.cfg --ignore-errors +``` + +> [!WARNING] +> With `--ignore-errors`, migration errors are silently skipped and logged to stdout. Always review the output to ensure no critical data was lost. + +## Migration Order + +Metadata is migrated in dependency order to avoid foreign key violations: + +1. **Users** +2. **Tokens** (depend on users) +3. **Uploads** (including soft-deleted, to maintain FK integrity with files) +4. **Files** (metadata records) +5. **Settings** + +File **data blobs** are migrated after metadata, in parallel. Files with status `uploaded` or `removed` are copied (both still exist in the data backend). Files with status `missing`, `uploading`, or `deleted` are skipped. + +## Verifying After Migration + +After migration, verify the target by querying the HTTP API (with the server running against the new config): + +```sh +# Check server stats via API +curl http://localhost:8080/version +``` + +Or query record counts directly in your database. For example with SQLite: + +```sh +sqlite3 /path/to/new/plik.db "SELECT COUNT(*) FROM uploads; SELECT COUNT(*) FROM files;" +``` diff --git a/server/ARCHITECTURE.md b/server/ARCHITECTURE.md index b49c961a..d909673d 100644 --- a/server/ARCHITECTURE.md +++ b/server/ARCHITECTURE.md @@ -35,6 +35,7 @@ The server binary `plikd` uses [cobra](https://github.com/spf13/cobra) for CLI m | `clean.go` | `plikd clean` | Run metadata cleanup | | `import.go` | `plikd import [input-file]` | Import metadata from gob + Snappy binary | | `export.go` | `plikd export [output-file]` | Export metadata to gob + Snappy binary | +| `migrate.go` | `plikd migrate --to ` | Live backend-to-backend migration (metadata + data) | Config loading order: `--config` flag → `PLIKD_CONFIG` env → `./plikd.cfg` → `/etc/plikd.cfg`. @@ -143,6 +144,7 @@ Uses GORM with gormigrate for schema management across SQLite3, PostgreSQL, and | `stats.go` | Aggregate statistics queries | | `exporter.go` | gob + Snappy export of all metadata | | `importer.go` | gob + Snappy import | +| `migrator.go` | Live metadata migration: `Migrate(src, dst, opts)` — streams all records using `ForEach*`/`Create*`, supports `--ignore-errors` | ### Import / Export @@ -151,10 +153,19 @@ The `plikd export` and `plikd import` commands dump and restore all metadata (us - **Format**: Go [gob](https://pkg.go.dev/encoding/gob) encoding compressed with [Snappy](https://github.com/golang/snappy). Architecture-independent (portable across `amd64`/`arm64`), streaming (constant memory), Go-specific (not human-readable). - **Export order**: users → tokens → uploads (including soft-deleted) → files → settings. CLI auth sessions are intentionally excluded (ephemeral). - **Import**: decodes sequentially, calls `Create*` on the metadata backend. Supports `--ignore-errors` to skip problematic records. -- **Use cases**: backend migration (e.g. SQLite → PostgreSQL), backups, disaster recovery. +- **Use cases**: backend backups, disaster recovery. > **Note**: Only metadata is exported — file data in the data backend must be migrated separately. +### Live Migration (`plikd migrate`) + +The `plikd migrate --to ` command performs a **direct, live backend-to-backend migration** of both metadata and file data. No intermediate files are created. + +- **Metadata** (`metadata/migrator.go`): Streaming `ForEach*` + `Create*` pipeline, same migration order as export/import. CLI auth sessions are excluded (ephemeral). +- **File data** (`data/migrator.go`): Parallel worker pool (`--workers`, default: 4) reading from `src.GetFile()` and writing to `dst.AddFile()`. Copies files with status `uploaded` or `removed` (both still exist in storage); skips `missing`, `uploading`, `deleted`. +- **Dry-run**: Lists every individual item that would be migrated, writes nothing. +- **Target config**: Full `plikd.cfg` via `--to`; only the backend fields need to differ from the source config. + ### Migration Dump Tests When adding a new migration, `TestGenerateSQLDump` and `TestGenerateExport` in `migrations_test.go` auto-generate dump files on first run. Each dump captures the full DB state after all migrations and test data have been applied. `TestMigrationsFromSQLDumps` and `TestLoadExports` then load **all** existing dumps and verify migrations can be replayed forward to the current schema. diff --git a/server/cmd/migrate.go b/server/cmd/migrate.go new file mode 100644 index 00000000..57cb2580 --- /dev/null +++ b/server/cmd/migrate.go @@ -0,0 +1,194 @@ +package cmd + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/root-gg/plik/server/common" + "github.com/root-gg/plik/server/data" + "github.com/root-gg/plik/server/metadata" + "github.com/root-gg/plik/server/server" +) + +type migrateFlagParams struct { + to string + dataOnly bool + metadataOnly bool + ignoreErrors bool + workers int + dryRun bool +} + +var migrateParams = migrateFlagParams{} + +// migrateCmd migrates metadata and/or data from one backend to another +var migrateCmd = &cobra.Command{ + Use: "migrate", + Short: "Migrate metadata and/or data backend", + Long: `Migrate metadata (users, tokens, uploads, files, settings) and file data +from the configured source backends to the backends configured in the target config file. + +Both the source plikd.cfg (--config flag or default location) and the target plikd.cfg +(--to flag) are required. The target config only needs to specify the backend fields +that differ from the source. + +Typical use cases: + - SQLite → PostgreSQL (metadata only): plikd migrate --to new.cfg --metadata-only + - Local files → S3 (data only): plikd migrate --to new.cfg --data-only + - Full migration: plikd migrate --to new.cfg`, + Example: ` plikd migrate --to /etc/plikd-new.cfg + plikd migrate --to /etc/plikd-new.cfg --metadata-only + plikd migrate --to /etc/plikd-new.cfg --data-only --workers 8 + plikd migrate --to /etc/plikd-new.cfg --dry-run + plikd migrate --to /etc/plikd-new.cfg --ignore-errors`, + Run: migrateBackends, +} + +func init() { + migrateCmd.Flags().StringVar(&migrateParams.to, "to", "", "path to target plikd.cfg (required)") + migrateCmd.Flags().BoolVar(&migrateParams.dataOnly, "data-only", false, "migrate file data only (skip metadata)") + migrateCmd.Flags().BoolVar(&migrateParams.metadataOnly, "metadata-only", false, "migrate metadata only (skip file data)") + migrateCmd.Flags().BoolVar(&migrateParams.ignoreErrors, "ignore-errors", false, "continue on individual record/file errors") + migrateCmd.Flags().IntVar(&migrateParams.workers, "workers", 4, "number of parallel file copy workers") + migrateCmd.Flags().BoolVar(&migrateParams.dryRun, "dry-run", false, "print what would be migrated without writing anything") + _ = migrateCmd.MarkFlagRequired("to") + rootCmd.AddCommand(migrateCmd) +} + +func migrateBackends(cmd *cobra.Command, args []string) { + if migrateParams.dataOnly && migrateParams.metadataOnly { + fmt.Println("--data-only and --metadata-only are mutually exclusive") + os.Exit(1) + } + + start := time.Now() + + if migrateParams.dryRun { + fmt.Println("[dry-run mode — nothing will be written]") + } + + // Load target config + targetConfig, err := common.LoadConfiguration(migrateParams.to) + if err != nil { + fmt.Printf("unable to load target config %s: %s\n", migrateParams.to, err) + os.Exit(1) + } + + // Initialize source backends (from primary config, already loaded by initConfig) + initializeMetadataBackend() + srcMeta := metadataBackend + + // Initialize target metadata backend + var dstMeta *metadata.Backend + if !migrateParams.dataOnly { + dstMeta, err = server.NewMetadataBackend(targetConfig.MetadataBackendConfig, targetConfig.NewLogger()) + if err != nil { + fmt.Printf("unable to initialize target metadata backend: %s\n", err) + os.Exit(1) + } + + fmt.Printf("Migrating metadata: %s → %s\n", + config.MetadataBackendConfig["Driver"], + targetConfig.MetadataBackendConfig["Driver"]) + + if !migrateParams.dryRun { + metaOpts := &metadata.MigrateOptions{ + IgnoreErrors: migrateParams.ignoreErrors, + } + metaStats, err := metadata.Migrate(srcMeta, dstMeta, metaOpts) + if err != nil { + fmt.Printf("metadata migration failed: %s\n", err) + os.Exit(1) + } + totalMetaErrors := metaStats.UserErrors + metaStats.TokenErrors + + metaStats.UploadErrors + metaStats.FileErrors + metaStats.SettingErrors + fmt.Printf("metadata migration complete: %d users, %d tokens, %d uploads, %d files, %d settings (%d errors)\n", + metaStats.Users, metaStats.Tokens, metaStats.Uploads, metaStats.Files, metaStats.Settings, + totalMetaErrors) + } else { + // Dry-run: enumerate and print all metadata items that would be migrated + fmt.Println("[dry-run] would migrate:") + var dryRunErr error + dryRunErr = srcMeta.ForEachUsers(func(u *common.User) error { + fmt.Printf(" user %s (%s)\n", u.ID, u.Login) + return nil + }) + if dryRunErr != nil { + fmt.Printf("error enumerating users: %s\n", dryRunErr) + os.Exit(1) + } + dryRunErr = srcMeta.ForEachToken(func(t *common.Token) error { + fmt.Printf(" token %s (user: %s)\n", t.Token, t.UserID) + return nil + }) + if dryRunErr != nil { + fmt.Printf("error enumerating tokens: %s\n", dryRunErr) + os.Exit(1) + } + dryRunErr = srcMeta.ForEachUploadUnscoped(func(u *common.Upload) error { + fmt.Printf(" upload %s\n", u.ID) + return nil + }) + if dryRunErr != nil { + fmt.Printf("error enumerating uploads: %s\n", dryRunErr) + os.Exit(1) + } + dryRunErr = srcMeta.ForEachFile(func(f *common.File) error { + fmt.Printf(" file %s (%s, %s)\n", f.ID, f.Name, f.Status) + return nil + }) + if dryRunErr != nil { + fmt.Printf("error enumerating files: %s\n", dryRunErr) + os.Exit(1) + } + dryRunErr = srcMeta.ForEachSetting(func(s *common.Setting) error { + fmt.Printf(" setting %s\n", s.Key) + return nil + }) + if dryRunErr != nil { + fmt.Printf("error enumerating settings: %s\n", dryRunErr) + os.Exit(1) + } + } + } + + // Migrate data + if !migrateParams.metadataOnly { + initializeDataBackend() + srcData := dataBackend + + dstDataBackend, err := server.NewDataBackend(targetConfig.DataBackend, targetConfig.DataBackendConfig) + if err != nil { + fmt.Printf("unable to initialize target data backend: %s\n", err) + os.Exit(1) + } + + fmt.Printf("Migrating file data: %s → %s\n", config.DataBackend, targetConfig.DataBackend) + + dataOpts := &data.MigrateOptions{ + IgnoreErrors: migrateParams.ignoreErrors, + Workers: migrateParams.workers, + DryRun: migrateParams.dryRun, + } + + // Use source metadata to enumerate files (even if we just migrated metadata, + // the source metadata backend is always the canonical file list) + dataStats, err := data.MigrateFiles(srcData, dstDataBackend, srcMeta, dataOpts) + if err != nil { + fmt.Printf("data migration failed: %s\n", err) + os.Exit(1) + } + + prefix := "" + if migrateParams.dryRun { + prefix = "[dry-run] " + } + fmt.Printf("%sdata migration complete: %d files copied, %d skipped, %d errors, %d bytes\n", + prefix, dataStats.Copied, dataStats.Skipped, dataStats.Errors, dataStats.Bytes) + } + + fmt.Printf("migration completed in %s\n", time.Since(start).Round(time.Millisecond)) +} diff --git a/server/data/migrator.go b/server/data/migrator.go new file mode 100644 index 00000000..065069b3 --- /dev/null +++ b/server/data/migrator.go @@ -0,0 +1,154 @@ +package data + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "github.com/root-gg/plik/server/common" + "github.com/root-gg/plik/server/metadata" +) + +// MigrateOptions controls the behavior of MigrateFiles +type MigrateOptions struct { + IgnoreErrors bool + Workers int + DryRun bool +} + +// MigrateStats holds counters returned by MigrateFiles +type MigrateStats struct { + Copied int64 + Skipped int64 + Errors int64 + Bytes int64 +} + +// MigrateFiles copies file blobs for all uploaded/removed files from src to dst. +// It uses a configurable worker pool (default: 4) for parallel transfers. +// Files with status missing, uploading, or deleted are skipped (no data in backend). +func MigrateFiles(src Backend, dst Backend, metaBackend *metadata.Backend, options *MigrateOptions) (stats MigrateStats, err error) { + if options == nil { + options = &MigrateOptions{} + } + workers := options.Workers + if workers <= 0 { + workers = 4 + } + + type job struct { + file *common.File + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + jobs := make(chan job, workers*2) + var wg sync.WaitGroup + + var copied, skipped, errCount, byteCount int64 + + // fatalErr holds the first non-ignorable worker error. + var fatalErr error + var fatalMu sync.Mutex + + workerFn := func() { + defer wg.Done() + for j := range jobs { + file := j.file + + // Only copy files that actually have data in the backend + if file.Status != common.FileUploaded && file.Status != common.FileRemoved { + atomic.AddInt64(&skipped, 1) + continue + } + + if options.DryRun { + fmt.Printf("[dry-run] would copy file %s (%s, %d bytes, status: %s)\n", + file.ID, file.Name, file.Size, file.Status) + atomic.AddInt64(&copied, 1) + atomic.AddInt64(&byteCount, file.Size) + continue + } + + reader, e := src.GetFile(file) + if e != nil { + atomic.AddInt64(&errCount, 1) + msg := fmt.Sprintf("error reading file %s (%s): %s", file.ID, file.Name, e) + if options.IgnoreErrors { + fmt.Println(msg) + continue + } + fatalMu.Lock() + if fatalErr == nil { + fatalErr = fmt.Errorf("%s", msg) + } + fatalMu.Unlock() + cancel() + return + } + + // Caller (us) is responsible for closing the reader — AddFile does not close it. + e = dst.AddFile(file, reader) + _ = reader.Close() + if e != nil { + atomic.AddInt64(&errCount, 1) + msg := fmt.Sprintf("error writing file %s (%s): %s", file.ID, file.Name, e) + if options.IgnoreErrors { + fmt.Println(msg) + continue + } + fatalMu.Lock() + if fatalErr == nil { + fatalErr = fmt.Errorf("%s", msg) + } + fatalMu.Unlock() + cancel() + return + } + + atomic.AddInt64(&copied, 1) + atomic.AddInt64(&byteCount, file.Size) + } + } + + for range workers { + wg.Add(1) + go workerFn() + } + + // Enumerate all files and feed to workers; stop early if a fatal error is signalled. + iterErr := metaBackend.ForEachFile(func(file *common.File) error { + select { + case <-ctx.Done(): + fatalMu.Lock() + e := fatalErr + fatalMu.Unlock() + return e + case jobs <- job{file: file}: + return nil + } + }) + + close(jobs) + wg.Wait() + + stats = MigrateStats{ + Copied: atomic.LoadInt64(&copied), + Skipped: atomic.LoadInt64(&skipped), + Errors: atomic.LoadInt64(&errCount), + Bytes: atomic.LoadInt64(&byteCount), + } + + if iterErr != nil { + return stats, iterErr + } + fatalMu.Lock() + e := fatalErr + fatalMu.Unlock() + if e != nil { + return stats, e + } + return stats, nil +} diff --git a/server/data/migrator_test.go b/server/data/migrator_test.go new file mode 100644 index 00000000..ca325b11 --- /dev/null +++ b/server/data/migrator_test.go @@ -0,0 +1,184 @@ +package data_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/root-gg/logger" + "github.com/stretchr/testify/require" + + "github.com/root-gg/plik/server/common" + "github.com/root-gg/plik/server/data" + data_testing "github.com/root-gg/plik/server/data/testing" + "github.com/root-gg/plik/server/metadata" +) + +// newTestMeta creates a fresh SQLite-backed metadata backend for data migration tests. +func newTestMeta(t *testing.T) *metadata.Backend { + t.Helper() + cfg := &metadata.Config{ + Driver: "sqlite3", + ConnectionString: "/tmp/plik.data.migrate.test.db", + EraseFirst: true, + } + b, err := metadata.NewBackend(cfg, logger.NewLogger()) + require.NoError(t, err, "unable to create test metadata backend") + t.Cleanup(func() { _ = b.Shutdown() }) + return b +} + +// seedUploadedFile creates an upload + file record with status uploaded, and puts content in the data backend. +// Note: CreateUpload cascades file creation via GORM associations, so we must not call CreateFile separately. +func seedUploadedFile(t *testing.T, mb *metadata.Backend, src data.Backend, content string) (*common.Upload, *common.File) { + t.Helper() + upload := &common.Upload{} + file := upload.NewFile() + file.Status = common.FileUploaded + file.Size = int64(len(content)) + upload.InitializeForTests() + err := mb.CreateUpload(upload) + require.NoError(t, err) + // File is already created by CreateUpload via GORM cascading (upload.Files) + err = src.AddFile(file, strings.NewReader(content)) + require.NoError(t, err) + return upload, file +} + +func TestMigrateFiles_Basic(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + _, file := seedUploadedFile(t, mb, src, "hello migration") + + stats, err := data.MigrateFiles(src, dst, mb, nil) + require.NoError(t, err) + + require.Equal(t, int64(1), stats.Copied, "expected 1 file copied") + require.Equal(t, int64(0), stats.Errors) + require.Equal(t, int64(len("hello migration")), stats.Bytes) + + // Verify data in destination + reader, err := dst.GetFile(file) + require.NoError(t, err) + buf := make([]byte, 100) + n, _ := reader.Read(buf) + require.Equal(t, "hello migration", string(buf[:n])) +} + +func TestMigrateFiles_CopiesRemovedFiles(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + upload := &common.Upload{} + file := upload.NewFile() + file.Status = common.FileRemoved // still has data in backend + file.Size = int64(len("removed data")) + upload.InitializeForTests() + err := mb.CreateUpload(upload) + require.NoError(t, err) + // File is already created by CreateUpload via GORM cascading + err = src.AddFile(file, strings.NewReader("removed data")) + require.NoError(t, err) + + stats, err := data.MigrateFiles(src, dst, mb, nil) + require.NoError(t, err) + + require.Equal(t, int64(1), stats.Copied, "removed file should be copied") + require.Equal(t, int64(0), stats.Skipped) +} + +func TestMigrateFiles_SkipsMissingAndDeleted(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + for _, status := range []string{common.FileMissing, common.FileDeleted, common.FileUploading} { + upload := &common.Upload{} + f := upload.NewFile() + f.Status = status + upload.InitializeForTests() + err := mb.CreateUpload(upload) + require.NoError(t, err) + // No data added to src — these files have no backing storage + } + + stats, err := data.MigrateFiles(src, dst, mb, nil) + require.NoError(t, err) + + require.Equal(t, int64(0), stats.Copied) + require.Equal(t, int64(3), stats.Skipped, "missing/deleted/uploading should be skipped") +} + +func TestMigrateFiles_DryRun(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + _, _ = seedUploadedFile(t, mb, src, "dry run content") + + opts := &data.MigrateOptions{DryRun: true} + stats, err := data.MigrateFiles(src, dst, mb, opts) + require.NoError(t, err) + + require.Equal(t, int64(1), stats.Copied, "dry-run should still count") + // Destination should be empty — nothing was actually written + require.Empty(t, dst.GetFiles(), "dry-run must not write to destination") +} + +func TestMigrateFiles_IgnoreErrors(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + // Seed two files + _, file1 := seedUploadedFile(t, mb, src, "file one") + _, file2 := seedUploadedFile(t, mb, src, "file two") + + // Simulate src error for file1 by overwriting its content with a sentinel + // Use a backend that errors on GetFile for file1 specifically + // Instead: pre-add file1 to dst so AddFile returns "file exists" error + err := dst.AddFile(file1, bytes.NewReader([]byte("pre-exists"))) + require.NoError(t, err) + + // Without ignore-errors, should fail + _, err = data.MigrateFiles(src, dst, mb, &data.MigrateOptions{IgnoreErrors: false, Workers: 1}) + require.Error(t, err) + + // Reset dst (file2 should be written, file1 conflicts) + dst = data_testing.NewBackend() + err = dst.AddFile(file1, bytes.NewReader([]byte("pre-exists"))) + require.NoError(t, err) + + // With ignore-errors, should succeed + stats, err := data.MigrateFiles(src, dst, mb, &data.MigrateOptions{IgnoreErrors: true, Workers: 1}) + require.NoError(t, err) + + require.Equal(t, int64(1), stats.Copied, "file2 should be copied") + require.Equal(t, int64(1), stats.Errors, "file1 conflict should be counted") + + // Verify file2 was copied + reader, err := dst.GetFile(file2) + require.NoError(t, err) + buf := make([]byte, 100) + n, _ := reader.Read(buf) + require.Equal(t, "file two", string(buf[:n])) +} + +func TestMigrateFiles_MultipleWorkers(t *testing.T) { + mb := newTestMeta(t) + src := data_testing.NewBackend() + dst := data_testing.NewBackend() + + for i := range 20 { + content := strings.Repeat("x", i+1) + seedUploadedFile(t, mb, src, content) + } + + stats, err := data.MigrateFiles(src, dst, mb, &data.MigrateOptions{Workers: 8}) + require.NoError(t, err) + require.Equal(t, int64(20), stats.Copied) + require.Equal(t, int64(0), stats.Errors) +} diff --git a/server/metadata/migrator.go b/server/metadata/migrator.go new file mode 100644 index 00000000..1c88426f --- /dev/null +++ b/server/metadata/migrator.go @@ -0,0 +1,123 @@ +package metadata + +import ( + "fmt" + + "github.com/root-gg/plik/server/common" +) + +// MigrateOptions controls the behavior of Migrate +type MigrateOptions struct { + IgnoreErrors bool +} + +// MigrateStats holds counters returned by Migrate +type MigrateStats struct { + Users int + UserErrors int + Tokens int + TokenErrors int + Uploads int + UploadErrors int + Files int + FileErrors int + Settings int + SettingErrors int +} + +// Migrate copies all metadata from src to dst backend. +// Order: users → tokens → uploads (including soft-deleted) → files → settings. +// CLI auth sessions are excluded (they are ephemeral). +func Migrate(src *Backend, dst *Backend, options *MigrateOptions) (stats MigrateStats, err error) { + if options == nil { + options = &MigrateOptions{} + } + + handleErr := func(label string, e error) error { + if e == nil { + return nil + } + if options.IgnoreErrors { + src.log.Warningf("error migrating %s: %s", label, e) + return nil + } + return e + } + + // --- Users --- + err = src.ForEachUsers(func(user *common.User) error { + e := dst.CreateUser(user) + if e != nil { + stats.UserErrors++ + return handleErr(fmt.Sprintf("user %s", user.ID), e) + } + stats.Users++ + return nil + }) + if err != nil { + return stats, fmt.Errorf("error iterating users: %w", err) + } + src.log.Infof("migrated %d/%d users", stats.Users, stats.Users+stats.UserErrors) + + // --- Tokens --- + err = src.ForEachToken(func(token *common.Token) error { + e := dst.CreateToken(token) + if e != nil { + stats.TokenErrors++ + return handleErr(fmt.Sprintf("token %s", token.Token), e) + } + stats.Tokens++ + return nil + }) + if err != nil { + return stats, fmt.Errorf("error iterating tokens: %w", err) + } + src.log.Infof("migrated %d/%d tokens", stats.Tokens, stats.Tokens+stats.TokenErrors) + + // --- Uploads (including soft-deleted, to preserve FK integrity with files) --- + err = src.ForEachUploadUnscoped(func(upload *common.Upload) error { + e := dst.CreateUpload(upload) + if e != nil { + stats.UploadErrors++ + return handleErr(fmt.Sprintf("upload %s", upload.ID), e) + } + stats.Uploads++ + return nil + }) + if err != nil { + return stats, fmt.Errorf("error iterating uploads: %w", err) + } + src.log.Infof("migrated %d/%d uploads", stats.Uploads, stats.Uploads+stats.UploadErrors) + + // --- Files --- + err = src.ForEachFile(func(file *common.File) error { + e := dst.CreateFile(file) + if e != nil { + stats.FileErrors++ + return handleErr(fmt.Sprintf("file %s", file.ID), e) + } + stats.Files++ + return nil + }) + if err != nil { + return stats, fmt.Errorf("error iterating files: %w", err) + } + src.log.Infof("migrated %d/%d files", stats.Files, stats.Files+stats.FileErrors) + + // --- Settings --- + err = src.ForEachSetting(func(setting *common.Setting) error { + e := dst.CreateSetting(setting) + if e != nil { + stats.SettingErrors++ + return handleErr(fmt.Sprintf("setting %s", setting.Key), e) + } + stats.Settings++ + return nil + }) + if err != nil { + return stats, fmt.Errorf("error iterating settings: %w", err) + } + src.log.Infof("migrated %d/%d settings", stats.Settings, stats.Settings+stats.SettingErrors) + + return stats, nil +} diff --git a/server/metadata/migrator_test.go b/server/metadata/migrator_test.go new file mode 100644 index 00000000..1b72d464 --- /dev/null +++ b/server/metadata/migrator_test.go @@ -0,0 +1,128 @@ +package metadata + +import ( + "testing" + + "github.com/root-gg/logger" + "github.com/stretchr/testify/require" + + "github.com/root-gg/plik/server/common" +) + +// newSecondaryTestMetadataBackend creates a fresh (erased) SQLite backend +// at a different path, usable as a migration target alongside the primary test backend. +func newSecondaryTestMetadataBackend() *Backend { + cfg := &Config{Driver: "sqlite3", ConnectionString: "/tmp/plik.migrate.dst.db", EraseFirst: true, Debug: false} + b, err := NewBackend(cfg, logger.NewLogger()) + if err != nil { + panic("unable to create secondary metadata backend: " + err.Error()) + } + return b +} + +func TestMigrate_Basic(t *testing.T) { + src := newTestMetadataBackend() + defer shutdownTestMetadataBackend(src) + + // Populate source + user := common.NewUser(common.ProviderLocal, "miguser") + user.NewToken() + createUser(t, src, user) + + upload := &common.Upload{} + upload.NewFile() + upload.User = user.ID + createUpload(t, src, upload) + + setting := &common.Setting{Key: "migrate-key", Value: "migrate-value"} + err := src.CreateSetting(setting) + require.NoError(t, err) + + // Migrate to dst + dst := newSecondaryTestMetadataBackend() + defer shutdownTestMetadataBackend(dst) + + stats, err := Migrate(src, dst, nil) + require.NoError(t, err) + + require.Equal(t, 1, stats.Users, "expected 1 user migrated") + require.Equal(t, 0, stats.UserErrors) + require.Equal(t, 1, stats.Tokens, "expected 1 token migrated") + require.Equal(t, 0, stats.TokenErrors) + require.Equal(t, 1, stats.Uploads, "expected 1 upload migrated") + require.Equal(t, 0, stats.UploadErrors) + require.Equal(t, 1, stats.Files, "expected 1 file migrated") + require.Equal(t, 0, stats.FileErrors) + require.Equal(t, 1, stats.Settings, "expected 1 setting migrated") + require.Equal(t, 0, stats.SettingErrors) + + // Verify dst has the data + gotUser, err := dst.GetUser(user.ID) + require.NoError(t, err) + require.NotNil(t, gotUser) + require.Equal(t, user.Login, gotUser.Login) + + gotUpload, err := dst.GetUpload(upload.ID) + require.NoError(t, err) + require.NotNil(t, gotUpload) + require.Equal(t, upload.ID, gotUpload.ID) + + gotSetting, err := dst.GetSetting(setting.Key) + require.NoError(t, err) + require.NotNil(t, gotSetting) + require.Equal(t, setting.Value, gotSetting.Value) +} + +func TestMigrate_SoftDeletedUpload(t *testing.T) { + src := newTestMetadataBackend() + defer shutdownTestMetadataBackend(src) + + upload := &common.Upload{} + upload.NewFile() + createUpload(t, src, upload) + + // Soft-delete the upload + err := src.RemoveUpload(upload.ID) + require.NoError(t, err) + + dst := newSecondaryTestMetadataBackend() + defer shutdownTestMetadataBackend(dst) + + stats, err := Migrate(src, dst, nil) + require.NoError(t, err) + + // Even soft-deleted uploads and their files should be migrated (FK integrity) + require.Equal(t, 1, stats.Uploads) + require.Equal(t, 1, stats.Files) +} + +func TestMigrate_IgnoreErrors(t *testing.T) { + src := newTestMetadataBackend() + defer shutdownTestMetadataBackend(src) + + // Create two users with the same login (will cause duplicate key on dst) + user1 := common.NewUser(common.ProviderLocal, "dupuser") + createUser(t, src, user1) + + dst := newSecondaryTestMetadataBackend() + defer shutdownTestMetadataBackend(dst) + + // Pre-populate dst with the same user to force a conflict + createUser(t, dst, user1) + + // Without ignore-errors: should fail + _, err := Migrate(src, dst, &MigrateOptions{IgnoreErrors: false}) + require.Error(t, err, "expected error on duplicate user") + + // Reset dst + shutdownTestMetadataBackend(dst) + dst = newSecondaryTestMetadataBackend() + defer shutdownTestMetadataBackend(dst) + createUser(t, dst, user1) + + // With ignore-errors: should succeed and count the error + stats, err := Migrate(src, dst, &MigrateOptions{IgnoreErrors: true}) + require.NoError(t, err) + require.Equal(t, 0, stats.Users, "user should not have been counted as success") + require.Equal(t, 1, stats.UserErrors, "expected 1 user error") +} diff --git a/testing/ARCHITECTURE.md b/testing/ARCHITECTURE.md index 45abf834..03f36cbb 100644 --- a/testing/ARCHITECTURE.md +++ b/testing/ARCHITECTURE.md @@ -10,6 +10,7 @@ testing/ ├── test_backends.sh ← orchestrator: runs all or specific backend tests ├── utils.sh ← shared helpers (docker, server start/stop, test assertions) +├── migrate/ ← plikd migrate e2e test (no Docker required) ├── mariadb/ ← MariaDB metadata backend test ├── mysql/ ← MySQL metadata backend test ├── postgres/ ← PostgreSQL metadata backend test @@ -63,6 +64,7 @@ PLIKD_CONFIG=$PWD/testing/minio/plikd.cfg go test ./server/data/s3/... -v -race | Backend | Tests | Type | |---------|-------|------| +| Migrate | fakedb → migrate → verify record counts, dry-run, idempotent re-run | CLI / Metadata | | MariaDB | Metadata CRUD, migrations | Metadata | | MySQL | Metadata CRUD, migrations | Metadata | | PostgreSQL | Metadata CRUD, migrations | Metadata | diff --git a/testing/migrate/run.sh b/testing/migrate/run.sh new file mode 100755 index 00000000..ab6f99d8 --- /dev/null +++ b/testing/migrate/run.sh @@ -0,0 +1,199 @@ +#!/bin/bash + +set -e +cd "$(dirname "$0")" + +BACKEND="migrate" +CMD=$1 +TEST=$2 + +source ../utils.sh + +ROOT=$(realpath ../..) + +# Build plikd if not already built +if [[ ! -x "$ROOT/server/plikd" ]]; then + echo "Building plikd..." + ( cd "$ROOT" && make server ) +fi + +PLIKD="$ROOT/server/plikd" +WORKDIR=$(mktemp -d) + +trap "rm -rf $WORKDIR" EXIT + +SRC_DB="$WORKDIR/src.db" +DST_DB="$WORKDIR/dst.db" +SRC_CFG="$WORKDIR/src.cfg" +DST_CFG="$WORKDIR/dst.cfg" + +function count_records { + local db="$1" + local table="$2" + sqlite3 "$db" "SELECT COUNT(*) FROM $table;" 2>/dev/null || echo "0" +} + +function start { + echo "nothing to start for migrate tests" +} + +function stop { + echo "nothing to stop for migrate tests" +} + +function status { + true +} + +function run_tests { + BACKEND="$1" + TEST="$2" + + echo "" + echo "=== Migration E2E Test ===" + echo "" + + # --- Step 1: Generate a small fake database --- + echo " - Generating fake source database..." + "$PLIKD" fakedb \ + --users 5 \ + --tokens 2 \ + --uploads 3 \ + --files 2 \ + --anon-uploads 3 \ + --output "$SRC_DB" + + echo "" + + # Count source records + SRC_USERS=$(count_records "$SRC_DB" "users") + SRC_TOKENS=$(count_records "$SRC_DB" "tokens") + SRC_UPLOADS=$(count_records "$SRC_DB" "uploads") + SRC_FILES=$(count_records "$SRC_DB" "files") + SRC_SETTINGS=$(count_records "$SRC_DB" "settings") + + echo " - Source database contains:" + echo " users: $SRC_USERS" + echo " tokens: $SRC_TOKENS" + echo " uploads: $SRC_UPLOADS" + echo " files: $SRC_FILES" + echo " settings:$SRC_SETTINGS" + echo "" + + # Sanity — source must not be empty + if [[ "$SRC_USERS" -eq 0 ]] || [[ "$SRC_UPLOADS" -eq 0 ]] || [[ "$SRC_FILES" -eq 0 ]]; then + echo "FAIL: source database is empty after fakedb" + exit 1 + fi + + # --- Step 2: Create config files --- + cat > "$SRC_CFG" < "$DST_CFG" <&1) + echo "$DRY_OUTPUT" + echo "" + + # Destination should be empty after dry-run (only schema, no data) + DST_UPLOADS_DRY=$(count_records "$DST_DB" "uploads") + if [[ "$DST_UPLOADS_DRY" -ne 0 ]]; then + echo "FAIL: dry-run wrote data to destination (uploads=$DST_UPLOADS_DRY)" + exit 1 + fi + echo " ✓ Dry-run did not write any data" + echo "" + + # --- Step 6: Re-run migration with --ignore-errors (idempotency) --- + echo " - Running: plikd migrate --metadata-only --ignore-errors (re-run on existing destination)" + + # Migrate again into the non-empty dst from step 3 + rm -f "$DST_DB" + "$PLIKD" --config "$SRC_CFG" migrate --to "$DST_CFG" --metadata-only + RERUN_OUTPUT=$("$PLIKD" --config "$SRC_CFG" migrate --to "$DST_CFG" --metadata-only --ignore-errors 2>&1) + echo "$RERUN_OUTPUT" + echo "" + + # After re-run with --ignore-errors, counts should still match + DST_USERS_2=$(count_records "$DST_DB" "users") + DST_UPLOADS_2=$(count_records "$DST_DB" "uploads") + if [[ "$SRC_USERS" -ne "$DST_USERS_2" ]] || [[ "$SRC_UPLOADS" -ne "$DST_UPLOADS_2" ]]; then + echo "FAIL: re-run counts mismatch after --ignore-errors" + exit 1 + fi + echo " ✓ Re-run with --ignore-errors preserved existing data" + echo "" + + echo "=== ALL MIGRATION TESTS PASSED ===" +} + +run_cmd diff --git a/testing/test_backends.sh b/testing/test_backends.sh index 816c004c..212d5148 100755 --- a/testing/test_backends.sh +++ b/testing/test_backends.sh @@ -7,6 +7,7 @@ source ./utils.sh check_docker_connectivity BACKENDS=( + migrate mariadb mysql postgres