From 7ff8cbbdfb47fff4c85f40328e1b6fb091bc2def Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 12 Aug 2026 11:27:15 +1000 Subject: [PATCH 1/2] testutil: use RDS-managed password rotation in the Ministack harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hand-rolled ModifyDBCluster password seam with the real managed flow Ministack v1.4.15 supports: ManageMasterUserPassword at cluster creation, master-password resolution through Secrets Manager, and RotateMasterUserPassword for the rotation itself — the same control-plane path production credentials take. No fixture password remains in the source. --- docs/testing.md | 5 +- go.mod | 9 +- go.sum | 18 +-- internal/testutil/ministack.go | 133 ++++++++++++++---- .../testutil/ministack_integration_test.go | 6 +- 5 files changed, 125 insertions(+), 46 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 1a056f1..27638ad 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -210,8 +210,9 @@ password rotation last. Each subtest pins one AWS seam: code that real AWS omits — was fixed upstream in Ministack v1.4.14 at pg-sprite's request; the pinned image carries the fix, so no divergence workaround remains); -- **password rotation** — what a rotation does to a running schema change - (see below), plus pg-sprite's contract that the resulting auth failure +- **password rotation** — RDS-managed password generation, rotation, and + Secrets Manager resolution, plus what a rotation does to a running schema + change (see below) and pg-sprite's contract that the resulting auth failure is terminal, not retryable. ### What a password rotation does to a running schema change diff --git a/go.mod b/go.mod index e582d9d..632faf5 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.26 require ( github.com/alecthomas/kong v1.15.0 - github.com/aws/aws-sdk-go-v2 v1.43.4 + github.com/aws/aws-sdk-go-v2 v1.43.5 github.com/aws/aws-sdk-go-v2/credentials v1.17.5 github.com/aws/aws-sdk-go-v2/service/rds v1.124.1 - github.com/aws/smithy-go v1.27.6 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5 github.com/jackc/pgx/v5 v5.10.0 github.com/moby/moby/api v1.54.2 github.com/moby/moby/client v0.4.0 @@ -22,10 +22,11 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 // indirect + github.com/aws/smithy-go v1.27.7 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0d9e5a2..7ed364b 100644 --- a/go.sum +++ b/go.sum @@ -12,22 +12,24 @@ github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLV github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE= -github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ= +github.com/aws/aws-sdk-go-v2 v1.43.5 h1:yKT5GYnFWhuDo+DqKvE5ZPwVn3RjC4MAeBtZGlh6AVM= +github.com/aws/aws-sdk-go-v2 v1.43.5/go.mod h1:wZjAJppCntyOGgVSmgVTfDyRJK5PHOasO6Wsy8U7Axk= github.com/aws/aws-sdk-go-v2/credentials v1.17.5 h1:yn3zSvIKC2NZIs40cY3kckcy9Zma96PrRR07N54PCvY= github.com/aws/aws-sdk-go-v2/credentials v1.17.5/go.mod h1:8JcKPAGZVnDWuR5lusAwmrSDtZnDIAnpQWaDC9RFt2g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 h1:kzVuGlatQtYinwBJEEyLAbggepCoavosiaHHX9+fD+c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35/go.mod h1:0yLx0yEI+SfqeJMPvOtIEFoZbiQYXMGszBueiutQyaI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 h1:WK6CjihTuLisCjSKKbildJ79sGZZgbBz3iNa7VsKIhU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35/go.mod h1:KYleN57luLoe97R7vTnx8PMcVrr9gAcRECtOjl91DNg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 h1:5CrzwxDqf4w3x1Vs3/NiZ0nsC34Hbm3pIDMWbsLebOE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36/go.mod h1:A3gHdKZIvG/QXERzZwcxNS3RNDFcRCuhhTFBYp+V/nw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 h1:A4N2f4YPcST0v+dWtX+xrpPPCL9VTBhoIFFUWYqbacE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36/go.mod h1:B/Qr859uxWUEfZeGotK5KAEoof4Q9YWgNtPSwV6jcyk= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 h1:JJLBQxwY+AFwuPAi5ivGc1ChnTdUt4cXMv7e76m2c/Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15/go.mod h1:lQknBIe78MVL0cQOQDlag8KGflMbMEVFx9mB6O8ENvk= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 h1:BBEElKh4a+rKshvjrfpajTe9CbpZvrbb4Jkg2PB7RzA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35/go.mod h1:zaZk983w//8beSruBVec/mr4CmDwgZitW/qzGhAAX0g= github.com/aws/aws-sdk-go-v2/service/rds v1.124.1 h1:tEeu5kuP2MLQ7drmlN4qYiBKVoUftyBiS6dSFN67pYc= github.com/aws/aws-sdk-go-v2/service/rds v1.124.1/go.mod h1:qciN0v66sYiwRf+YRkus1mQR0XldavqGIQEzTxc2vb0= -github.com/aws/smithy-go v1.27.6 h1:0zjT8jgK3jbrTT7JJ3EE6JsMhX8JTrZ+f1sEndYDXrA= -github.com/aws/smithy-go v1.27.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5 h1:Bly2ZxYuCW925rQrAUop7E1bVda2kJQahuqqPUSVjsA= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5/go.mod h1:1v44JgDoT1ZSy/b+aACyg4iHb9jTyRsOnybgVmZ5FTM= +github.com/aws/smithy-go v1.27.7 h1:Zgj5z4LfcDYoQIVk+n/yGdTkP/2y6ZT5vYxe0fp7bqE= +github.com/aws/smithy-go v1.27.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/testutil/ministack.go b/internal/testutil/ministack.go index 1a076cf..e0bad9e 100644 --- a/internal/testutil/ministack.go +++ b/internal/testutil/ministack.go @@ -11,6 +11,7 @@ import ( "context" "crypto/sha1" // not cryptographic: reproduces Ministack's container-name derivation "encoding/hex" + "encoding/json" "fmt" "net" "os" @@ -21,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/jackc/pgx/v5" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/network" @@ -45,7 +47,7 @@ const ( auroraProvisionDeadline = 5 * time.Minute auroraProvisionPoll = 2 * time.Second // rotationDeadline bounds how long a rotated master password may take - // to land on the running database after ModifyDBCluster returns. + // to land on the running database after managed rotation returns. rotationDeadline = time.Minute rotationPoll = time.Second // rdsStatusAvailable is the RDS API status of a usable instance. @@ -65,11 +67,10 @@ const ( // a test harness, but the reason this tier must never run against a // shared Docker host it does not own. dockerSocket = "/var/run/docker.sock" - // fixtureUser, fixturePassword, and fixtureDatabase are emulator-only - // test fixtures, never real credentials: Ministack hands them to the - // sibling database container it creates for the cluster. + // fixtureUser and fixtureDatabase are emulator-only test fixtures: + // Ministack hands them to the sibling database container it creates for + // the cluster. The master password is generated and managed by RDS. fixtureUser = "pgsprite" - fixturePassword = "test-password-do-not-use" fixtureDatabase = "pgsprite" // awsAccountID and awsRegion identify the emulator's default account. // Ministack scopes the sibling container's name by @@ -137,6 +138,9 @@ type AuroraCluster struct { // Rotate keeps it in sync with the control plane so URL never goes // silently stale after a rotation. password string + // secrets is the Secrets Manager client used to resolve the RDS-managed + // master password through the same gateway. + secrets *secretsmanager.Client } // URL returns a connection URL for the cluster's database using the @@ -159,24 +163,40 @@ func (c *AuroraCluster) URLWithPassword(password string) string { fixtureUser, password, c.addr, fixtureDatabase) } -// Rotate changes the cluster's master password through ModifyDBCluster, -// waits until the running database accepts the new password, and updates -// the handle so URL reflects the credentials the cluster now accepts. -// The previous password remains available to the caller for -// deliberately-stale connections via URLWithPassword. -func (c *AuroraCluster) Rotate(t *testing.T, newPassword string) { +// Rotate asks RDS to generate a new managed master password, resolves it +// from Secrets Manager, waits until the running database accepts it, and +// updates the handle so URL reflects the credentials the cluster now accepts. +func (c *AuroraCluster) Rotate(t *testing.T) { t.Helper() ctx := t.Context() _, err := c.Client.ModifyDBCluster(ctx, &rds.ModifyDBClusterInput{ - DBClusterIdentifier: aws.String(c.ClusterID), - MasterUserPassword: aws.String(newPassword), - ApplyImmediately: aws.Bool(true), + DBClusterIdentifier: aws.String(c.ClusterID), + RotateMasterUserPassword: aws.Bool(true), + ApplyImmediately: aws.Bool(true), }) - require.NoError(t, err, "rotate master password via ModifyDBCluster") + require.NoError(t, err, "rotate RDS-managed master password") + + // Rotation and the secret write are the control plane's to sequence: + // poll until the secret no longer resolves to the pre-rotation + // password, rather than assuming the write landed before + // ModifyDBCluster returned. + var rotated string + require.Eventuallyf(t, func() bool { + password, err := resolveManagedMasterPassword(ctx, c.Client, c.secrets, c.ClusterID) + if err != nil { + return false + } + if password == c.password { + return false + } + rotated = password + return true + }, rotationDeadline, rotationPoll, + "managed rotation did not produce a new password within the deadline") // The rotation must land on the real database, not just the control // plane's metadata: poll until the new password authenticates. - rotatedURL := c.URLWithPassword(newPassword) + rotatedURL := c.URLWithPassword(rotated) require.Eventuallyf(t, func() bool { conn, err := pgx.Connect(ctx, rotatedURL) if err != nil { @@ -189,7 +209,7 @@ func (c *AuroraCluster) Rotate(t *testing.T, newPassword string) { }, rotationDeadline, rotationPoll, "rotated master password did not become usable within the deadline") - c.password = newPassword + c.password = rotated } // ProvisionAuroraPostgres starts a Ministack container, provisions an @@ -230,20 +250,21 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster { } }) - clnt := rdsClient(t, ctr) + clnt, secrets := awsClients(t, ctr) major, err := strconv.Atoi(PGVersion()) require.NoError(t, err, "PG_VERSION must be a PostgreSQL major number") const clusterID = "pgsprite-test" _, err = clnt.CreateDBCluster(ctx, &rds.CreateDBClusterInput{ - DBClusterIdentifier: aws.String(clusterID), - Engine: aws.String("aurora-postgresql"), - EngineVersion: aws.String(auroraEngineVersion(major)), - DatabaseName: aws.String(fixtureDatabase), - MasterUsername: aws.String(fixtureUser), - MasterUserPassword: aws.String(fixturePassword), + DBClusterIdentifier: aws.String(clusterID), + Engine: aws.String("aurora-postgresql"), + EngineVersion: aws.String(auroraEngineVersion(major)), + DatabaseName: aws.String(fixtureDatabase), + MasterUsername: aws.String(fixtureUser), + ManageMasterUserPassword: aws.Bool(true), }) require.NoError(t, err, "create aurora-postgresql cluster") + password := managedMasterPassword(t, clnt, secrets, clusterID) // The database backing the cluster is a sibling Docker container, not a // child of the Ministack container — terminating Ministack alone would // leak it. Deleting the cluster through the API reaps it. Cleanups run @@ -317,7 +338,8 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster { ClusterID: clusterID, InstanceID: instanceID, addr: addr, - password: fixturePassword, + password: password, + secrets: secrets, } } @@ -370,9 +392,9 @@ func siblingHostAddr(t *testing.T, ctr testcontainers.Container, clusterID strin return net.JoinHostPort(host, bindings[0].HostPort) } -// rdsClient returns an RDS API client pointed at the container's gateway -// with the emulator's conventional static test credentials. -func rdsClient(t *testing.T, ctr testcontainers.Container) *rds.Client { +// awsClients returns RDS and Secrets Manager clients pointed at the +// container's gateway with the emulator's conventional static credentials. +func awsClients(t *testing.T, ctr testcontainers.Container) (*rds.Client, *secretsmanager.Client) { t.Helper() ctx := t.Context() host, err := ctr.Host(ctx) @@ -391,7 +413,60 @@ func rdsClient(t *testing.T, ctr testcontainers.Container) *rds.Client { Region: awsRegion, Credentials: credentials.NewStaticCredentialsProvider("test", "test", ""), } - return rds.NewFromConfig(cfg, func(o *rds.Options) { + rdsClient := rds.NewFromConfig(cfg, func(o *rds.Options) { + o.BaseEndpoint = aws.String(endpoint) + }) + secretsClient := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) { o.BaseEndpoint = aws.String(endpoint) }) + return rdsClient, secretsClient +} + +// managedMasterPassword resolves the cluster's RDS-managed master password, +// failing the test on any resolution error. +func managedMasterPassword(t *testing.T, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID string) string { + t.Helper() + password, err := resolveManagedMasterPassword(t.Context(), rdsClient, secretsClient, clusterID) + require.NoError(t, err, "resolve managed master password") + return password +} + +// resolveManagedMasterPassword discovers the cluster's managed master-user +// secret through the RDS control plane and decodes the credentials it holds +// in Secrets Manager, verifying the secret belongs to the fixture master +// user. It returns errors rather than asserting so pollers can retry it. +func resolveManagedMasterPassword(ctx context.Context, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID string) (string, error) { + clusters, err := rdsClient.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{ + DBClusterIdentifier: aws.String(clusterID), + }) + if err != nil { + return "", fmt.Errorf("describe cluster %s: %w", clusterID, err) + } + if len(clusters.DBClusters) != 1 { + return "", fmt.Errorf("describe cluster %s: expected one cluster, got %d", clusterID, len(clusters.DBClusters)) + } + secretMeta := clusters.DBClusters[0].MasterUserSecret + if secretMeta == nil || aws.ToString(secretMeta.SecretArn) == "" { + return "", fmt.Errorf("cluster %s exposes no managed master-user secret ARN", clusterID) + } + secret, err := secretsClient.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: secretMeta.SecretArn, + }) + if err != nil { + return "", fmt.Errorf("get managed master-user secret for cluster %s: %w", clusterID, err) + } + var creds struct { + Username string `json:"username"` + Password string `json:"password"` + } + if err := json.Unmarshal([]byte(aws.ToString(secret.SecretString)), &creds); err != nil { + return "", fmt.Errorf("decode managed master-user secret for cluster %s: %w", clusterID, err) + } + if creds.Username != fixtureUser { + return "", fmt.Errorf("managed secret username %q does not match master user %q", creds.Username, fixtureUser) + } + if creds.Password == "" { + return "", fmt.Errorf("managed secret for cluster %s holds an empty password", clusterID) + } + return creds.Password, nil } diff --git a/internal/testutil/ministack_integration_test.go b/internal/testutil/ministack_integration_test.go index ecaa883..aa2307e 100644 --- a/internal/testutil/ministack_integration_test.go +++ b/internal/testutil/ministack_integration_test.go @@ -113,8 +113,9 @@ func errorContract(t *testing.T, cluster *testutil.AuroraCluster) { func passwordRotation(t *testing.T, cluster *testutil.AuroraCluster) { // A pool dialed with the pre-rotation password, with one session // checked out — a schema change in flight. + staleURL := cluster.URL() pool, err := dbconn.NewPool(t.Context(), dbconn.Config{ - URL: cluster.URL(), + URL: staleURL, LockTimeout: 300 * time.Millisecond, }) require.NoError(t, err, "connect with the pre-rotation password") @@ -124,8 +125,7 @@ func passwordRotation(t *testing.T, cluster *testutil.AuroraCluster) { var result int require.NoError(t, held.QueryRow(t.Context(), "SELECT 1").Scan(&result)) - const rotatedPassword = "test-password-rotated-do-not-use" - cluster.Rotate(t, rotatedPassword) + cluster.Rotate(t) // The established session sails through the rotation: PostgreSQL // authenticates at connection time only. From 890bf8520e3e9df26efbb47d93f764698614bfc8 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 12 Aug 2026 14:28:48 +1000 Subject: [PATCH 2/2] escape managed passwords in URLs and poll secret resolution RDS-generated passwords are outside the harness's control and may carry URL-reserved characters, so connection URLs are now assembled with net/url instead of string interpolation. The cluster-delete cleanup registers before anything fallible touches the cluster, so a provisioning failure can no longer leak the sibling container, and secret resolution polls with the last error reported on deadline. --- internal/testutil/ministack.go | 86 ++++++++++++-------- internal/testutil/ministack_internal_test.go | 39 +++++++++ 2 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 internal/testutil/ministack_internal_test.go diff --git a/internal/testutil/ministack.go b/internal/testutil/ministack.go index e0bad9e..b8be00a 100644 --- a/internal/testutil/ministack.go +++ b/internal/testutil/ministack.go @@ -14,6 +14,7 @@ import ( "encoding/json" "fmt" "net" + "net/url" "os" "strconv" "testing" @@ -27,6 +28,7 @@ import ( "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/network" "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -46,8 +48,11 @@ const ( // the sibling database container, which dominates this budget. auroraProvisionDeadline = 5 * time.Minute auroraProvisionPoll = 2 * time.Second - // rotationDeadline bounds how long a rotated master password may take - // to land on the running database after managed rotation returns. + // rotationDeadline bounds each phase of the managed-password flow + // separately: how long a written secret may take to become resolvable + // through the control plane, and how long a rotated password may take + // to land on the running database. A rotation that exhausts both + // budgets therefore takes up to twice this value. rotationDeadline = time.Minute rotationPoll = time.Second // rdsStatusAvailable is the RDS API status of a usable instance. @@ -147,20 +152,28 @@ type AuroraCluster struct { // master password the cluster currently accepts. After Rotate, that is // the rotated password. func (c *AuroraCluster) URL() string { - return c.URLWithPassword(c.password) + return c.urlWithPassword(c.password) } -// URLWithPassword returns a connection URL using the given master -// password — for tests that deliberately present stale or wrong -// credentials. +// urlWithPassword returns a connection URL using the given master +// password. The password is RDS-generated — the harness does not choose +// it — so it may contain URL-reserved characters; the URL is assembled +// with net/url, which escapes each component, never by string +// interpolation. // // sslmode=disable: the sibling database container runs plain PostgreSQL // without TLS, and the endpoint is not an *.rds.amazonaws.com hostname, // so the production TLS path is out of scope for this tier (it is // proven by pkg/dbconn's TLS integration tests). -func (c *AuroraCluster) URLWithPassword(password string) string { - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", - fixtureUser, password, c.addr, fixtureDatabase) +func (c *AuroraCluster) urlWithPassword(password string) string { + u := url.URL{ + Scheme: "postgres", + User: url.UserPassword(fixtureUser, password), + Host: c.addr, + Path: "/" + fixtureDatabase, + RawQuery: "sslmode=disable", + } + return u.String() } // Rotate asks RDS to generate a new managed master password, resolves it @@ -176,27 +189,11 @@ func (c *AuroraCluster) Rotate(t *testing.T) { }) require.NoError(t, err, "rotate RDS-managed master password") - // Rotation and the secret write are the control plane's to sequence: - // poll until the secret no longer resolves to the pre-rotation - // password, rather than assuming the write landed before - // ModifyDBCluster returned. - var rotated string - require.Eventuallyf(t, func() bool { - password, err := resolveManagedMasterPassword(ctx, c.Client, c.secrets, c.ClusterID) - if err != nil { - return false - } - if password == c.password { - return false - } - rotated = password - return true - }, rotationDeadline, rotationPoll, - "managed rotation did not produce a new password within the deadline") + rotated := awaitManagedMasterPassword(t, c.Client, c.secrets, c.ClusterID, c.password) // The rotation must land on the real database, not just the control // plane's metadata: poll until the new password authenticates. - rotatedURL := c.URLWithPassword(rotated) + rotatedURL := c.urlWithPassword(rotated) require.Eventuallyf(t, func() bool { conn, err := pgx.Connect(ctx, rotatedURL) if err != nil { @@ -264,13 +261,14 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster { ManageMasterUserPassword: aws.Bool(true), }) require.NoError(t, err, "create aurora-postgresql cluster") - password := managedMasterPassword(t, clnt, secrets, clusterID) // The database backing the cluster is a sibling Docker container, not a // child of the Ministack container — terminating Ministack alone would // leak it. Deleting the cluster through the API reaps it. Cleanups run // last-in-first-out, so the instance delete registered below runs // before this — matching the RDS rule that a cluster cannot be deleted - // while it still has instances. + // while it still has instances. Registered before anything fallible + // touches the cluster, so a failure later in provisioning cannot leak + // the sibling container. t.Cleanup(func() { cleanupCtx := context.WithoutCancel(t.Context()) if _, err := clnt.DeleteDBCluster(cleanupCtx, &rds.DeleteDBClusterInput{ @@ -280,6 +278,7 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster { t.Logf("delete cluster %s: %v", clusterID, err) } }) + password := awaitManagedMasterPassword(t, clnt, secrets, clusterID, "") instanceID := clusterID + "-1" _, err = clnt.CreateDBInstance(ctx, &rds.CreateDBInstanceInput{ @@ -422,13 +421,30 @@ func awsClients(t *testing.T, ctr testcontainers.Container) (*rds.Client, *secre return rdsClient, secretsClient } -// managedMasterPassword resolves the cluster's RDS-managed master password, -// failing the test on any resolution error. -func managedMasterPassword(t *testing.T, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID string) string { +// awaitManagedMasterPassword polls until the cluster's RDS-managed master +// password resolves through the control plane and differs from previous, +// then returns it. The secret write is the control plane's to sequence — +// after CreateDBCluster and after a rotation alike, the caller cannot +// assume the write landed before the API call returned, so a transient +// resolution failure is retried rather than failing the test. Pass +// previous "" during provisioning, when any resolved password is +// acceptable. A deadline failure reports the last resolution error. +func awaitManagedMasterPassword(t *testing.T, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID, previous string) string { t.Helper() - password, err := resolveManagedMasterPassword(t.Context(), rdsClient, secretsClient, clusterID) - require.NoError(t, err, "resolve managed master password") - return password + var resolved string + require.EventuallyWithTf(t, func(collect *assert.CollectT) { + password, err := resolveManagedMasterPassword(t.Context(), rdsClient, secretsClient, clusterID) + if !assert.NoErrorf(collect, err, "resolve managed master password for cluster %s", clusterID) { + return + } + if !assert.NotEqual(collect, previous, password, + "managed secret still resolves to the previous password") { + return + } + resolved = password + }, rotationDeadline, rotationPoll, + "managed master password for cluster %s did not become resolvable within the deadline", clusterID) + return resolved } // resolveManagedMasterPassword discovers the cluster's managed master-user diff --git a/internal/testutil/ministack_internal_test.go b/internal/testutil/ministack_internal_test.go new file mode 100644 index 0000000..5fe9c6e --- /dev/null +++ b/internal/testutil/ministack_internal_test.go @@ -0,0 +1,39 @@ +//go:build ministack + +package testutil + +import ( + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAuroraControlPlaneURLEscaping proves connection URLs survive +// RDS-generated passwords containing URL-reserved characters: the harness +// does not choose the master password, so every character the generator +// may emit must round-trip through URL construction and pgx parsing back +// to the identical password. Needs no Docker — it exercises only the URL +// seam — but lives in the ministack tier with the code it proves. +func TestAuroraControlPlaneURLEscaping(t *testing.T) { + cluster := &AuroraCluster{addr: "db.internal.example:5432"} + for _, password := range []string{ + "pass%zzword", // invalid percent-escape when interpolated raw + "pass%40word", // decodes to a different password when unescaped + "pass#word", // fragment truncation + "pass?sslmode=off", // query truncation + "pass:word", // userinfo separator + "pass&word=1", // query separator + "pass word", // space + "pass@word/end", // authority and path separators + } { + cfg, err := pgx.ParseConfig(cluster.urlWithPassword(password)) + require.NoErrorf(t, err, "URL with password %q must parse", password) + assert.Equalf(t, password, cfg.Password, "password %q must round-trip", password) + assert.Equal(t, fixtureUser, cfg.User) + assert.Equal(t, "db.internal.example", cfg.Host) + assert.Equal(t, uint16(5432), cfg.Port) + assert.Equal(t, fixtureDatabase, cfg.Database) + } +}