testutil: use RDS-managed password rotation in the Ministack harness

docs/testing.md

@@ -210,8 +210,9 @@ password rotation last. Each subtest pins one AWS seam:
   code that real AWS omits — was fixed upstream in Ministack v1.4.14 at
   pg-sprite's request; the pinned image carries the fix, so no divergence
   workaround remains);
-- **password rotation** — what a rotation does to a running schema change
-  (see below), plus pg-sprite's contract that the resulting auth failure
+- **password rotation** — RDS-managed password generation, rotation, and
+  Secrets Manager resolution, plus what a rotation does to a running schema
+  change (see below) and pg-sprite's contract that the resulting auth failure
   is terminal, not retryable.
 
 ### What a password rotation does to a running schema change

go.mod

@@ -4,10 +4,10 @@ go 1.26
 
 require (
 	github.com/alecthomas/kong v1.15.0
-	github.com/aws/aws-sdk-go-v2 v1.43.4
+	github.com/aws/aws-sdk-go-v2 v1.43.5
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.5
 	github.com/aws/aws-sdk-go-v2/service/rds v1.124.1
-	github.com/aws/smithy-go v1.27.6
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/moby/moby/api v1.54.2
 	github.com/moby/moby/client v0.4.0
@@ -22,10 +22,11 @@ require (
 	dario.cat/mergo v1.0.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 // indirect
+	github.com/aws/smithy-go v1.27.7 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/errdefs v1.0.0 // indirect

go.sum

@@ -12,22 +12,24 @@ github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLV
 github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
 github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
 github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE=
-github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ=
+github.com/aws/aws-sdk-go-v2 v1.43.5 h1:yKT5GYnFWhuDo+DqKvE5ZPwVn3RjC4MAeBtZGlh6AVM=
+github.com/aws/aws-sdk-go-v2 v1.43.5/go.mod h1:wZjAJppCntyOGgVSmgVTfDyRJK5PHOasO6Wsy8U7Axk=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.5 h1:yn3zSvIKC2NZIs40cY3kckcy9Zma96PrRR07N54PCvY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.5/go.mod h1:8JcKPAGZVnDWuR5lusAwmrSDtZnDIAnpQWaDC9RFt2g=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 h1:kzVuGlatQtYinwBJEEyLAbggepCoavosiaHHX9+fD+c=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35/go.mod h1:0yLx0yEI+SfqeJMPvOtIEFoZbiQYXMGszBueiutQyaI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 h1:WK6CjihTuLisCjSKKbildJ79sGZZgbBz3iNa7VsKIhU=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35/go.mod h1:KYleN57luLoe97R7vTnx8PMcVrr9gAcRECtOjl91DNg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36 h1:5CrzwxDqf4w3x1Vs3/NiZ0nsC34Hbm3pIDMWbsLebOE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.36/go.mod h1:A3gHdKZIvG/QXERzZwcxNS3RNDFcRCuhhTFBYp+V/nw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36 h1:A4N2f4YPcST0v+dWtX+xrpPPCL9VTBhoIFFUWYqbacE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.36/go.mod h1:B/Qr859uxWUEfZeGotK5KAEoof4Q9YWgNtPSwV6jcyk=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 h1:JJLBQxwY+AFwuPAi5ivGc1ChnTdUt4cXMv7e76m2c/Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15/go.mod h1:lQknBIe78MVL0cQOQDlag8KGflMbMEVFx9mB6O8ENvk=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 h1:BBEElKh4a+rKshvjrfpajTe9CbpZvrbb4Jkg2PB7RzA=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35/go.mod h1:zaZk983w//8beSruBVec/mr4CmDwgZitW/qzGhAAX0g=
 github.com/aws/aws-sdk-go-v2/service/rds v1.124.1 h1:tEeu5kuP2MLQ7drmlN4qYiBKVoUftyBiS6dSFN67pYc=
 github.com/aws/aws-sdk-go-v2/service/rds v1.124.1/go.mod h1:qciN0v66sYiwRf+YRkus1mQR0XldavqGIQEzTxc2vb0=
-github.com/aws/smithy-go v1.27.6 h1:0zjT8jgK3jbrTT7JJ3EE6JsMhX8JTrZ+f1sEndYDXrA=
-github.com/aws/smithy-go v1.27.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5 h1:Bly2ZxYuCW925rQrAUop7E1bVda2kJQahuqqPUSVjsA=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.44.5/go.mod h1:1v44JgDoT1ZSy/b+aACyg4iHb9jTyRsOnybgVmZ5FTM=
+github.com/aws/smithy-go v1.27.7 h1:Zgj5z4LfcDYoQIVk+n/yGdTkP/2y6ZT5vYxe0fp7bqE=
+github.com/aws/smithy-go v1.27.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=

internal/testutil/ministack.go

@@ -11,8 +11,10 @@ import (
 	"context"
 	"crypto/sha1" // not cryptographic: reproduces Ministack's container-name derivation
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"net"
+	"net/url"
 	"os"
 	"strconv"
 	"testing"
@@ -21,10 +23,12 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/rds"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	"github.com/jackc/pgx/v5"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/network"
 	"github.com/moby/moby/client"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
@@ -44,8 +48,11 @@ const (
 	// the sibling database container, which dominates this budget.
 	auroraProvisionDeadline = 5 * time.Minute
 	auroraProvisionPoll     = 2 * time.Second
-	// rotationDeadline bounds how long a rotated master password may take
-	// to land on the running database after ModifyDBCluster returns.
+	// rotationDeadline bounds each phase of the managed-password flow
+	// separately: how long a written secret may take to become resolvable
+	// through the control plane, and how long a rotated password may take
+	// to land on the running database. A rotation that exhausts both
+	// budgets therefore takes up to twice this value.
 	rotationDeadline = time.Minute
 	rotationPoll     = time.Second
 	// rdsStatusAvailable is the RDS API status of a usable instance.
@@ -65,11 +72,10 @@ const (
 	// a test harness, but the reason this tier must never run against a
 	// shared Docker host it does not own.
 	dockerSocket = "/var/run/docker.sock"
-	// fixtureUser, fixturePassword, and fixtureDatabase are emulator-only
-	// test fixtures, never real credentials: Ministack hands them to the
-	// sibling database container it creates for the cluster.
+	// fixtureUser and fixtureDatabase are emulator-only test fixtures:
+	// Ministack hands them to the sibling database container it creates for
+	// the cluster. The master password is generated and managed by RDS.
 	fixtureUser     = "pgsprite"
-	fixturePassword = "test-password-do-not-use"
 	fixtureDatabase = "pgsprite"
 	// awsAccountID and awsRegion identify the emulator's default account.
 	// Ministack scopes the sibling container's name by
@@ -137,46 +143,57 @@ type AuroraCluster struct {
 	// Rotate keeps it in sync with the control plane so URL never goes
 	// silently stale after a rotation.
 	password string
+	// secrets is the Secrets Manager client used to resolve the RDS-managed
+	// master password through the same gateway.
+	secrets *secretsmanager.Client
 }
 
 // URL returns a connection URL for the cluster's database using the
 // master password the cluster currently accepts. After Rotate, that is
 // the rotated password.
 func (c *AuroraCluster) URL() string {
-	return c.URLWithPassword(c.password)
+	return c.urlWithPassword(c.password)
 }
 
-// URLWithPassword returns a connection URL using the given master
-// password — for tests that deliberately present stale or wrong
-// credentials.
+// urlWithPassword returns a connection URL using the given master
+// password. The password is RDS-generated — the harness does not choose
+// it — so it may contain URL-reserved characters; the URL is assembled
+// with net/url, which escapes each component, never by string
+// interpolation.
 //
 // sslmode=disable: the sibling database container runs plain PostgreSQL
 // without TLS, and the endpoint is not an *.rds.amazonaws.com hostname,
 // so the production TLS path is out of scope for this tier (it is
 // proven by pkg/dbconn's TLS integration tests).
-func (c *AuroraCluster) URLWithPassword(password string) string {
-	return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable",
-		fixtureUser, password, c.addr, fixtureDatabase)
+func (c *AuroraCluster) urlWithPassword(password string) string {
+	u := url.URL{
+		Scheme:   "postgres",
+		User:     url.UserPassword(fixtureUser, password),
+		Host:     c.addr,
+		Path:     "/" + fixtureDatabase,
+		RawQuery: "sslmode=disable",
+	}
+	return u.String()
 }
 
-// Rotate changes the cluster's master password through ModifyDBCluster,
-// waits until the running database accepts the new password, and updates
-// the handle so URL reflects the credentials the cluster now accepts.
-// The previous password remains available to the caller for
-// deliberately-stale connections via URLWithPassword.
-func (c *AuroraCluster) Rotate(t *testing.T, newPassword string) {
+// Rotate asks RDS to generate a new managed master password, resolves it
+// from Secrets Manager, waits until the running database accepts it, and
+// updates the handle so URL reflects the credentials the cluster now accepts.
+func (c *AuroraCluster) Rotate(t *testing.T) {
 	t.Helper()
 	ctx := t.Context()
 	_, err := c.Client.ModifyDBCluster(ctx, &rds.ModifyDBClusterInput{
-		DBClusterIdentifier: aws.String(c.ClusterID),
-		MasterUserPassword:  aws.String(newPassword),
-		ApplyImmediately:    aws.Bool(true),
+		DBClusterIdentifier:      aws.String(c.ClusterID),
+		RotateMasterUserPassword: aws.Bool(true),
+		ApplyImmediately:         aws.Bool(true),
 	})
-	require.NoError(t, err, "rotate master password via ModifyDBCluster")
+	require.NoError(t, err, "rotate RDS-managed master password")
+
+	rotated := awaitManagedMasterPassword(t, c.Client, c.secrets, c.ClusterID, c.password)
 
 	// The rotation must land on the real database, not just the control
 	// plane's metadata: poll until the new password authenticates.
-	rotatedURL := c.URLWithPassword(newPassword)
+	rotatedURL := c.urlWithPassword(rotated)
 	require.Eventuallyf(t, func() bool {
 		conn, err := pgx.Connect(ctx, rotatedURL)
 		if err != nil {
@@ -189,7 +206,7 @@ func (c *AuroraCluster) Rotate(t *testing.T, newPassword string) {
 	}, rotationDeadline, rotationPoll,
 		"rotated master password did not become usable within the deadline")
 
-	c.password = newPassword
+	c.password = rotated
 }
 
 // ProvisionAuroraPostgres starts a Ministack container, provisions an
@@ -230,26 +247,28 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster {
 		}
 	})
 
-	clnt := rdsClient(t, ctr)
+	clnt, secrets := awsClients(t, ctr)
 	major, err := strconv.Atoi(PGVersion())
 	require.NoError(t, err, "PG_VERSION must be a PostgreSQL major number")
 
 	const clusterID = "pgsprite-test"
 	_, err = clnt.CreateDBCluster(ctx, &rds.CreateDBClusterInput{
-		DBClusterIdentifier: aws.String(clusterID),
-		Engine:              aws.String("aurora-postgresql"),
-		EngineVersion:       aws.String(auroraEngineVersion(major)),
-		DatabaseName:        aws.String(fixtureDatabase),
-		MasterUsername:      aws.String(fixtureUser),
-		MasterUserPassword:  aws.String(fixturePassword),
+		DBClusterIdentifier:      aws.String(clusterID),
+		Engine:                   aws.String("aurora-postgresql"),
+		EngineVersion:            aws.String(auroraEngineVersion(major)),
+		DatabaseName:             aws.String(fixtureDatabase),
+		MasterUsername:           aws.String(fixtureUser),
+		ManageMasterUserPassword: aws.Bool(true),
 	})
 	require.NoError(t, err, "create aurora-postgresql cluster")
 	// The database backing the cluster is a sibling Docker container, not a
 	// child of the Ministack container — terminating Ministack alone would
 	// leak it. Deleting the cluster through the API reaps it. Cleanups run
 	// last-in-first-out, so the instance delete registered below runs
 	// before this — matching the RDS rule that a cluster cannot be deleted
-	// while it still has instances.
+	// while it still has instances. Registered before anything fallible
+	// touches the cluster, so a failure later in provisioning cannot leak
+	// the sibling container.
 	t.Cleanup(func() {
 		cleanupCtx := context.WithoutCancel(t.Context())
 		if _, err := clnt.DeleteDBCluster(cleanupCtx, &rds.DeleteDBClusterInput{
@@ -259,6 +278,7 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster {
 			t.Logf("delete cluster %s: %v", clusterID, err)
 		}
 	})
+	password := awaitManagedMasterPassword(t, clnt, secrets, clusterID, "")
 
 	instanceID := clusterID + "-1"
 	_, err = clnt.CreateDBInstance(ctx, &rds.CreateDBInstanceInput{
@@ -317,7 +337,8 @@ func ProvisionAuroraPostgres(t *testing.T) *AuroraCluster {
 		ClusterID:  clusterID,
 		InstanceID: instanceID,
 		addr:       addr,
-		password:   fixturePassword,
+		password:   password,
+		secrets:    secrets,
 	}
 }
 
@@ -370,9 +391,9 @@ func siblingHostAddr(t *testing.T, ctr testcontainers.Container, clusterID strin
 	return net.JoinHostPort(host, bindings[0].HostPort)
 }
 
-// rdsClient returns an RDS API client pointed at the container's gateway
-// with the emulator's conventional static test credentials.
-func rdsClient(t *testing.T, ctr testcontainers.Container) *rds.Client {
+// awsClients returns RDS and Secrets Manager clients pointed at the
+// container's gateway with the emulator's conventional static credentials.
+func awsClients(t *testing.T, ctr testcontainers.Container) (*rds.Client, *secretsmanager.Client) {
 	t.Helper()
 	ctx := t.Context()
 	host, err := ctr.Host(ctx)
@@ -391,7 +412,77 @@ func rdsClient(t *testing.T, ctr testcontainers.Container) *rds.Client {
 		Region:      awsRegion,
 		Credentials: credentials.NewStaticCredentialsProvider("test", "test", ""),
 	}
-	return rds.NewFromConfig(cfg, func(o *rds.Options) {
+	rdsClient := rds.NewFromConfig(cfg, func(o *rds.Options) {
 		o.BaseEndpoint = aws.String(endpoint)
 	})
+	secretsClient := secretsmanager.NewFromConfig(cfg, func(o *secretsmanager.Options) {
+		o.BaseEndpoint = aws.String(endpoint)
+	})
+	return rdsClient, secretsClient
+}
+
+// awaitManagedMasterPassword polls until the cluster's RDS-managed master
+// password resolves through the control plane and differs from previous,
+// then returns it. The secret write is the control plane's to sequence —
+// after CreateDBCluster and after a rotation alike, the caller cannot
+// assume the write landed before the API call returned, so a transient
+// resolution failure is retried rather than failing the test. Pass
+// previous "" during provisioning, when any resolved password is
+// acceptable. A deadline failure reports the last resolution error.
+func awaitManagedMasterPassword(t *testing.T, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID, previous string) string {
+	t.Helper()
+	var resolved string
+	require.EventuallyWithTf(t, func(collect *assert.CollectT) {
+		password, err := resolveManagedMasterPassword(t.Context(), rdsClient, secretsClient, clusterID)
+		if !assert.NoErrorf(collect, err, "resolve managed master password for cluster %s", clusterID) {
+			return
+		}
+		if !assert.NotEqual(collect, previous, password,
+			"managed secret still resolves to the previous password") {
+			return
+		}
+		resolved = password
+	}, rotationDeadline, rotationPoll,
+		"managed master password for cluster %s did not become resolvable within the deadline", clusterID)
+	return resolved
+}
+
+// resolveManagedMasterPassword discovers the cluster's managed master-user
+// secret through the RDS control plane and decodes the credentials it holds
+// in Secrets Manager, verifying the secret belongs to the fixture master
+// user. It returns errors rather than asserting so pollers can retry it.
+func resolveManagedMasterPassword(ctx context.Context, rdsClient *rds.Client, secretsClient *secretsmanager.Client, clusterID string) (string, error) {
+	clusters, err := rdsClient.DescribeDBClusters(ctx, &rds.DescribeDBClustersInput{
+		DBClusterIdentifier: aws.String(clusterID),
+	})
+	if err != nil {
+		return "", fmt.Errorf("describe cluster %s: %w", clusterID, err)
+	}
+	if len(clusters.DBClusters) != 1 {
+		return "", fmt.Errorf("describe cluster %s: expected one cluster, got %d", clusterID, len(clusters.DBClusters))
+	}
+	secretMeta := clusters.DBClusters[0].MasterUserSecret
+	if secretMeta == nil || aws.ToString(secretMeta.SecretArn) == "" {
+		return "", fmt.Errorf("cluster %s exposes no managed master-user secret ARN", clusterID)
+	}
+	secret, err := secretsClient.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+		SecretId: secretMeta.SecretArn,
+	})
+	if err != nil {
+		return "", fmt.Errorf("get managed master-user secret for cluster %s: %w", clusterID, err)
+	}
+	var creds struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := json.Unmarshal([]byte(aws.ToString(secret.SecretString)), &creds); err != nil {
+		return "", fmt.Errorf("decode managed master-user secret for cluster %s: %w", clusterID, err)
+	}
+	if creds.Username != fixtureUser {
+		return "", fmt.Errorf("managed secret username %q does not match master user %q", creds.Username, fixtureUser)
+	}
+	if creds.Password == "" {
+		return "", fmt.Errorf("managed secret for cluster %s holds an empty password", clusterID)
+	}
+	return creds.Password, nil
 }

internal/testutil/ministack_integration_test.go

@@ -113,8 +113,9 @@ func errorContract(t *testing.T, cluster *testutil.AuroraCluster) {
 func passwordRotation(t *testing.T, cluster *testutil.AuroraCluster) {
 	// A pool dialed with the pre-rotation password, with one session
 	// checked out — a schema change in flight.
+	staleURL := cluster.URL()
 	pool, err := dbconn.NewPool(t.Context(), dbconn.Config{
-		URL:         cluster.URL(),
+		URL:         staleURL,
 		LockTimeout: 300 * time.Millisecond,
 	})
 	require.NoError(t, err, "connect with the pre-rotation password")
@@ -124,8 +125,7 @@ func passwordRotation(t *testing.T, cluster *testutil.AuroraCluster) {
 	var result int
 	require.NoError(t, held.QueryRow(t.Context(), "SELECT 1").Scan(&result))
 
-	const rotatedPassword = "test-password-rotated-do-not-use"
-	cluster.Rotate(t, rotatedPassword)
+	cluster.Rotate(t)
 
 	// The established session sails through the rotation: PostgreSQL
 	// authenticates at connection time only.