node: send certificates for inter-node TLS connections

CHANGELOG.md

@@ -8,6 +8,7 @@ Changelog for NeoFS Node
 ### Fixed
 
 ### Changed
+- SNs exchange TLS certificates on inter-node connections (#4097)
 
 ### Removed
 

cmd/neofs-node/config.go

@@ -409,9 +409,10 @@ func initCfg(appCfg *config.Config) *cfg {
 	minConnTimeout := appCfg.APIClient.MinConnectionTime
 	pingInterval := appCfg.APIClient.PingInterval
 	pingTimeout := appCfg.APIClient.PingTimeout
+	getClientCertificate := clientCertificateProvider(appCfg.GRPC)
 	newClientCache := func(scope string) *cache.Clients {
 		return cache.NewClients(c.log.With(zap.String("scope", scope)), &buffers, streamTimeout,
-			minConnTimeout, pingInterval, pingTimeout, neofsecdsa.Signer(key.PrivateKey))
+			minConnTimeout, pingInterval, pingTimeout, neofsecdsa.Signer(key.PrivateKey), getClientCertificate)
 	}
 	c.shared = shared{
 		basics:            basicSharedConfig,

cmd/neofs-node/grpc.go

@@ -204,6 +204,7 @@ func buildSingleGRPCServer(c *cfg, sc grpcconfig.GRPC, maxRecvMsgSizeOpt grpc.Se
 				}
 				return &tls.Config{
 					Certificates: []tls.Certificate{cert},
+					ClientAuth:   tls.RequestClientCert,
 				}, nil
 			},
 		})

cmd/neofs-node/mtls.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+
+	grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc"
+)
+
+func clientCertificateProvider(cfgs []grpcconfig.GRPC) func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+	for i := range cfgs {
+		if !cfgs[i].TLS.Enabled {
+			continue
+		}
+
+		certFile, keyFile := cfgs[i].TLS.Certificate, cfgs[i].TLS.Key
+		return func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+			if err != nil {
+				return nil, fmt.Errorf("reload TLS client certificate: %w", err)
+			}
+			return &cert, nil
+		}
+	}
+
+	return nil
+}

cmd/neofs-node/mtls_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"testing"
+
+	grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClientCertificateProvider(t *testing.T) {
+	require.Nil(t, clientCertificateProvider(nil))
+
+	provider := clientCertificateProvider([]grpcconfig.GRPC{{
+		TLS: grpcconfig.TLS{
+			Enabled:     true,
+			Certificate: "missing-certificate",
+			Key:         "missing-key",
+		},
+	}})
+	require.NotNil(t, provider)
+	_, err := provider(nil)
+	require.ErrorContains(t, err, "reload TLS client certificate")
+
+	provider = clientCertificateProvider([]grpcconfig.GRPC{
+		{TLS: grpcconfig.TLS{Enabled: false, Certificate: "ignored-certificate", Key: "ignored-key"}},
+		{TLS: grpcconfig.TLS{Enabled: true, Certificate: "client-certificate", Key: "client-key"}},
+	})
+	_, err = provider(nil)
+	require.ErrorContains(t, err, "client-certificate")
+}

docs/storage-node-configuration.md

@@ -67,6 +67,11 @@ element.
 
 ## `tls` subsection
 
+The certificate and key from the first gRPC endpoint with TLS enabled are also
+used as the client certificate for outgoing inter-node TLS connections. They
+are reloaded for every TLS handshake, so certificate rotation does not require
+restarting the node.
+
 | Parameter             | Type     | Default value | Description                                                               |
 |-----------------------|----------|---------------|---------------------------------------------------------------------------|
 | `enabled`             | `bool`   | `false`       | Address that control service listener binds to.                           |

internal/crypto/requests.go

@@ -1,11 +1,13 @@
 package crypto
 
 import (
+	"context"
 	"crypto/sha256"
 	"errors"
 	"fmt"
 
 	"github.com/nspcc-dev/neo-go/pkg/crypto/hash"
+	"github.com/nspcc-dev/neofs-node/pkg/network/peerauth"
 	apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status"
 	neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto"
 	"github.com/nspcc-dev/neofs-sdk-go/proto/refs"
@@ -19,6 +21,15 @@ func VerifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedR
 	return verifyRequestSignatures(req, nil)
 }
 
+// VerifyRequestSignaturesWithContext is same as [VerifyRequestSignatures], but
+// skips verification for an authenticated inter-node request with TTL equal to one.
+func VerifyRequestSignaturesWithContext[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B]) error {
+	if !requestNeedsSignature(ctx, req) {
+		return nil
+	}
+	return verifyRequestSignatures(req, nil)
+}
+
 func verifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedRequest[B], verifyN3 func(data, invocScript, verifScript []byte) error) error {
 	err := neofscrypto.VerifyRequestWithBufferN3(req, nil, verifyN3)
 	if err != nil {
@@ -31,7 +42,10 @@ func verifyRequestSignatures[B neofscrypto.ProtoMessage](req neofscrypto.SignedR
 
 // VerifyRequestSignaturesN3 is same as [VerifyRequestSignatures] but supports
 // [neofscrypto.N3] scheme.
-func VerifyRequestSignaturesN3[B neofscrypto.ProtoMessage](req neofscrypto.SignedRequest[B], fsChain N3ScriptRunner) error {
+func VerifyRequestSignaturesN3[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B], fsChain N3ScriptRunner) error {
+	if !requestNeedsSignature(ctx, req) {
+		return nil
+	}
 	return verifyRequestSignatures(req, func(data, invocScript, verifScript []byte) error {
 		verifScriptHash := hash.Hash160(verifScript)
 		return verifyN3ScriptsNow(fsChain, verifScriptHash, invocScript, verifScript, func() [sha256.Size]byte {
@@ -40,6 +54,18 @@ func VerifyRequestSignaturesN3[B neofscrypto.ProtoMessage](req neofscrypto.Signe
 	})
 }
 
+func requestNeedsSignature[B neofscrypto.ProtoMessage](ctx context.Context, req neofscrypto.SignedRequest[B]) bool {
+	if req.GetVerifyHeader() != nil {
+		return true
+	}
+	meta := req.GetMetaHeader()
+	if meta == nil || meta.GetTtl() != 1 {
+		return true
+	}
+	key, err := peerauth.PeerPublicKey(ctx)
+	return err != nil || key == nil
+}
+
 // GetRequestAuthor returns ID of the request author along with public key from
 // the request verification header.
 func GetRequestAuthor(vh *protosession.RequestVerificationHeader) (user.ID, []byte, error) {

internal/crypto/requests_test.go

@@ -2,6 +2,12 @@ package crypto_test
 
 import (
 	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	cryptorand "crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
 	"math/rand/v2"
 	"testing"
 
@@ -15,6 +21,8 @@ import (
 	protosession "github.com/nspcc-dev/neofs-sdk-go/proto/session"
 	"github.com/nspcc-dev/neofs-sdk-go/user"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/peer"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -25,6 +33,24 @@ func assertInvalidRequestSignatureError(t testing.TB, actual error, expected str
 	require.Equal(t, expected, st.Message())
 }
 
+func TestVerifyRequestSignaturesWithContext(t *testing.T) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	require.NoError(t, err)
+	ctx := peer.NewContext(context.Background(), &peer.Peer{
+		AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{
+			PeerCertificates: []*x509.Certificate{{PublicKey: &key.PublicKey}},
+		}},
+	})
+	req := &protoobject.GetRequest{MetaHeader: &protosession.RequestMetaHeader{Ttl: 1}}
+
+	require.NoError(t, icrypto.VerifyRequestSignaturesWithContext(ctx, req))
+	require.NoError(t, icrypto.VerifyRequestSignaturesN3(ctx, req, nil))
+
+	req.MetaHeader.Ttl = 0
+	err = icrypto.VerifyRequestSignaturesWithContext(ctx, req)
+	assertInvalidRequestSignatureError(t, err, "missing verification header")
+}
+
 func TestVerifyRequestSignatures(t *testing.T) {
 	t.Run("correctly signed", func(t *testing.T) {
 		err := icrypto.VerifyRequestSignatures(getObjectSignedRequest)

pkg/network/cache/clients.go

@@ -47,9 +47,10 @@ type Clients struct {
 	streamMsgTimeout time.Duration
 	signBufPool      *sync.Pool
 	// gRPC settings
-	minConnTimeout time.Duration
-	pingInterval   time.Duration
-	pingTimeout    time.Duration
+	minConnTimeout       time.Duration
+	pingInterval         time.Duration
+	pingTimeout          time.Duration
+	getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
 
 	mtx   sync.RWMutex
 	conns map[string]*connections // keys are public key bytes
@@ -59,16 +60,19 @@ type Clients struct {
 
 // NewClients constructs Clients initializing connection to any endpoint with
 // given parameters.
-func NewClients(l *zap.Logger, signBufPool *sync.Pool, streamTimeout, minConnTimeout, pingInterval, pingTimeout time.Duration, signer neofscrypto.Signer) *Clients {
+func NewClients(l *zap.Logger, signBufPool *sync.Pool, streamTimeout, minConnTimeout, pingInterval, pingTimeout time.Duration, signer neofscrypto.Signer,
+	getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error),
+) *Clients {
 	return &Clients{
-		log:              l,
-		streamMsgTimeout: streamTimeout,
-		signBufPool:      signBufPool,
-		minConnTimeout:   minConnTimeout,
-		pingInterval:     pingInterval,
-		pingTimeout:      pingTimeout,
-		conns:            make(map[string]*connections),
-		signer:           signer,
+		log:                  l,
+		streamMsgTimeout:     streamTimeout,
+		signBufPool:          signBufPool,
+		minConnTimeout:       minConnTimeout,
+		pingInterval:         pingInterval,
+		pingTimeout:          pingTimeout,
+		getClientCertificate: getClientCertificate,
+		conns:                make(map[string]*connections),
+		signer:               signer,
 	}
 }
 
@@ -228,7 +232,7 @@ func (x *Clients) initConnection(ctx context.Context, pub []byte, uri string) (*
 		if err != nil {
 			return nil, nil, fmt.Errorf("parse node public key: %w", err)
 		}
-		transportCreds = credentials.NewTLS(newNodeTLSConfig((*ecdsa.PublicKey)(expectedKey)))
+		transportCreds = credentials.NewTLS(newNodeTLSConfig((*ecdsa.PublicKey)(expectedKey), x.getClientCertificate))
 	} else {
 		transportCreds = insecure.NewCredentials()
 	}
@@ -279,9 +283,10 @@ func (x *Clients) initConnection(ctx context.Context, pub []byte, uri string) (*
 	return res, clientAPI.ProtoMessage(), nil
 }
 
-func newNodeTLSConfig(expectedKey *ecdsa.PublicKey) *tls.Config {
+func newNodeTLSConfig(expectedKey *ecdsa.PublicKey, getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)) *tls.Config {
 	return &tls.Config{
-		InsecureSkipVerify: true,
+		InsecureSkipVerify:   true,
+		GetClientCertificate: getClientCertificate,
 		VerifyConnection: func(state tls.ConnectionState) error {
 			if len(state.PeerCertificates) == 0 {
 				return errors.New("server did not provide TLS certificate")

pkg/network/cache/clients_internal_test.go

@@ -32,33 +32,43 @@ func TestNodeTLSConfig(t *testing.T) {
 	require.NoError(t, err)
 
 	t.Run("matching self-signed certificate", func(t *testing.T) {
-		cfg := newNodeTLSConfig(&expectedKey.PublicKey)
+		cfg := newNodeTLSConfig(&expectedKey.PublicKey, nil)
 		require.True(t, cfg.InsecureSkipVerify)
 		require.NoError(t, cfg.VerifyConnection(tls.ConnectionState{
 			PeerCertificates: []*x509.Certificate{newSelfSignedCertificate(t, expectedKey)},
 		}))
 	})
 
+	t.Run("client certificate", func(t *testing.T) {
+		expected := new(tls.Certificate)
+		cfg := newNodeTLSConfig(&expectedKey.PublicKey, func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			return expected, nil
+		})
+		actual, err := cfg.GetClientCertificate(nil)
+		require.NoError(t, err)
+		require.Same(t, expected, actual)
+	})
+
 	t.Run("wrong key", func(t *testing.T) {
 		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 		require.NoError(t, err)
 
-		err = newNodeTLSConfig(&expectedKey.PublicKey).VerifyConnection(tls.ConnectionState{
+		err = newNodeTLSConfig(&expectedKey.PublicKey, nil).VerifyConnection(tls.ConnectionState{
 			PeerCertificates: []*x509.Certificate{newSelfSignedCertificate(t, key)},
 		})
 		require.ErrorIs(t, err, clientcore.ErrWrongPublicKey)
 	})
 
 	t.Run("no certificate", func(t *testing.T) {
-		err := newNodeTLSConfig(&expectedKey.PublicKey).VerifyConnection(tls.ConnectionState{})
+		err := newNodeTLSConfig(&expectedKey.PublicKey, nil).VerifyConnection(tls.ConnectionState{})
 		require.EqualError(t, err, "server did not provide TLS certificate")
 	})
 
 	t.Run("unsupported key type", func(t *testing.T) {
 		key, err := rsa.GenerateKey(rand.Reader, 2048)
 		require.NoError(t, err)
 
-		err = newNodeTLSConfig(&expectedKey.PublicKey).VerifyConnection(tls.ConnectionState{
+		err = newNodeTLSConfig(&expectedKey.PublicKey, nil).VerifyConnection(tls.ConnectionState{
 			PeerCertificates: []*x509.Certificate{newSelfSignedCertificate(t, key)},
 		})
 		require.EqualError(t, err, "server TLS certificate has unsupported public key type *rsa.PublicKey")
@@ -68,7 +78,7 @@ func TestNodeTLSConfig(t *testing.T) {
 		key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
 		require.NoError(t, err)
 
-		err = newNodeTLSConfig(&expectedKey.PublicKey).VerifyConnection(tls.ConnectionState{
+		err = newNodeTLSConfig(&expectedKey.PublicKey, nil).VerifyConnection(tls.ConnectionState{
 			PeerCertificates: []*x509.Certificate{newSelfSignedCertificate(t, key)},
 		})
 		require.EqualError(t, err, "server TLS certificate has unsupported elliptic curve P-384")

pkg/network/peerauth/peerauth.go

@@ -0,0 +1,46 @@
+package peerauth
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/x509"
+	"fmt"
+
+	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/peer"
+)
+
+// CertificatePublicKey returns the P-256 public key from cert.
+func CertificatePublicKey(cert *x509.Certificate) (*keys.PublicKey, error) {
+	pub, ok := cert.PublicKey.(*ecdsa.PublicKey)
+	if !ok {
+		return nil, fmt.Errorf("unsupported public key type %T", cert.PublicKey)
+	}
+	if pub.Curve != elliptic.P256() {
+		return nil, fmt.Errorf("unsupported elliptic curve %s", pub.Curve.Params().Name)
+	}
+	return (*keys.PublicKey)(pub), nil
+}
+
+// PeerPublicKey returns the public key authenticated by the TLS connection.
+// It returns nil when the request has no TLS client certificate.
+func PeerPublicKey(ctx context.Context) (*keys.PublicKey, error) {
+	p, ok := peer.FromContext(ctx)
+	if !ok {
+		return nil, nil
+	}
+	info, ok := p.AuthInfo.(credentials.TLSInfo)
+	if !ok {
+		return nil, nil
+	}
+	if len(info.State.PeerCertificates) == 0 {
+		return nil, nil
+	}
+	key, err := CertificatePublicKey(info.State.PeerCertificates[0])
+	if err != nil {
+		return nil, fmt.Errorf("invalid TLS peer certificate: %w", err)
+	}
+	return key, nil
+}