298 lines
9.8 KiB
Go
298 lines
9.8 KiB
Go
/*
|
|
Copyright IBM Corp. All Rights Reserved.
|
|
|
|
SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package comm
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"time"
|
|
|
|
"github.com/hyperledger/fabric/common/flogging"
|
|
"github.com/hyperledger/fabric/common/metrics"
|
|
"github.com/pkg/errors"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/keepalive"
|
|
)
|
|
|
|
// Configuration defaults
|
|
|
|
// Max send and receive bytes for grpc clients and servers
|
|
const (
|
|
DefaultMaxRecvMsgSize = 100 * 1024 * 1024
|
|
DefaultMaxSendMsgSize = 100 * 1024 * 1024
|
|
)
|
|
|
|
var (
|
|
// Default peer keepalive options
|
|
DefaultKeepaliveOptions = KeepaliveOptions{
|
|
ClientInterval: time.Duration(1) * time.Minute, // 1 min
|
|
ClientTimeout: time.Duration(20) * time.Second, // 20 sec - gRPC default
|
|
ServerInterval: time.Duration(2) * time.Hour, // 2 hours - gRPC default
|
|
ServerTimeout: time.Duration(20) * time.Second, // 20 sec - gRPC default
|
|
ServerMinInterval: time.Duration(1) * time.Minute, // match ClientInterval
|
|
}
|
|
// strong TLS cipher suites
|
|
DefaultTLSCipherSuites = []uint16{
|
|
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
|
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
|
|
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
|
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
|
|
tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
|
|
tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
|
|
}
|
|
// default connection timeout
|
|
DefaultConnectionTimeout = 5 * time.Second
|
|
)
|
|
|
|
// ServerConfig defines the parameters for configuring a GRPCServer instance
|
|
type ServerConfig struct {
|
|
// ConnectionTimeout specifies the timeout for connection establishment
|
|
// for all new connections
|
|
ConnectionTimeout time.Duration
|
|
// SecOpts defines the security parameters
|
|
SecOpts SecureOptions
|
|
// KaOpts defines the keepalive parameters
|
|
KaOpts KeepaliveOptions
|
|
// StreamInterceptors specifies a list of interceptors to apply to
|
|
// streaming RPCs. They are executed in order.
|
|
StreamInterceptors []grpc.StreamServerInterceptor
|
|
// UnaryInterceptors specifies a list of interceptors to apply to unary
|
|
// RPCs. They are executed in order.
|
|
UnaryInterceptors []grpc.UnaryServerInterceptor
|
|
// Logger specifies the logger the server will use
|
|
Logger *flogging.FabricLogger
|
|
// HealthCheckEnabled enables the gRPC Health Checking Protocol for the server
|
|
HealthCheckEnabled bool
|
|
// ServerStatsHandler should be set if metrics on connections are to be reported.
|
|
ServerStatsHandler *ServerStatsHandler
|
|
// Maximum message size the server can receive
|
|
MaxRecvMsgSize int
|
|
// Maximum message size the server can send
|
|
MaxSendMsgSize int
|
|
}
|
|
|
|
// ClientConfig defines the parameters for configuring a GRPCClient instance
|
|
type ClientConfig struct {
|
|
// SecOpts defines the security parameters
|
|
SecOpts SecureOptions
|
|
// KaOpts defines the keepalive parameters
|
|
KaOpts KeepaliveOptions
|
|
// DialTimeout controls how long the client can block when attempting to
|
|
// establish a connection to a server
|
|
DialTimeout time.Duration
|
|
// AsyncConnect makes connection creation non blocking
|
|
AsyncConnect bool
|
|
// Maximum message size the client can receive
|
|
MaxRecvMsgSize int
|
|
// Maximum message size the client can send
|
|
MaxSendMsgSize int
|
|
}
|
|
|
|
// Convert the ClientConfig to the approriate set of grpc.DialOptions.
|
|
func (cc ClientConfig) DialOptions() ([]grpc.DialOption, error) {
|
|
var dialOpts []grpc.DialOption
|
|
dialOpts = append(dialOpts, grpc.WithKeepaliveParams(keepalive.ClientParameters{
|
|
Time: cc.KaOpts.ClientInterval,
|
|
Timeout: cc.KaOpts.ClientTimeout,
|
|
PermitWithoutStream: true,
|
|
}))
|
|
|
|
// Unless asynchronous connect is set, make connection establishment blocking.
|
|
if !cc.AsyncConnect {
|
|
dialOpts = append(dialOpts,
|
|
grpc.WithBlock(),
|
|
grpc.FailOnNonTempDialError(true),
|
|
)
|
|
}
|
|
// set send/recv message size to package defaults
|
|
maxRecvMsgSize := DefaultMaxRecvMsgSize
|
|
if cc.MaxRecvMsgSize != 0 {
|
|
maxRecvMsgSize = cc.MaxRecvMsgSize
|
|
}
|
|
maxSendMsgSize := DefaultMaxSendMsgSize
|
|
if cc.MaxSendMsgSize != 0 {
|
|
maxSendMsgSize = cc.MaxSendMsgSize
|
|
}
|
|
dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(
|
|
grpc.MaxCallRecvMsgSize(maxRecvMsgSize),
|
|
grpc.MaxCallSendMsgSize(maxSendMsgSize),
|
|
))
|
|
|
|
tlsConfig, err := cc.SecOpts.TLSConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tlsConfig != nil {
|
|
transportCreds := &DynamicClientCredentials{TLSConfig: tlsConfig}
|
|
dialOpts = append(dialOpts, grpc.WithTransportCredentials(transportCreds))
|
|
} else {
|
|
dialOpts = append(dialOpts, grpc.WithInsecure())
|
|
}
|
|
|
|
return dialOpts, nil
|
|
}
|
|
|
|
func (cc ClientConfig) Dial(address string) (*grpc.ClientConn, error) {
|
|
dialOpts, err := cc.DialOptions()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), cc.DialTimeout)
|
|
defer cancel()
|
|
|
|
conn, err := grpc.DialContext(ctx, address, dialOpts...)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to create new connection")
|
|
}
|
|
return conn, nil
|
|
}
|
|
|
|
// Clone clones this ClientConfig
|
|
func (cc ClientConfig) Clone() ClientConfig {
|
|
shallowClone := cc
|
|
return shallowClone
|
|
}
|
|
|
|
// SecureOptions defines the TLS security parameters for a GRPCServer or
|
|
// GRPCClient instance.
|
|
type SecureOptions struct {
|
|
// VerifyCertificate, if not nil, is called after normal
|
|
// certificate verification by either a TLS client or server.
|
|
// If it returns a non-nil error, the handshake is aborted and that error results.
|
|
VerifyCertificate func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error
|
|
// PEM-encoded X509 public key to be used for TLS communication
|
|
Certificate []byte
|
|
// PEM-encoded private key to be used for TLS communication
|
|
Key []byte
|
|
// Set of PEM-encoded X509 certificate authorities used by clients to
|
|
// verify server certificates
|
|
ServerRootCAs [][]byte
|
|
// Set of PEM-encoded X509 certificate authorities used by servers to
|
|
// verify client certificates
|
|
ClientRootCAs [][]byte
|
|
// Whether or not to use TLS for communication
|
|
UseTLS bool
|
|
// Whether or not TLS client must present certificates for authentication
|
|
RequireClientCert bool
|
|
// CipherSuites is a list of supported cipher suites for TLS
|
|
CipherSuites []uint16
|
|
// TimeShift makes TLS handshakes time sampling shift to the past by a given duration
|
|
TimeShift time.Duration
|
|
// ServerNameOverride is used to verify the hostname on the returned certificates. It
|
|
// is also included in the client's handshake to support virtual hosting
|
|
// unless it is an IP address.
|
|
ServerNameOverride string
|
|
}
|
|
|
|
func (so SecureOptions) TLSConfig() (*tls.Config, error) {
|
|
// if TLS is not enabled, return
|
|
if !so.UseTLS {
|
|
return nil, nil
|
|
}
|
|
|
|
tlsConfig := &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
ServerName: so.ServerNameOverride,
|
|
VerifyPeerCertificate: so.VerifyCertificate,
|
|
}
|
|
if len(so.ServerRootCAs) > 0 {
|
|
tlsConfig.RootCAs = x509.NewCertPool()
|
|
for _, certBytes := range so.ServerRootCAs {
|
|
if !tlsConfig.RootCAs.AppendCertsFromPEM(certBytes) {
|
|
return nil, errors.New("error adding root certificate")
|
|
}
|
|
}
|
|
}
|
|
|
|
if so.RequireClientCert {
|
|
cert, err := so.ClientCertificate()
|
|
if err != nil {
|
|
return nil, errors.WithMessage(err, "failed to load client certificate")
|
|
}
|
|
tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
|
|
}
|
|
|
|
if so.TimeShift > 0 {
|
|
tlsConfig.Time = func() time.Time {
|
|
return time.Now().Add((-1) * so.TimeShift)
|
|
}
|
|
}
|
|
|
|
return tlsConfig, nil
|
|
}
|
|
|
|
// ClientCertificate returns the client certificate that will be used
|
|
// for mutual TLS.
|
|
func (so SecureOptions) ClientCertificate() (tls.Certificate, error) {
|
|
if so.Key == nil || so.Certificate == nil {
|
|
return tls.Certificate{}, errors.New("both Key and Certificate are required when using mutual TLS")
|
|
}
|
|
cert, err := tls.X509KeyPair(so.Certificate, so.Key)
|
|
if err != nil {
|
|
return tls.Certificate{}, errors.WithMessage(err, "failed to create key pair")
|
|
}
|
|
return cert, nil
|
|
}
|
|
|
|
// KeepaliveOptions is used to set the gRPC keepalive settings for both
|
|
// clients and servers
|
|
type KeepaliveOptions struct {
|
|
// ClientInterval is the duration after which if the client does not see
|
|
// any activity from the server it pings the server to see if it is alive
|
|
ClientInterval time.Duration
|
|
// ClientTimeout is the duration the client waits for a response
|
|
// from the server after sending a ping before closing the connection
|
|
ClientTimeout time.Duration
|
|
// ServerInterval is the duration after which if the server does not see
|
|
// any activity from the client it pings the client to see if it is alive
|
|
ServerInterval time.Duration
|
|
// ServerTimeout is the duration the server waits for a response
|
|
// from the client after sending a ping before closing the connection
|
|
ServerTimeout time.Duration
|
|
// ServerMinInterval is the minimum permitted time between client pings.
|
|
// If clients send pings more frequently, the server will disconnect them
|
|
ServerMinInterval time.Duration
|
|
}
|
|
|
|
// ServerKeepaliveOptions returns gRPC keepalive options for a server.
|
|
func (ka KeepaliveOptions) ServerKeepaliveOptions() []grpc.ServerOption {
|
|
var serverOpts []grpc.ServerOption
|
|
kap := keepalive.ServerParameters{
|
|
Time: ka.ServerInterval,
|
|
Timeout: ka.ServerTimeout,
|
|
}
|
|
serverOpts = append(serverOpts, grpc.KeepaliveParams(kap))
|
|
kep := keepalive.EnforcementPolicy{
|
|
MinTime: ka.ServerMinInterval,
|
|
// allow keepalive w/o rpc
|
|
PermitWithoutStream: true,
|
|
}
|
|
serverOpts = append(serverOpts, grpc.KeepaliveEnforcementPolicy(kep))
|
|
return serverOpts
|
|
}
|
|
|
|
// ClientKeepaliveOptions returns gRPC keepalive dial options for clients.
|
|
func (ka KeepaliveOptions) ClientKeepaliveOptions() []grpc.DialOption {
|
|
var dialOpts []grpc.DialOption
|
|
kap := keepalive.ClientParameters{
|
|
Time: ka.ClientInterval,
|
|
Timeout: ka.ClientTimeout,
|
|
PermitWithoutStream: true,
|
|
}
|
|
dialOpts = append(dialOpts, grpc.WithKeepaliveParams(kap))
|
|
return dialOpts
|
|
}
|
|
|
|
type Metrics struct {
|
|
// OpenConnCounter keeps track of number of open connections
|
|
OpenConnCounter metrics.Counter
|
|
// ClosedConnCounter keeps track of number connections closed
|
|
ClosedConnCounter metrics.Counter
|
|
}
|